From fijal at codespeak.net Mon Oct 1 08:57:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 1 Oct 2007 08:57:51 +0200 (CEST) Subject: [pypy-svn] r47055 - in pypy/dist/pypy/module/rctime: . test Message-ID: <20071001065751.2B2B88125@code0.codespeak.net> Author: fijal Date: Mon Oct 1 08:57:49 2007 New Revision: 47055 Modified: pypy/dist/pypy/module/rctime/__init__.py pypy/dist/pypy/module/rctime/app_time.py pypy/dist/pypy/module/rctime/interp_time.py pypy/dist/pypy/module/rctime/test/test_rctime.py Log: Move rctime from rctypes to rffi (name makes no longer sense). Next step: * unify with time * check locking Modified: pypy/dist/pypy/module/rctime/__init__.py ============================================================================== --- pypy/dist/pypy/module/rctime/__init__.py (original) +++ pypy/dist/pypy/module/rctime/__init__.py Mon Oct 1 08:57:49 2007 @@ -18,6 +18,7 @@ 'localtime': 'interp_time.localtime', 'mktime': 'interp_time.mktime', 'strftime': 'interp_time.strftime', + 'sleep' : 'interp_time.sleep', } def buildloaders(cls): @@ -25,10 +26,7 @@ import os if os.name == "posix": - Module.appleveldefs['sleep'] = 'app_time.sleep' Module.interpleveldefs['tzset'] = 'interp_time.tzset' - elif os.name == "nt": - Module.interpleveldefs['sleep'] = 'interp_time.sleep' # this machinery is needed to expose constants # that have to be initialized one time only Modified: pypy/dist/pypy/module/rctime/app_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/app_time.py (original) +++ pypy/dist/pypy/module/rctime/app_time.py Mon Oct 1 08:57:49 2007 @@ -19,20 +19,6 @@ tm_yday = structseqfield(7) tm_isdst = structseqfield(8) - -if _POSIX: - from select import select - - def sleep(secs): - """sleep(seconds) - - Delay execution for a given number of seconds. The argument may be - a floating point number for subsecond precision.""" - if secs is None: - raise TypeError('a float is required') - select([], [], [], secs) - - def strptime(string, format="%a %b %d %H:%M:%S %Y"): """strptime(string, format) -> struct_time Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Mon Oct 1 08:57:49 2007 @@ -1,54 +1,55 @@ -from pypy.rpython.rctypes.tool import ctypes_platform -from pypy.rpython.rctypes.tool.libc import libc -import pypy.rpython.rctypes.implementation # this defines rctypes magic -from pypy.rpython.rctypes.aerrno import geterrno +from pypy.rpython.tool import rffi_platform as platform +from pypy.rpython.lltypesystem import rffi from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import W_Root, ObjSpace +from pypy.rpython.lltypesystem import lltype from pypy.rlib.rarithmetic import ovfcheck_float_to_int -from ctypes import * import math import os import sys +import time as pytime _POSIX = os.name == "posix" _WIN = os.name == "nt" -_include = "#include " +_includes = ["time.h"] if _POSIX: - _include = """%s - #include """ % _include - + _includes.append('sys/time.h') + class CConfig: - _header_ = _include - CLOCKS_PER_SEC = ctypes_platform.ConstantInteger("CLOCKS_PER_SEC") - clock_t = ctypes_platform.SimpleType("clock_t", c_ulong) - time_t = ctypes_platform.SimpleType("time_t", c_long) - size_t = ctypes_platform.SimpleType("size_t", c_long) + _includes_ = _includes + CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC") + clock_t = platform.SimpleType("clock_t", rffi.ULONG) + time_t = platform.SimpleType("time_t", rffi.LONG) + size_t = platform.SimpleType("size_t", rffi.LONG) + has_gettimeofday = platform.Has('gettimeofday') if _POSIX: - CConfig.timeval = ctypes_platform.Struct("struct timeval", [("tv_sec", c_int), - ("tv_usec", c_int)]) - CConfig.tm = ctypes_platform.Struct("struct tm", [("tm_sec", c_int), - ("tm_min", c_int), ("tm_hour", c_int), ("tm_mday", c_int), - ("tm_mon", c_int), ("tm_year", c_int), ("tm_wday", c_int), - ("tm_yday", c_int), ("tm_isdst", c_int), ("tm_gmtoff", c_long), - ("tm_zone", c_char_p)]) + calling_conv = 'c' + CConfig.timeval = platform.Struct("struct timeval", + [("tv_sec", rffi.INT), + ("tv_usec", rffi.INT)]) + CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT), + ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT), + ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT), + ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT), ("tm_gmtoff", rffi.LONG), + ("tm_zone", rffi.CCHARP)]) elif _WIN: - from ctypes import wintypes - CConfig.tm = ctypes_platform.Struct("struct tm", [("tm_sec", c_int), - ("tm_min", c_int), ("tm_hour", c_int), ("tm_mday", c_int), - ("tm_mon", c_int), ("tm_year", c_int), ("tm_wday", c_int), - ("tm_yday", c_int), ("tm_isdst", c_int)]) - - LARGE_INTEGER = wintypes.LARGE_INTEGER - BOOL = wintypes.BOOL - DWORD = wintypes.DWORD + calling_conf = 'win' + CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT), + ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT), + ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT), + ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)]) class cConfig: pass -cConfig.__dict__.update(ctypes_platform.configure(CConfig)) +cConfig.__dict__.update(platform.configure(CConfig)) cConfig.tm.__name__ = "_tm" +def external(name, args, result): + return rffi.llexternal(name, args, result, + includes=_includes, calling_conv=calling_conv) + if _POSIX: cConfig.timeval.__name__ = "_timeval" timeval = cConfig.timeval @@ -58,40 +59,35 @@ time_t = cConfig.time_t size_t = cConfig.size_t tm = cConfig.tm +glob_buf = lltype.malloc(tm, flavor='raw') -has_gettimeofday = False -if hasattr(libc, "gettimeofday"): - libc.gettimeofday.argtypes = [c_void_p, c_void_p] - libc.gettimeofday.restype = c_int - has_gettimeofday = True -libc.clock.restype = clock_t -libc.time.argtypes = [POINTER(time_t)] -libc.time.restype = time_t -libc.ctime.argtypes = [POINTER(time_t)] -libc.ctime.restype = c_char_p -libc.gmtime.argtypes = [POINTER(time_t)] -libc.gmtime.restype = POINTER(tm) -libc.localtime.argtypes = [POINTER(time_t)] -libc.localtime.restype = POINTER(tm) -libc.mktime.argtypes = [POINTER(tm)] -libc.mktime.restype = time_t -libc.asctime.argtypes = [POINTER(tm)] -libc.asctime.restype = c_char_p +if cConfig.has_gettimeofday: + c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT) +TIME_TP = lltype.Ptr(rffi.CFixedArray(time_t, 1)) +TM_P = lltype.Ptr(tm) +c_clock = external('clock', [TIME_TP], clock_t) +c_time = external('time', [TIME_TP], time_t) +c_ctime = external('ctime', [TIME_TP], rffi.CCHARP) +c_gmtime = external('gmtime', [TIME_TP], TM_P) +c_mktime = external('mktime', [TM_P], time_t) +c_asctime = external('asctime', [TM_P], rffi.CCHARP) +c_localtime = external('localtime', [TIME_TP], TM_P) if _POSIX: - libc.tzset.restype = None # tzset() returns void + c_tzset = external('tzset', [], lltype.Void) elif _WIN: - QueryPerformanceCounter = windll.kernel32.QueryPerformanceCounter - QueryPerformanceCounter.argtypes = [POINTER(LARGE_INTEGER)] - QueryPerformanceCounter.restype = BOOL - QueryPerformanceFrequency = windll.kernel32.QueryPerformanceFrequency - QueryPerformanceFrequency.argtypes = [POINTER(LARGE_INTEGER)] - QueryPerformanceFrequency.restype = BOOL - Sleep = windll.kernel32.Sleep - Sleep.argtypes = [DWORD] - Sleep.restype = None -libc.strftime.argtypes = [c_char_p, size_t, c_char_p, POINTER(tm)] -libc.strftime.restype = size_t - + XXX + #c_QueryPerformanceCounter = external( + #QueryPerformanceCounter = windll.kernel32.QueryPerformanceCounter + #QueryPerformanceCounter.argtypes = [POINTER(LARGE_INTEGER)] + #QueryPerformanceCounter.restype = BOOL + #QueryPerformanceFrequency = windll.kernel32.QueryPerformanceFrequency + #QueryPerformanceFrequency.argtypes = [POINTER(LARGE_INTEGER)] + #QueryPerformanceFrequency.restype = BOOL + #Sleep = windll.kernel32.Sleep + #Sleep.argtypes = [DWORD] + #Sleep.restype = None +c_strftime = external('strftime', [rffi.CCHARP, size_t, rffi.CCHARP, TM_P], + size_t) def _init_accept2dyear(): return (1, 0)[bool(os.getenv("PYTHONY2K"))] @@ -117,81 +113,42 @@ if _POSIX: YEAR = (365 * 24 + 6) * 3600 - t = (((libc.time(byref(time_t(0)))) / YEAR) * YEAR) - tt = time_t(t) - p = libc.localtime(byref(tt)).contents - janzone = -p.tm_gmtoff - janname = [" ", p.tm_zone][bool(p.tm_zone)] - tt = time_t(tt.value + YEAR / 2) - p = libc.localtime(byref(tt)).contents - julyzone = -p.tm_gmtoff - julyname = [" ", p.tm_zone][bool(p.tm_zone)] + t = (((c_time(lltype.nullptr(TIME_TP.TO))) / YEAR) * YEAR) + # we cannot have reference to stack variable, put it on the heap + t_ref = lltype.malloc(TIME_TP.TO, flavor='raw') + t_ref[0] = t + p = c_localtime(t_ref) + janzone = -p.c_tm_gmtoff + janname = [" ", p.c_tm_zone][bool(p.c_tm_zone)] + tt = t + YEAR / 2 + t_ref[0] = tt + p = c_localtime(t_ref) + lltype.free(t_ref, flavor='raw') + julyzone = -p.c_tm_gmtoff + julyname = [" ", p.c_tm_zone][bool(p.c_tm_zone)] if janzone < julyzone: # DST is reversed in the southern hemisphere timezone = julyzone altzone = janzone daylight = int(janzone != julyzone) - tzname = [julyname, janname] + tzname = (julyname, janname) else: timezone = janzone altzone = julyzone daylight = int(janzone != julyzone) - tzname = [janname, julyname] - - return timezone, daylight, tzname, altzone + tzname = (janname, julyname) + start, end = tzname + + return timezone, daylight, [rffi.charp2str(start), rffi.charp2str(end)], altzone def _get_error_msg(): - errno = geterrno() + errno = rff.get_errno() return os.strerror(errno) - -def _floattime(): - """ _floattime() -> computes time since the Epoch for various platforms. - - Since on some system gettimeofday may fail we fall back on ftime - or time. - gettimeofday() has a resolution in microseconds - ftime() has a resolution in milliseconds and it never fails - time() has a resolution in seconds - """ - - if has_gettimeofday: - t = timeval() - if libc.gettimeofday(byref(t), c_void_p(None)) == 0: - return float(t.tv_sec) + t.tv_usec * 0.000001 - - return float(libc.time(None)) - - # elif hasattr(_libc, "ftime"): - # t = _timeb() - # _libc.ftime.argtypes = [c_void_p] - # _libc.ftime(byref(t)) - # return float(t.time) + float(t.millitm) * 0.001 - # elif hasattr(_libc, "time"): - # t = c_long() - # _libc.time.argtypes = [c_void_p] - # _libc.time(byref(t)) - # return t.value - -if _WIN: - def sleep(space, w_secs): - """sleep(seconds) - - Delay execution for a given number of seconds. The argument may be - a floating point number for subsecond precision.""" - - secs = space.float_w(w_secs) - if secs < 0.0: - secs = 0.0 - msecs = secs * 1000.0 - try: - msecs = ovfcheck_float_to_int(msecs) - except OverflowError: - raise OperationError(space.w_OverflowError, - space.wrap("sleep length is too large")) - ul_millis = c_ulong(msecs) - Sleep(ul_millis) +def sleep(secs): + pytime.sleep(secs) +sleep.unwrap_spec = [float] def _get_module_object(space, obj_name): w_module = space.getbuiltinmodule('time') @@ -206,7 +163,7 @@ # w_seconds can be a wrapped None (it will be automatically wrapped # in the callers, so we never get a real None here). if space.is_w(w_seconds, space.w_None): - seconds = _floattime() + seconds = pytime.time() else: seconds = space.float_w(w_seconds) try: @@ -218,15 +175,15 @@ def _tm_to_tuple(space, t): time_tuple = [] - time_tuple.append(space.wrap(t.tm_year + 1900)) - time_tuple.append(space.wrap(t.tm_mon + 1)) # want january == 1 - time_tuple.append(space.wrap(t.tm_mday)) - time_tuple.append(space.wrap(t.tm_hour)) - time_tuple.append(space.wrap(t.tm_min)) - time_tuple.append(space.wrap(t.tm_sec)) - time_tuple.append(space.wrap((t.tm_wday + 6) % 7)) # want monday == 0 - time_tuple.append(space.wrap(t.tm_yday + 1)) # want january, 1 == 1 - time_tuple.append(space.wrap(t.tm_isdst)) + time_tuple.append(space.wrap(t.c_tm_year + 1900)) + time_tuple.append(space.wrap(t.c_tm_mon + 1)) # want january == 1 + time_tuple.append(space.wrap(t.c_tm_mday)) + time_tuple.append(space.wrap(t.c_tm_hour)) + time_tuple.append(space.wrap(t.c_tm_min)) + time_tuple.append(space.wrap(t.c_tm_sec)) + time_tuple.append(space.wrap((t.c_tm_wday + 6) % 7)) # want monday == 0 + time_tuple.append(space.wrap(t.c_tm_yday + 1)) # want january, 1 == 1 + time_tuple.append(space.wrap(t.c_tm_isdst)) w_struct_time = _get_module_object(space, 'struct_time') w_time_tuple = space.newtuple(time_tuple) @@ -235,12 +192,15 @@ def _gettmarg(space, w_tup, allowNone=True): if allowNone and space.is_w(w_tup, space.w_None): # default to the current local time - tt = time_t(int(_floattime())) - pbuf = libc.localtime(byref(tt)) + tt = int(pytime.time()) + t_ref = lltype.malloc(TIME_TP.TO, flavor='raw') + t_ref[0] = tt + pbuf = c_localtime(t_ref) + lltype.free(t_ref, flavor='raw') if not pbuf: raise OperationError(space.w_ValueError, space.wrap(_get_error_msg())) - return pbuf.contents + return pbuf tup_w = space.unpackiterable(w_tup) if len(tup_w) != 9: @@ -248,16 +208,16 @@ space.wrap("argument must be sequence of " "length 9, not %d" % len(tup_w))) - buf = tm() y = space.int_w(tup_w[0]) - buf.tm_mon = space.int_w(tup_w[1]) - buf.tm_mday = space.int_w(tup_w[2]) - buf.tm_hour = space.int_w(tup_w[3]) - buf.tm_min = space.int_w(tup_w[4]) - buf.tm_sec = space.int_w(tup_w[5]) - buf.tm_wday = space.int_w(tup_w[6]) - buf.tm_yday = space.int_w(tup_w[7]) - buf.tm_isdst = space.int_w(tup_w[8]) + glob_buf.c_tm_mon = space.int_w(tup_w[1]) + glob_buf.c_tm_mday = space.int_w(tup_w[2]) + glob_buf.c_tm_hour = space.int_w(tup_w[3]) + glob_buf.c_tm_min = space.int_w(tup_w[4]) + glob_buf.c_tm_sec = space.int_w(tup_w[5]) + glob_buf.c_tm_wday = space.int_w(tup_w[6]) + glob_buf.c_tm_yday = space.int_w(tup_w[7]) + glob_buf.c_tm_isdst = space.int_w(tup_w[8]) + glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO) w_accept2dyear = _get_module_object(space, "accept2dyear") accept2dyear = space.int_w(w_accept2dyear) @@ -275,16 +235,16 @@ raise OperationError(space.w_ValueError, space.wrap("year out of range")) - if buf.tm_wday < 0: + if glob_buf.c_tm_wday < 0: raise OperationError(space.w_ValueError, space.wrap("day of week out of range")) - buf.tm_year = y - 1900 - buf.tm_mon = buf.tm_mon - 1 - buf.tm_wday = (buf.tm_wday + 1) % 7 - buf.tm_yday = buf.tm_yday - 1 + glob_buf.c_tm_year = y - 1900 + glob_buf.c_tm_mon = glob_buf.c_tm_mon - 1 + glob_buf.c_tm_wday = (glob_buf.c_tm_wday + 1) % 7 + glob_buf.c_tm_yday = glob_buf.c_tm_yday - 1 - return buf + return glob_buf def time(space): """time() -> floating point number @@ -292,7 +252,7 @@ Return the current time in seconds since the Epoch. Fractions of a second may be present if the system clock provides them.""" - secs = _floattime() + secs = pytime.time() return space.wrap(secs) if _WIN: @@ -300,7 +260,7 @@ pass pccache = PCCache() pccache.divisor = 0.0 - pccache.ctrStart = LARGE_INTEGER() + pccache.ctrStart = 0 def clock(space): """clock() -> floating point number @@ -309,22 +269,7 @@ the first call to clock(). This has as much precision as the system records.""" - if _POSIX: - res = float(libc.clock()) / CLOCKS_PER_SEC - return space.wrap(res) - elif _WIN: - if pccache.divisor == 0.0: - freq = LARGE_INTEGER() - res = QueryPerformanceFrequency(byref(freq)) - if not res or not freq: - return space.wrap(float(libc.clock()) / CLOCKS_PER_SEC) - pccache.divisor = float(freq.value) - QueryPerformanceCounter(byref(pccache.ctrStart)) - - now = LARGE_INTEGER() - QueryPerformanceCounter(byref(now)) - diff = float(now.value - pccache.ctrStart.value) - return space.wrap(diff / pccache.divisor) + return space.wrap(pytime.clock()) def ctime(space, w_seconds=None): """ctime([seconds]) -> string @@ -334,14 +279,16 @@ not present, current time as returned by localtime() is used.""" seconds = _get_inttime(space, w_seconds) - tt = time_t(seconds) - - p = libc.ctime(byref(tt)) + + t_ref = lltype.malloc(TIME_TP.TO, flavor='raw') + t_ref[0] = seconds + p = c_ctime(t_ref) + lltype.free(t_ref, flavor='raw') if not p: raise OperationError(space.w_ValueError, space.wrap("unconvertible time")) - return space.wrap(p[:-1]) # get rid of new line + return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line ctime.unwrap_spec = [ObjSpace, W_Root] # by now w_tup is an optional argument (and not *args) @@ -353,12 +300,12 @@ When the time tuple is not present, current time as returned by localtime() is used.""" buf_value = _gettmarg(space, w_tup) - p = libc.asctime(byref(buf_value)) + p = c_asctime(buf_value) if not p: raise OperationError(space.w_ValueError, space.wrap("unconvertible time")) - return space.wrap(p[:-1]) # get rid of new line + return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line asctime.unwrap_spec = [ObjSpace, W_Root] def gmtime(space, w_seconds=None): @@ -372,12 +319,14 @@ # rpython does not support that a variable has two incompatible builtins # as value so we have to duplicate the code. NOT GOOD! see localtime() too seconds = _get_inttime(space, w_seconds) - whent = time_t(seconds) - p = libc.gmtime(byref(whent)) + t_ref = lltype.malloc(TIME_TP.TO, flavor='raw') + t_ref[0] = seconds + p = c_gmtime(t_ref) + lltype.free(t_ref, flavor='raw') if not p: raise OperationError(space.w_ValueError, space.wrap(_get_error_msg())) - return _tm_to_tuple(space, p.contents) + return _tm_to_tuple(space, p) gmtime.unwrap_spec = [ObjSpace, W_Root] def localtime(space, w_seconds=None): @@ -388,12 +337,14 @@ When 'seconds' is not passed in, convert the current time instead.""" seconds = _get_inttime(space, w_seconds) - whent = time_t(seconds) - p = libc.localtime(byref(whent)) + t_ref = lltype.malloc(TIME_TP.TO, flavor='raw') + t_ref[0] = seconds + p = c_localtime(t_ref) + lltype.free(t_ref, flavor='raw') if not p: raise OperationError(space.w_ValueError, space.wrap(_get_error_msg())) - return _tm_to_tuple(space, p.contents) + return _tm_to_tuple(space, p) localtime.unwrap_spec = [ObjSpace, W_Root] def mktime(space, w_tup): @@ -402,7 +353,7 @@ Convert a time tuple in local time to seconds since the Epoch.""" buf = _gettmarg(space, w_tup, allowNone=False) - tt = libc.mktime(byref(buf)) + tt = c_mktime(buf) if tt == -1: raise OperationError(space.w_OverflowError, space.wrap("mktime argument out of range")) @@ -424,7 +375,7 @@ the local timezone used by methods such as localtime, but this behaviour should not be relied on""" - libc.tzset() + c_tzset() # reset timezone, altzone, daylight and tzname timezone, daylight, tzname, altzone = _init_timezone() @@ -441,40 +392,39 @@ Convert a time tuple to a string according to a format specification. See the library reference manual for formatting codes. When the time tuple is not present, current time as returned by localtime() is used.""" - buf_value = _gettmarg(space, w_tup) - + # Checks added to make sure strftime() does not crash Python by # indexing blindly into some array for a textual representation # by some bad index (fixes bug #897625). # No check for year since handled in gettmarg(). - if buf_value.tm_mon < 0 or buf_value.tm_mon > 11: + if buf_value.c_tm_mon < 0 or buf_value.c_tm_mon > 11: raise OperationError(space.w_ValueError, - space.wrap("month out of range")) - if buf_value.tm_mday < 1 or buf_value.tm_mday > 31: + space.wrap("month out of range")) + if buf_value.c_tm_mday < 1 or buf_value.c_tm_mday > 31: raise OperationError(space.w_ValueError, - space.wrap("day of month out of range")) - if buf_value.tm_hour < 0 or buf_value.tm_hour > 23: + space.wrap("day of month out of range")) + if buf_value.c_tm_hour < 0 or buf_value.c_tm_hour > 23: raise OperationError(space.w_ValueError, - space.wrap("hour out of range")) - if buf_value.tm_min < 0 or buf_value.tm_min > 59: + space.wrap("hour out of range")) + if buf_value.c_tm_min < 0 or buf_value.c_tm_min > 59: raise OperationError(space.w_ValueError, - space.wrap("minute out of range")) - if buf_value.tm_sec < 0 or buf_value.tm_sec > 61: + space.wrap("minute out of range")) + if buf_value.c_tm_sec < 0 or buf_value.c_tm_sec > 61: raise OperationError(space.w_ValueError, - space.wrap("seconds out of range")) - if buf_value.tm_yday < 0 or buf_value.tm_yday > 365: + space.wrap("seconds out of range")) + if buf_value.c_tm_yday < 0 or buf_value.c_tm_yday > 365: raise OperationError(space.w_ValueError, - space.wrap("day of year out of range")) - if buf_value.tm_isdst < -1 or buf_value.tm_isdst > 1: + space.wrap("day of year out of range")) + if buf_value.c_tm_isdst < -1 or buf_value.c_tm_isdst > 1: raise OperationError(space.w_ValueError, - space.wrap("daylight savings flag out of range")) + space.wrap("daylight savings flag out of range")) i = 1024 while True: - outbuf = create_string_buffer(i) - buflen = libc.strftime(outbuf, i, format, byref(buf_value)) - + outbuf = lltype.malloc(rffi.CCHARP.TO, i + 1, flavor='raw') + buflen = c_strftime(outbuf, i, format, buf_value) + if buflen > 0 or i >= 256 * len(format): # if the buffer is 256 times as long as the format, # it's probably not failing for lack of room! @@ -482,7 +432,10 @@ # e.g. an empty format, or %Z when the timezone # is unknown. if buflen < 0: buflen = 0 # should not occur - return space.wrap(outbuf.value[:buflen]) + outbuf[buflen] = '\x00' + result = rffi.charp2str(outbuf) + lltype.free(outbuf, flavor='raw') + return space.wrap(result) i += i strftime.unwrap_spec = [ObjSpace, str, W_Root] Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py ============================================================================== --- pypy/dist/pypy/module/rctime/test/test_rctime.py (original) +++ pypy/dist/pypy/module/rctime/test/test_rctime.py Mon Oct 1 08:57:49 2007 @@ -198,7 +198,7 @@ elif os.environ.has_key('TZ'): del os.environ['TZ'] rctime.tzset() - + def test_strftime(self): import time as rctime From fijal at codespeak.net Mon Oct 1 09:09:01 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 1 Oct 2007 09:09:01 +0200 (CEST) Subject: [pypy-svn] r47056 - pypy/dist/pypy/module/rctime Message-ID: <20071001070901.23E0C8125@code0.codespeak.net> Author: fijal Date: Mon Oct 1 09:08:59 2007 New Revision: 47056 Modified: pypy/dist/pypy/module/rctime/interp_time.py Log: Fix translation (thanks checkmodule) Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Mon Oct 1 09:08:59 2007 @@ -119,31 +119,31 @@ t_ref[0] = t p = c_localtime(t_ref) janzone = -p.c_tm_gmtoff - janname = [" ", p.c_tm_zone][bool(p.c_tm_zone)] + tm_zone = rffi.charp2str(p.c_tm_zone) + janname = [" ", tm_zone][bool(tm_zone)] tt = t + YEAR / 2 t_ref[0] = tt p = c_localtime(t_ref) lltype.free(t_ref, flavor='raw') julyzone = -p.c_tm_gmtoff - julyname = [" ", p.c_tm_zone][bool(p.c_tm_zone)] + julyname = [" ", tm_zone][bool(tm_zone)] if janzone < julyzone: # DST is reversed in the southern hemisphere timezone = julyzone altzone = janzone daylight = int(janzone != julyzone) - tzname = (julyname, janname) + tzname = [julyname, janname] else: timezone = janzone altzone = julyzone daylight = int(janzone != julyzone) - tzname = (janname, julyname) - start, end = tzname + tzname = [janname, julyname] - return timezone, daylight, [rffi.charp2str(start), rffi.charp2str(end)], altzone + return timezone, daylight, tzname, altzone def _get_error_msg(): - errno = rff.get_errno() + errno = rffi.get_errno() return os.strerror(errno) def sleep(secs): From fijal at codespeak.net Mon Oct 1 09:17:12 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 1 Oct 2007 09:17:12 +0200 (CEST) Subject: [pypy-svn] r47057 - pypy/dist/pypy/rpython/module Message-ID: <20071001071712.A98498129@code0.codespeak.net> Author: fijal Date: Mon Oct 1 09:17:12 2007 New Revision: 47057 Modified: pypy/dist/pypy/rpython/module/ll_time.py Log: add threadsafe=False flag to most of the low level time interface Modified: pypy/dist/pypy/rpython/module/ll_time.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_time.py (original) +++ pypy/dist/pypy/rpython/module/ll_time.py Mon Oct 1 09:17:12 2007 @@ -56,11 +56,11 @@ if self.GETTIMEOFDAY_NO_TZ: c_gettimeofday = self.llexternal('gettimeofday', [self.TIMEVALP], rffi.INT, - _nowrapper=True) + _nowrapper=True, threadsafe=False) else: c_gettimeofday = self.llexternal('gettimeofday', [self.TIMEVALP, rffi.VOIDP], rffi.INT, - _nowrapper=True) + _nowrapper=True, threadsafe=False) else: c_gettimeofday = None @@ -68,12 +68,12 @@ self.configure(CConfigForFTime) c_ftime = self.llexternal('ftime', [lltype.Ptr(self.TIMEB)], lltype.Void, - _nowrapper=True) + _nowrapper=True, threadsafe=False) else: c_ftime = None # to not confuse the flow space c_time = self.llexternal('time', [rffi.VOIDP], self.TIME_T, - _nowrapper=True) + _nowrapper=True, threadsafe=False) def time_time_llimpl(): void = lltype.nullptr(rffi.VOIDP.TO) @@ -105,14 +105,17 @@ @registering(time.clock) def register_time_clock(self): - c_clock = self.llexternal('clock', [], self.CLOCK_T) + c_clock = self.llexternal('clock', [], self.CLOCK_T, + threadsafe=False) if sys.platform == 'win32': # hacking to avoid LARGE_INTEGER which is a union... A = lltype.FixedSizeArray(lltype.SignedLongLong, 1) QueryPerformanceCounter = self.llexternal( - 'QueryPerformanceCounter', [lltype.Ptr(A)], lltype.Void) + 'QueryPerformanceCounter', [lltype.Ptr(A)], lltype.Void, + threadsafe=False) QueryPerformanceFrequency = self.llexternal( - 'QueryPerformanceFrequency', [lltype.Ptr(A)], rffi.INT) + 'QueryPerformanceFrequency', [lltype.Ptr(A)], rffi.INT, + threadsafe=False) class State(object): pass state = State() From fijal at codespeak.net Mon Oct 1 09:18:17 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 1 Oct 2007 09:18:17 +0200 (CEST) Subject: [pypy-svn] r47058 - pypy/dist/pypy/module/rctime Message-ID: <20071001071817.86AB18129@code0.codespeak.net> Author: fijal Date: Mon Oct 1 09:18:17 2007 New Revision: 47058 Modified: pypy/dist/pypy/module/rctime/interp_time.py Log: all externals defined here are not thread safe (sleep is defined somewhere else) Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Mon Oct 1 09:18:17 2007 @@ -48,7 +48,8 @@ def external(name, args, result): return rffi.llexternal(name, args, result, - includes=_includes, calling_conv=calling_conv) + includes=_includes, calling_conv=calling_conv, + threadsafe=False) if _POSIX: cConfig.timeval.__name__ = "_timeval" From cfbolz at codespeak.net Mon Oct 1 09:52:04 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 1 Oct 2007 09:52:04 +0200 (CEST) Subject: [pypy-svn] r47059 - pypy/dist/pypy/module/crypt Message-ID: <20071001075204.EC6798120@code0.codespeak.net> Author: cfbolz Date: Mon Oct 1 09:52:04 2007 New Revision: 47059 Modified: pypy/dist/pypy/module/crypt/interp_crypt.py Log: make crypt use rffi instead of rctypes. test on darwin still msissing. Modified: pypy/dist/pypy/module/crypt/interp_crypt.py ============================================================================== --- pypy/dist/pypy/module/crypt/interp_crypt.py (original) +++ pypy/dist/pypy/module/crypt/interp_crypt.py Mon Oct 1 09:52:04 2007 @@ -1,26 +1,19 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.rpython.rctypes.tool import ctypes_platform -from pypy.rpython.rctypes.tool.util import find_library, load_library +from pypy.rpython.tool import rffi_platform as platform +from pypy.rpython.rctypes.tool.util import find_library +from pypy.rpython.lltypesystem import rffi, lltype import sys -from ctypes import * -class CConfig: - _includes_ = ('unistd.h',) - if sys.platform != 'darwin': - cryptlib = ctypes_platform.Library('crypt') +#if sys.platform == 'darwin': +# dllname = find_library('c') +# assert dllname is not None +# cryptlib = cdll.LoadLibrary(dllname) -globals().update(ctypes_platform.configure(CConfig)) - -if sys.platform == 'darwin': - dllname = find_library('c') - assert dllname is not None - cryptlib = cdll.LoadLibrary(dllname) - -c_crypt = cryptlib.crypt -c_crypt.argtypes = [c_char_p, c_char_p] -c_crypt.restype = c_char_p +c_crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP, + libraries=["crypt"]) def crypt(space, word, salt): """word will usually be a user's password. salt is a 2-character string @@ -29,6 +22,9 @@ the hashed password as a string, which will be composed of characters from the same alphabet as the salt.""" res = c_crypt(word, salt) - return space.wrap(res) + if not res: + return space.w_None + str_res = rffi.charp2str(res) + return space.wrap(str_res) crypt.unwrap_spec = [ObjSpace, str, str] From cfbolz at codespeak.net Mon Oct 1 10:13:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 1 Oct 2007 10:13:28 +0200 (CEST) Subject: [pypy-svn] r47060 - pypy/dist/pypy/module/crypt Message-ID: <20071001081328.51D7F8124@code0.codespeak.net> Author: cfbolz Date: Mon Oct 1 10:13:27 2007 New Revision: 47060 Modified: pypy/dist/pypy/module/crypt/interp_crypt.py Log: it seems crypt is not threadsafe (if I interpret yesterdays IRC discussions right) Modified: pypy/dist/pypy/module/crypt/interp_crypt.py ============================================================================== --- pypy/dist/pypy/module/crypt/interp_crypt.py (original) +++ pypy/dist/pypy/module/crypt/interp_crypt.py Mon Oct 1 10:13:27 2007 @@ -13,7 +13,7 @@ # cryptlib = cdll.LoadLibrary(dllname) c_crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP, - libraries=["crypt"]) + libraries=["crypt"], threadsafe=False) def crypt(space, word, salt): """word will usually be a user's password. salt is a 2-character string From fijal at codespeak.net Mon Oct 1 10:41:44 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 1 Oct 2007 10:41:44 +0200 (CEST) Subject: [pypy-svn] r47061 - pypy/dist/pypy/module/rctime Message-ID: <20071001084144.E545780F8@code0.codespeak.net> Author: fijal Date: Mon Oct 1 10:41:44 2007 New Revision: 47061 Modified: pypy/dist/pypy/module/rctime/interp_time.py Log: Make annotator happier Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Mon Oct 1 10:41:44 2007 @@ -218,7 +218,10 @@ glob_buf.c_tm_wday = space.int_w(tup_w[6]) glob_buf.c_tm_yday = space.int_w(tup_w[7]) glob_buf.c_tm_isdst = space.int_w(tup_w[8]) - glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO) + if _POSIX: + # actually never happens, but makes annotator happy + glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO) + glob_buf.c_tm_gmtoff = 0 w_accept2dyear = _get_module_object(space, "accept2dyear") accept2dyear = space.int_w(w_accept2dyear) From antocuni at codespeak.net Mon Oct 1 10:48:19 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 1 Oct 2007 10:48:19 +0200 (CEST) Subject: [pypy-svn] r47062 - pypy/dist/pypy/module/rbench Message-ID: <20071001084819.E55D180F8@code0.codespeak.net> Author: antocuni Date: Mon Oct 1 10:48:19 2007 New Revision: 47062 Added: pypy/dist/pypy/module/rbench/ pypy/dist/pypy/module/rbench/__init__.py (contents, props changed) pypy/dist/pypy/module/rbench/app_bench.py (contents, props changed) Log: this module contains geninterpreted version of pystone and richards. Useful to measure the interpretation overhead of the various pypy-*. Added: pypy/dist/pypy/module/rbench/__init__.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/rbench/__init__.py Mon Oct 1 10:48:19 2007 @@ -0,0 +1,12 @@ +# Package initialisation +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + """geninterpreted benchmarks""" + + appleveldefs = { + 'pystone': 'app_bench.pystone', + 'richards': 'app_bench.richards', + } + + interpleveldefs = {} Added: pypy/dist/pypy/module/rbench/app_bench.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/rbench/app_bench.py Mon Oct 1 10:48:19 2007 @@ -0,0 +1,2 @@ +from pypy.translator.test.rpystone import main as pystone +from pypy.translator.goal.richards import main as richards From fijal at codespeak.net Mon Oct 1 10:53:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 1 Oct 2007 10:53:54 +0200 (CEST) Subject: [pypy-svn] r47063 - pypy/dist/pypy/config Message-ID: <20071001085354.8EA7380FE@code0.codespeak.net> Author: fijal Date: Mon Oct 1 10:53:54 2007 New Revision: 47063 Modified: pypy/dist/pypy/config/pypyoption.py Log: Enable tproxy by default Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Mon Oct 1 10:53:54 2007 @@ -115,7 +115,7 @@ OptionDescription("std", "Standard Object Space Options", [ BoolOption("withtproxy", "support transparent proxies", - default=False), + default=True), BoolOption("withsmallint", "use tagged integers", default=False, From cfbolz at codespeak.net Mon Oct 1 10:56:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 1 Oct 2007 10:56:47 +0200 (CEST) Subject: [pypy-svn] r47064 - pypy/dist/pypy/doc Message-ID: <20071001085647.6673B80FE@code0.codespeak.net> Author: cfbolz Date: Mon Oct 1 10:56:47 2007 New Revision: 47064 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: crypt uses rffi now Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Mon Oct 1 10:56:47 2007 @@ -34,7 +34,6 @@ - rewrite the following rctypes modules using rffi: - bz2 - - crypt - _demo - _ssl - fcntl From cfbolz at codespeak.net Mon Oct 1 11:13:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 1 Oct 2007 11:13:12 +0200 (CEST) Subject: [pypy-svn] r47065 - pypy/dist/pypy/doc Message-ID: <20071001091312.CFB458124@code0.codespeak.net> Author: cfbolz Date: Mon Oct 1 11:13:10 2007 New Revision: 47065 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: make valid rest Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Mon Oct 1 11:13:10 2007 @@ -33,6 +33,7 @@ critical - rewrite the following rctypes modules using rffi: + - bz2 - _demo - _ssl From antocuni at codespeak.net Mon Oct 1 11:22:52 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 1 Oct 2007 11:22:52 +0200 (CEST) Subject: [pypy-svn] r47066 - pypy/dist/pypy/doc/config Message-ID: <20071001092252.414AC812A@code0.codespeak.net> Author: antocuni Date: Mon Oct 1 11:22:51 2007 New Revision: 47066 Added: pypy/dist/pypy/doc/config/objspace.usemodules.rbench.txt (contents, props changed) Log: doc for rbench Added: pypy/dist/pypy/doc/config/objspace.usemodules.rbench.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/config/objspace.usemodules.rbench.txt Mon Oct 1 11:22:51 2007 @@ -0,0 +1,4 @@ +Use the built-in 'rbench' module. +This module contains geninterpreted versions of pystone and richards, +so it is useful to measure the interpretation overhead of the various +pypy-*. From fijal at codespeak.net Mon Oct 1 11:53:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 1 Oct 2007 11:53:51 +0200 (CEST) Subject: [pypy-svn] r47068 - pypy/dist/pypy/module/rctime Message-ID: <20071001095351.265A08131@code0.codespeak.net> Author: fijal Date: Mon Oct 1 11:53:50 2007 New Revision: 47068 Modified: pypy/dist/pypy/module/rctime/interp_time.py Log: typo. Windows machine needed! Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Mon Oct 1 11:53:50 2007 @@ -35,7 +35,7 @@ ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT), ("tm_gmtoff", rffi.LONG), ("tm_zone", rffi.CCHARP)]) elif _WIN: - calling_conf = 'win' + calling_conv = 'win' CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT), ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT), ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT), From arigo at codespeak.net Mon Oct 1 13:37:41 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 1 Oct 2007 13:37:41 +0200 (CEST) Subject: [pypy-svn] r47070 - pypy/dist/pypy/module/rctime Message-ID: <20071001113741.84959810F@code0.codespeak.net> Author: arigo Date: Mon Oct 1 13:37:40 2007 New Revision: 47070 Modified: pypy/dist/pypy/module/rctime/interp_time.py Log: genc doesn't like the uninitialized content of this global structure. For now let's simply initialize it with zeroes. Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Mon Oct 1 13:37:40 2007 @@ -60,7 +60,7 @@ time_t = cConfig.time_t size_t = cConfig.size_t tm = cConfig.tm -glob_buf = lltype.malloc(tm, flavor='raw') +glob_buf = lltype.malloc(tm, flavor='raw', zero=True) if cConfig.has_gettimeofday: c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT) From fijal at codespeak.net Mon Oct 1 13:55:20 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 1 Oct 2007 13:55:20 +0200 (CEST) Subject: [pypy-svn] r47071 - pypy/dist/pypy/module/rctime Message-ID: <20071001115520.2ED2D810F@code0.codespeak.net> Author: fijal Date: Mon Oct 1 13:55:19 2007 New Revision: 47071 Modified: pypy/dist/pypy/module/rctime/interp_time.py Log: kill XXX. sleep is implemented on rpython-level by now Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Mon Oct 1 13:55:19 2007 @@ -75,18 +75,6 @@ c_localtime = external('localtime', [TIME_TP], TM_P) if _POSIX: c_tzset = external('tzset', [], lltype.Void) -elif _WIN: - XXX - #c_QueryPerformanceCounter = external( - #QueryPerformanceCounter = windll.kernel32.QueryPerformanceCounter - #QueryPerformanceCounter.argtypes = [POINTER(LARGE_INTEGER)] - #QueryPerformanceCounter.restype = BOOL - #QueryPerformanceFrequency = windll.kernel32.QueryPerformanceFrequency - #QueryPerformanceFrequency.argtypes = [POINTER(LARGE_INTEGER)] - #QueryPerformanceFrequency.restype = BOOL - #Sleep = windll.kernel32.Sleep - #Sleep.argtypes = [DWORD] - #Sleep.restype = None c_strftime = external('strftime', [rffi.CCHARP, size_t, rffi.CCHARP, TM_P], size_t) From fijal at codespeak.net Mon Oct 1 14:02:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 1 Oct 2007 14:02:04 +0200 (CEST) Subject: [pypy-svn] r47072 - pypy/dist/pypy/module/rctime Message-ID: <20071001120204.959A08119@code0.codespeak.net> Author: fijal Date: Mon Oct 1 14:02:04 2007 New Revision: 47072 Modified: pypy/dist/pypy/module/rctime/interp_time.py Log: uh. fix for stupid bug. Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Mon Oct 1 14:02:04 2007 @@ -114,6 +114,7 @@ t_ref[0] = tt p = c_localtime(t_ref) lltype.free(t_ref, flavor='raw') + tm_zone = rffi.charp2str(p.c_tm_zone) julyzone = -p.c_tm_gmtoff julyname = [" ", tm_zone][bool(tm_zone)] From arigo at codespeak.net Mon Oct 1 19:35:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 1 Oct 2007 19:35:50 +0200 (CEST) Subject: [pypy-svn] r47077 - pypy/dist/pypy/translator/js/examples/bnb Message-ID: <20071001173550.9130580FA@code0.codespeak.net> Author: arigo Date: Mon Oct 1 19:35:48 2007 New Revision: 47077 Modified: pypy/dist/pypy/translator/js/examples/bnb/bnb.py Log: Compute the port lazily as a quick workaround for the fact that this logic hangs forever if there is a non-responsive process listening on localhost:8000. Mostly untested, sorry if I break something. Modified: pypy/dist/pypy/translator/js/examples/bnb/bnb.py ============================================================================== --- pypy/dist/pypy/translator/js/examples/bnb/bnb.py (original) +++ pypy/dist/pypy/translator/js/examples/bnb/bnb.py Mon Oct 1 19:35:48 2007 @@ -70,15 +70,22 @@ _spriteManagers = {} host = 'localhost' - try: - port = re.findall('value=".*"', urllib.urlopen('http://%s:8000' % host).read())[0] - port = int(port[7:-1]) - except IOError: - log("ERROR: Can't connect to BnB server on %s:8000" % host) -# sys.exit() - except IndexError: - log("ERROR: Connected to BnB server but unable to detect a running game") -# sys.exit() + + def getport(self): + if hasattr(self, '_port'): + return self._port + try: + port = re.findall('value=".*"', urllib.urlopen('http://%s:8000' % host).read())[0] + port = int(port[7:-1]) + except IOError: + log("ERROR: Can't connect to BnB server on %s:8000" % host) + raise IOError + except IndexError: + log("ERROR: Connected to BnB server but unable to detect a running game") + raise IOError + self._port = port + return port + port = property(getport) #def _close(self, sessionid): # if sessionid in self._serverMessage: From pypy-svn at codespeak.net Mon Oct 1 21:15:28 2007 From: pypy-svn at codespeak.net (VIAGRA ® Official Site) Date: Mon, 1 Oct 2007 21:15:28 +0200 (CEST) Subject: [pypy-svn] October 71% OFF Message-ID: <20071001121712.39073.qmail@dsl.static8510550159.ttnet.net.tr> An HTML attachment was scrubbed... URL: From fijal at codespeak.net Tue Oct 2 08:59:34 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 2 Oct 2007 08:59:34 +0200 (CEST) Subject: [pypy-svn] r47080 - pypy/dist/demo/distribution Message-ID: <20071002065934.547C8813D@code0.codespeak.net> Author: fijal Date: Tue Oct 2 08:59:33 2007 New Revision: 47080 Added: pypy/dist/demo/distribution/ Log: Add a directory for distribution demos From fijal at codespeak.net Tue Oct 2 08:59:55 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 2 Oct 2007 08:59:55 +0200 (CEST) Subject: [pypy-svn] r47081 - in pypy/dist/demo: . tproxy Message-ID: <20071002065955.45CBE8140@code0.codespeak.net> Author: fijal Date: Tue Oct 2 08:59:54 2007 New Revision: 47081 Added: pypy/dist/demo/tproxy/ pypy/dist/demo/tproxy/persistance.py - copied unchanged from r47079, pypy/dist/demo/tp-persistence.py pypy/dist/demo/tproxy/print_operations.py - copied unchanged from r47079, pypy/dist/demo/tp-print-builtin-operation.py Removed: pypy/dist/demo/tp-persistence.py pypy/dist/demo/tp-print-builtin-operation.py Log: Move tproxy demos to it's own directory From arigo at codespeak.net Tue Oct 2 10:30:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 2 Oct 2007 10:30:09 +0200 (CEST) Subject: [pypy-svn] r47082 - pypy/dist/pypy/module/rctime/test Message-ID: <20071002083009.A523D813D@code0.codespeak.net> Author: arigo Date: Tue Oct 2 10:30:08 2007 New Revision: 47082 Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py Log: A print to show progress when running with -s. I thought the test was hanging. Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py ============================================================================== --- pypy/dist/pypy/module/rctime/test/test_rctime.py (original) +++ pypy/dist/pypy/module/rctime/test/test_rctime.py Tue Oct 2 10:30:08 2007 @@ -263,6 +263,7 @@ 'j', 'm', 'M', 'p', 'S', 'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'): format = ' %' + directive + print format rctime.strptime(rctime.strftime(format, tt), format) def test_pickle(self): From arigo at codespeak.net Tue Oct 2 10:30:25 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 2 Oct 2007 10:30:25 +0200 (CEST) Subject: [pypy-svn] r47083 - pypy/dist/pypy/module/crypt Message-ID: <20071002083025.BBDBA813D@code0.codespeak.net> Author: arigo Date: Tue Oct 2 10:30:25 2007 New Revision: 47083 Modified: pypy/dist/pypy/module/crypt/__init__.py pypy/dist/pypy/module/crypt/interp_crypt.py Log: Kill dead imports. Modified: pypy/dist/pypy/module/crypt/__init__.py ============================================================================== --- pypy/dist/pypy/module/crypt/__init__.py (original) +++ pypy/dist/pypy/module/crypt/__init__.py Tue Oct 2 10:30:25 2007 @@ -1,7 +1,7 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - """A demo built-in module based on ctypes.""" + """A demo built-in module based on rffi.""" interpleveldefs = { 'crypt' : 'interp_crypt.crypt', Modified: pypy/dist/pypy/module/crypt/interp_crypt.py ============================================================================== --- pypy/dist/pypy/module/crypt/interp_crypt.py (original) +++ pypy/dist/pypy/module/crypt/interp_crypt.py Tue Oct 2 10:30:25 2007 @@ -1,16 +1,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import ObjSpace, W_Root -from pypy.rpython.rctypes.tool import ctypes_platform -from pypy.rpython.tool import rffi_platform as platform -from pypy.rpython.rctypes.tool.util import find_library from pypy.rpython.lltypesystem import rffi, lltype -import sys - -#if sys.platform == 'darwin': -# dllname = find_library('c') -# assert dllname is not None -# cryptlib = cdll.LoadLibrary(dllname) c_crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP, libraries=["crypt"], threadsafe=False) From arigo at codespeak.net Tue Oct 2 10:31:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 2 Oct 2007 10:31:00 +0200 (CEST) Subject: [pypy-svn] r47084 - in pypy/dist/pypy: module/zlib module/zlib/test rlib rlib/test Message-ID: <20071002083100.24A66813D@code0.codespeak.net> Author: arigo Date: Tue Oct 2 10:30:59 2007 New Revision: 47084 Modified: pypy/dist/pypy/module/zlib/interp_zlib.py pypy/dist/pypy/module/zlib/test/test_zlib.py pypy/dist/pypy/rlib/rzlib.py pypy/dist/pypy/rlib/test/test_rzlib.py Log: This mostly finishes the zlib module. Modified: pypy/dist/pypy/module/zlib/interp_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/interp_zlib.py (original) +++ pypy/dist/pypy/module/zlib/interp_zlib.py Tue Oct 2 10:30:59 2007 @@ -1,6 +1,7 @@ +import sys from pypy.interpreter.gateway import ObjSpace, W_Root, interp2app from pypy.interpreter.baseobjspace import Wrappable -from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.typedef import TypeDef, interp_attrproperty from pypy.interpreter.error import OperationError from pypy.rlib.rarithmetic import intmask @@ -61,8 +62,7 @@ try: stream = rzlib.deflateInit(level) except ValueError: - raise OperationError(space.w_ValueError, - space.wrap("Invalid initialization option")) + raise zlib_error(space, "Bad compression level") try: result = rzlib.compress(stream, string, rzlib.Z_FINISH) finally: @@ -78,16 +78,15 @@ decompress(string[, wbits[, bufsize]]) -- Return decompressed string. Optional arg wbits is the window buffer size. Optional arg bufsize is - the initial output buffer size. + only for compatibility with CPython and is ignored. """ try: try: stream = rzlib.inflateInit(wbits) except ValueError: - raise OperationError(space.w_ValueError, - space.wrap("Invalid initialization option")) + raise zlib_error(space, "Bad window buffer size") try: - result = rzlib.decompress(stream, string, rzlib.Z_FINISH) + result, _, _ = rzlib.decompress(stream, string, rzlib.Z_FINISH) finally: rzlib.inflateEnd(stream) except rzlib.RZlibError, e: @@ -114,6 +113,7 @@ except ValueError: raise OperationError(space.w_ValueError, space.wrap("Invalid initialization option")) + self.lock = space.allocate_lock() def __del__(self): """Automatically free the resources used by the stream.""" @@ -132,7 +132,12 @@ Call the flush() method to clear these buffers. """ try: - result = rzlib.compress(self.stream, data) + lock = self.lock + lock.acquire(True) + try: + result = rzlib.compress(self.stream, data) + finally: + lock.release() except rzlib.RZlibError, e: raise zlib_error(self.space, e.msg) return self.space.wrap(result) @@ -152,7 +157,12 @@ compressed. """ try: - result = rzlib.compress(self.stream, '', mode) + lock = self.lock + lock.acquire(True) + try: + result = rzlib.compress(self.stream, '', mode) + finally: + lock.release() except rzlib.RZlibError, e: raise zlib_error(self.space, e.msg) return self.space.wrap(result) @@ -198,6 +208,8 @@ inflateInit2. """ self.space = space + self.unused_data = '' + self.unconsumed_tail = '' try: self.stream = rzlib.inflateInit(wbits) except rzlib.RZlibError, e: @@ -205,6 +217,7 @@ except ValueError: raise OperationError(space.w_ValueError, space.wrap("Invalid initialization option")) + self.lock = space.allocate_lock() def __del__(self): """Automatically free the resources used by the stream.""" @@ -222,14 +235,33 @@ no longer than max_length. Unconsumed input data will be stored in the unconsumed_tail attribute. """ - if max_length != 0: # XXX - raise OperationError(self.space.w_NotImplementedError, - self.space.wrap("max_length != 0")) + if max_length == 0: + max_length = sys.maxint + elif max_length < 0: + raise OperationError(self.space.w_ValueError, + self.space.wrap("max_length must be " + "greater than zero")) try: - result = rzlib.decompress(self.stream, data) + lock = self.lock + lock.acquire(True) + try: + result = rzlib.decompress(self.stream, data, + max_length = max_length) + finally: + lock.release() except rzlib.RZlibError, e: raise zlib_error(self.space, e.msg) - return self.space.wrap(result) + + string, finished, unused_len = result + unused_start = len(data) - unused_len + assert unused_start >= 0 + tail = data[unused_start:] + if finished: + self.unconsumed_tail = '' + self.unused_data = tail + else: + self.unconsumed_tail = tail + return self.space.wrap(string) decompress.unwrap_spec = ['self', str, int] @@ -264,6 +296,8 @@ __new__ = interp2app(Decompress___new__), decompress = interp2app(Decompress.decompress), flush = interp2app(Decompress.flush), + unused_data = interp_attrproperty('unused_data', Decompress), + unconsumed_tail = interp_attrproperty('unconsumed_tail', Decompress), __doc__ = """decompressobj([wbits]) -- Return a decompressor object. Optional arg wbits is the window buffer size. Modified: pypy/dist/pypy/module/zlib/test/test_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/test/test_zlib.py (original) +++ pypy/dist/pypy/module/zlib/test/test_zlib.py Tue Oct 2 10:30:59 2007 @@ -133,5 +133,45 @@ def test_decompress_invalid_input(self): + """ + Try to feed garbage to zlib.decompress(). + """ raises(self.zlib.error, self.zlib.decompress, self.compressed[:-2]) raises(self.zlib.error, self.zlib.decompress, 'foobar') + + + def test_unused_data(self): + """ + Try to feed too much data to zlib.decompress(). + It should show up in the unused_data attribute. + """ + d = self.zlib.decompressobj() + s = d.decompress(self.compressed + 'extrastuff') + assert s == self.expanded + assert d.unused_data == 'extrastuff' + # try again with several decompression steps + d = self.zlib.decompressobj() + s1 = d.decompress(self.compressed[:10]) + assert d.unused_data == '' + s2 = d.decompress(self.compressed[10:-3]) + assert d.unused_data == '' + s3 = d.decompress(self.compressed[-3:] + 'spam' * 100) + assert d.unused_data == 'spam' * 100 + assert s1 + s2 + s3 == self.expanded + s4 = d.decompress('egg' * 50) + assert d.unused_data == 'egg' * 50 + assert s4 == '' + + + def test_max_length(self): + """ + Test the max_length argument of the decompress() method + and the corresponding unconsumed_tail attribute. + """ + d = self.zlib.decompressobj() + data = self.compressed + for i in range(0, 100, 10): + s1 = d.decompress(data, 10) + assert s1 == self.expanded[i:i+10] + data = d.unconsumed_tail + assert not data Modified: pypy/dist/pypy/rlib/rzlib.py ============================================================================== --- pypy/dist/pypy/rlib/rzlib.py (original) +++ pypy/dist/pypy/rlib/rzlib.py Tue Oct 2 10:30:59 2007 @@ -1,3 +1,4 @@ +import sys from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform @@ -263,15 +264,24 @@ """ # Warning, reentrant calls to the zlib with a given stream can cause it # to crash. The caller of pypy.rlib.rzlib should use locks if needed. - return _operate(stream, data, flush, False, _deflate, "while compressing") - - -def decompress(stream, data, flush=Z_SYNC_FLUSH): - """ - Feed more data into an inflate stream. Returns a string containing - (a part of) the decompressed data. If flush != Z_NO_FLUSH, this also - flushes the output data; see zlib.h or the documentation of the - zlib module for the possible values of 'flush'. + data, _, avail_in = _operate(stream, data, flush, sys.maxint, _deflate, + "while compressing") + assert not avail_in, "not all input consumed by deflate" + return data + + +def decompress(stream, data, flush=Z_SYNC_FLUSH, max_length=sys.maxint): + """ + Feed more data into an inflate stream. Returns a tuple (string, + finished, unused_data_length). The string contains (a part of) the + decompressed data. If flush != Z_NO_FLUSH, this also flushes the + output data; see zlib.h or the documentation of the zlib module for + the possible values of 'flush'. + + The 'string' is never longer than 'max_length'. The + 'unused_data_length' is the number of unprocessed input characters, + either because they are after the end of the compressed stream or + because processing it would cause the 'max_length' to be exceeded. """ # Warning, reentrant calls to the zlib with a given stream can cause it # to crash. The caller of pypy.rlib.rzlib should use locks if needed. @@ -284,11 +294,18 @@ should_finish = True else: should_finish = False - return _operate(stream, data, flush, should_finish, _inflate, - "while decompressing") + result = _operate(stream, data, flush, max_length, _inflate, + "while decompressing") + if should_finish: + # detect incomplete input in the Z_FINISHED case + finished = result[1] + if not finished: + raise RZlibError("the input compressed stream of data is " + "incomplete") + return result -def _operate(stream, data, flush, should_finish, cfunc, while_doing): +def _operate(stream, data, flush, max_length, cfunc, while_doing): """Common code for compress() and decompress(). """ # Prepare the input buffer for the stream @@ -312,12 +329,19 @@ while True: stream.c_next_out = rffi.cast(Bytefp, outbuf) - rffi.setintfield(stream, 'c_avail_out', OUTPUT_BUFFER_SIZE) + bufsize = OUTPUT_BUFFER_SIZE + if max_length < bufsize: + if max_length <= 0: + err = Z_OK + break + bufsize = max_length + max_length -= bufsize + rffi.setintfield(stream, 'c_avail_out', bufsize) err = cfunc(stream, flush) if err == Z_OK or err == Z_STREAM_END: # accumulate data into 'result' avail_out = rffi.cast(lltype.Signed, stream.c_avail_out) - for i in xrange(OUTPUT_BUFFER_SIZE - avail_out): + for i in xrange(bufsize - avail_out): result.append(outbuf[i]) # if the output buffer is full, there might be more data # so we need to try again. Otherwise, we're done. @@ -335,7 +359,7 @@ # the output buffer was full but there wasn't more # output when we tried again, so it is not an error # condition. - if avail_out == OUTPUT_BUFFER_SIZE: + if avail_out == bufsize: break # fallback case: report this error @@ -347,10 +371,8 @@ lltype.free(inbuf, flavor='raw') # When decompressing, if the compressed stream of data was truncated, - # then the zlib simply returns Z_OK and waits for more. Let's detect - # this situation and complain. - if should_finish and err != Z_STREAM_END: - raise RZlibError("the input compressed stream of data is not complete") - - assert not stream.c_avail_in, "not all input consumed by deflate/inflate" - return ''.join(result) + # then the zlib simply returns Z_OK and waits for more. If it is + # complete it returns Z_STREAM_END. + return (''.join(result), + err == Z_STREAM_END, + rffi.cast(lltype.Signed, stream.c_avail_in)) Modified: pypy/dist/pypy/rlib/test/test_rzlib.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rzlib.py (original) +++ pypy/dist/pypy/rlib/test/test_rzlib.py Tue Oct 2 10:30:59 2007 @@ -121,10 +121,14 @@ should allow us to decompress bytes. """ stream = rzlib.inflateInit() - bytes = rzlib.decompress(stream, compressed) - bytes += rzlib.decompress(stream, "", rzlib.Z_FINISH) + bytes1, finished1, unused1 = rzlib.decompress(stream, compressed) + bytes2, finished2, unused2 = rzlib.decompress(stream, "", rzlib.Z_FINISH) rzlib.inflateEnd(stream) - assert bytes == expanded + assert bytes1 + bytes2 == expanded + assert finished1 is True + assert finished2 is True + assert unused1 == 0 + assert unused2 == 0 def test_decompression_lots_of_data(): @@ -135,9 +139,12 @@ compressed = zlib.compress(expanded) print len(compressed), '=>', len(expanded) stream = rzlib.inflateInit() - bytes = rzlib.decompress(stream, compressed, rzlib.Z_FINISH) + bytes, finished, unused = rzlib.decompress(stream, compressed, + rzlib.Z_FINISH) rzlib.inflateEnd(stream) assert bytes == expanded + assert finished is True + assert unused == 0 def test_decompression_truncated_input(): @@ -149,11 +156,75 @@ compressed = zlib.compress(expanded) print len(compressed), '=>', len(expanded) stream = rzlib.inflateInit() - data = rzlib.decompress(stream, compressed[:1000]) + data, finished1, unused1 = rzlib.decompress(stream, compressed[:1000]) assert expanded.startswith(data) - data += rzlib.decompress(stream, compressed[1000:2000]) + assert finished1 is False + assert unused1 == 0 + data2, finished2, unused2 = rzlib.decompress(stream, compressed[1000:2000]) + data += data2 + assert finished2 is False + assert unused2 == 0 assert expanded.startswith(data) py.test.raises(rzlib.RZlibError, rzlib.decompress, stream, compressed[2000:-500], rzlib.Z_FINISH) rzlib.inflateEnd(stream) + + +def test_decompression_too_much_input(): + """ + Check the case where we feed extra data to decompress(). + """ + stream = rzlib.inflateInit() + data1, finished1, unused1 = rzlib.decompress(stream, compressed[:-5]) + assert finished1 is False + assert unused1 == 0 + data2, finished2, unused2 = rzlib.decompress(stream, + compressed[-5:] + 'garbage') + assert finished2 is True + assert unused2 == len('garbage') + assert data1 + data2 == expanded + data3, finished3, unused3 = rzlib.decompress(stream, 'more_garbage') + assert finished3 is True + assert unused3 == len('more_garbage') + assert data3 == '' + + +def test_decompress_max_length(): + """ + Test the max_length argument of decompress(). + """ + stream = rzlib.inflateInit() + data1, finished1, unused1 = rzlib.decompress(stream, compressed, + max_length = 17) + assert data1 == expanded[:17] + assert finished1 is False + assert unused1 > 0 + data2, finished2, unused2 = rzlib.decompress(stream, compressed[-unused1:]) + assert data2 == expanded[17:] + assert finished2 is True + assert unused2 == 0 + + +def test_cornercases(): + """ + Test degenerate arguments. + """ + stream = rzlib.deflateInit() + bytes = rzlib.compress(stream, "") + bytes += rzlib.compress(stream, "") + bytes += rzlib.compress(stream, "", rzlib.Z_FINISH) + assert zlib.decompress(bytes) == "" + + stream = rzlib.inflateInit() + data, finished, unused = rzlib.decompress(stream, "") + assert data == "" + assert finished is False + assert unused == 0 + buf = compressed + for i in range(10): + data, finished, unused = rzlib.decompress(stream, buf, max_length=0) + assert data == "" + assert finished is False + assert unused > 0 + buf = buf[-unused:] From fijal at codespeak.net Tue Oct 2 10:31:40 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 2 Oct 2007 10:31:40 +0200 (CEST) Subject: [pypy-svn] r47085 - in pypy/dist: demo/distribution pypy/lib/distributed/demo Message-ID: <20071002083140.3D61E813D@code0.codespeak.net> Author: fijal Date: Tue Oct 2 10:31:39 2007 New Revision: 47085 Added: pypy/dist/demo/distribution/fileclient.py - copied unchanged from r47070, pypy/dist/pypy/lib/distributed/demo/fileclient.py pypy/dist/demo/distribution/fileserver.py - copied unchanged from r47070, pypy/dist/pypy/lib/distributed/demo/fileserver.py Removed: pypy/dist/pypy/lib/distributed/demo/fileclient.py pypy/dist/pypy/lib/distributed/demo/fileserver.py Log: Move distribution demos to it's own directory in demo From arigo at codespeak.net Tue Oct 2 10:36:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 2 Oct 2007 10:36:02 +0200 (CEST) Subject: [pypy-svn] r47086 - in pypy/dist/pypy/translator: cli/test js/test jvm/test Message-ID: <20071002083602.39B73813D@code0.codespeak.net> Author: arigo Date: Tue Oct 2 10:36:01 2007 New Revision: 47086 Modified: pypy/dist/pypy/translator/cli/test/test_float.py pypy/dist/pypy/translator/js/test/test_rfloat.py pypy/dist/pypy/translator/jvm/test/test_float.py Log: Skip the SingleFloat test on non-C backends for now. Modified: pypy/dist/pypy/translator/cli/test/test_float.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_float.py (original) +++ pypy/dist/pypy/translator/cli/test/test_float.py Tue Oct 2 10:36:01 2007 @@ -16,3 +16,6 @@ expected = fn(i) res = self.interpret(fn, [i]) assert res == expected + + def test_r_singlefloat(self): + py.test.skip("not implemented: single-precision floats") Modified: pypy/dist/pypy/translator/js/test/test_rfloat.py ============================================================================== --- pypy/dist/pypy/translator/js/test/test_rfloat.py (original) +++ pypy/dist/pypy/translator/js/test/test_rfloat.py Tue Oct 2 10:36:01 2007 @@ -22,3 +22,6 @@ assert x != big y = fn(x) assert fn(x) == 9223372026854775808 + + def test_r_singlefloat(self): + py.test.skip("not implemented: single-precision floats") Modified: pypy/dist/pypy/translator/jvm/test/test_float.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_float.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_float.py Tue Oct 2 10:36:01 2007 @@ -19,3 +19,6 @@ expected = fn(i) res = self.interpret(fn, [i]) assert res == expected + + def test_r_singlefloat(self): + py.test.skip("not implemented: single-precision floats") From arigo at codespeak.net Tue Oct 2 10:57:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 2 Oct 2007 10:57:43 +0200 (CEST) Subject: [pypy-svn] r47087 - pypy/dist/pypy/translator/c Message-ID: <20071002085743.BD3068142@code0.codespeak.net> Author: arigo Date: Tue Oct 2 10:57:43 2007 New Revision: 47087 Modified: pypy/dist/pypy/translator/c/database.py pypy/dist/pypy/translator/c/node.py pypy/dist/pypy/translator/c/support.py Log: Never use 'typedef' in generated C code. Just don't. You work like type macros in that we can always spell out the whole type instead whenever it's used - and they *don't* work like macros in that some C compilers don't like typedefs that involve arrays of not-yet-defined structures so you'd have to carefully order declarations. Don't bother. This fixes test_structarray_nolength which failed on gcc 4 (but not gcc 3). Modified: pypy/dist/pypy/translator/c/database.py ============================================================================== --- pypy/dist/pypy/translator/c/database.py (original) +++ pypy/dist/pypy/translator/c/database.py Tue Oct 2 10:57:43 2007 @@ -9,10 +9,10 @@ from pypy.translator.c.primitive import PrimitiveName, PrimitiveType from pypy.translator.c.primitive import PrimitiveErrorValue from pypy.translator.c.node import StructDefNode, ArrayDefNode -from pypy.translator.c.node import FixedSizeArrayDefNode +from pypy.translator.c.node import FixedSizeArrayDefNode, BareBoneArrayDefNode from pypy.translator.c.node import ContainerNodeFactory, ExtTypeOpaqueDefNode from pypy.translator.c.support import cdecl, CNameManager, ErrorValue -from pypy.translator.c.support import log +from pypy.translator.c.support import log, barebonearray from pypy.translator.c.extfunc import do_the_getting from pypy import conftest from pypy.translator.c import gc @@ -83,7 +83,10 @@ else: node = StructDefNode(self, T, varlength) elif isinstance(T, Array): - node = ArrayDefNode(self, T, varlength) + if barebonearray(T): + node = BareBoneArrayDefNode(self, T, varlength) + else: + node = ArrayDefNode(self, T, varlength) elif isinstance(T, OpaqueType) and hasattr(T, '_exttypeinfo'): node = ExtTypeOpaqueDefNode(self, T) elif T == WeakRef: Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Tue Oct 2 10:57:43 2007 @@ -214,15 +214,6 @@ self.name) = db.namespace.uniquename(basename, with_number=with_number, bare=True) self.dependencies = {} - # a non-gc array with no length doesn't need a 'struct' wrapper at - # all; rffi kind of expects such arrays to be "bare" C arrays. - self.barebone = barebonearray(ARRAY) - if self.barebone: - self.setup() # to get self.itemtypename - typename = self.itemtypename.replace('@', '(@)[%d]' % ( - self.varlength,)) - self.forward_decl = 'typedef %s;' % (cdecl(typename, self.name),) - self.typetag = '' def setup(self): if hasattr(self, 'itemtypename'): @@ -247,32 +238,22 @@ return '%s %s @' % (self.typetag, self.name) def getptrtype(self): - if self.barebone: - return self.itemtypename.replace('@', '*@') - else: - return '%s %s *@' % (self.typetag, self.name) + return '%s %s *@' % (self.typetag, self.name) def access_expr(self, baseexpr, index): - if self.barebone: - return '%s[%d]' % (baseexpr, index) - else: - return '%s.items[%d]' % (baseexpr, index) + return '%s.items[%d]' % (baseexpr, index) def ptr_access_expr(self, baseexpr, index): assert 0 <= index <= sys.maxint, "invalid constant index %r" % (index,) return self.itemindex_access_expr(baseexpr, index) def itemindex_access_expr(self, baseexpr, indexexpr): - if self.barebone: - return 'RPyBareItem(%s, %s)' % (baseexpr, indexexpr) - elif self.ARRAY._hints.get('nolength', False): + if self.ARRAY._hints.get('nolength', False): return 'RPyNLenItem(%s, %s)' % (baseexpr, indexexpr) else: return 'RPyItem(%s, %s)' % (baseexpr, indexexpr) def definition(self): - if self.barebone: - return gcpolicy = self.db.gcpolicy yield 'struct %s {' % self.name for fname, typename in self.gcfields: @@ -316,7 +297,7 @@ yield 'offsetof(struct %s, length)' % (self.name,) else: yield '-1' - if self.ARRAY.OF is not Void and not self.barebone: + if self.ARRAY.OF is not Void: yield 'offsetof(struct %s, items[0])' % (self.name,) yield 'offsetof(struct %s, items[1])' % (self.name,) else: @@ -324,6 +305,56 @@ yield '-1' +class BareBoneArrayDefNode: + """For 'simple' array types which don't need a length nor GC headers. + Implemented directly as a C array instead of a struct with an items field. + rffi kind of expects such arrays to be 'bare' C arrays. + """ + gcinfo = None + name = None + forward_decl = None + + def __init__(self, db, ARRAY, varlength=1): + self.db = db + self.ARRAY = ARRAY + self.LLTYPE = ARRAY + self.varlength = varlength + self.dependencies = {} + self.itemtypename = db.gettype(ARRAY.OF, who_asks=self) + + def setup(self): + """Array loops are forbidden by ForwardReference.become() because + there is no way to declare them in C.""" + + def gettype(self): + return self.itemtypename.replace('@', '(@)[%d]' % (self.varlength,)) + + def getptrtype(self): + return self.itemtypename.replace('@', '*@') + + def access_expr(self, baseexpr, index): + return '%s[%d]' % (baseexpr, index) + + def ptr_access_expr(self, baseexpr, index): + assert 0 <= index <= sys.maxint, "invalid constant index %r" % (index,) + return self.itemindex_access_expr(baseexpr, index) + + def itemindex_access_expr(self, baseexpr, indexexpr): + return 'RPyBareItem(%s, %s)' % (baseexpr, indexexpr) + + def definition(self): + return [] # no declaration is needed + + def visitor_lines(self, prefix, on_item): + raise Exception("cannot visit C arrays - don't know the length") + + def debug_offsets(self): + # generate three offsets for debugging inspection, + yield '-1' # no length + yield '0' # first element is immediately at the start of the array + yield 'sizeof(%s)' % (cdecl(self.itemtypename, ''),) + + class FixedSizeArrayDefNode: gcinfo = None name = None Modified: pypy/dist/pypy/translator/c/support.py ============================================================================== --- pypy/dist/pypy/translator/c/support.py (original) +++ pypy/dist/pypy/translator/c/support.py Tue Oct 2 10:57:43 2007 @@ -17,7 +17,8 @@ def barebonearray(ARRAY): """Check if ARRAY is a 'simple' array type, i.e. doesn't need a length nor GC headers.""" - return ARRAY._hints.get('nolength', False) and ARRAY._gckind != 'gc' + return (ARRAY._hints.get('nolength', False) and ARRAY._gckind != 'gc' + and ARRAY.OF is not lltype.Void) # From fijal at codespeak.net Tue Oct 2 11:54:37 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 2 Oct 2007 11:54:37 +0200 (CEST) Subject: [pypy-svn] r47090 - pypy/dist/pypy/translator/goal Message-ID: <20071002095437.BCB6A8143@code0.codespeak.net> Author: fijal Date: Tue Oct 2 11:54:37 2007 New Revision: 47090 Modified: pypy/dist/pypy/translator/goal/app_main.py Log: Use flag in pyrepl, not to clear main Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Tue Oct 2 11:54:37 2007 @@ -321,7 +321,7 @@ except ImportError: success = run_toplevel(interactive_console, mainmodule) else: - main(print_banner=False) + main(print_banner=False, clear_main=False) success = True except SystemExit, e: return e.code From fijal at codespeak.net Tue Oct 2 12:02:07 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 2 Oct 2007 12:02:07 +0200 (CEST) Subject: [pypy-svn] r47091 - pypy/dist/demo/distribution Message-ID: <20071002100207.289798143@code0.codespeak.net> Author: fijal Date: Tue Oct 2 12:02:06 2007 New Revision: 47091 Modified: pypy/dist/demo/distribution/fileclient.py pypy/dist/demo/distribution/fileserver.py Log: Improve docs, use code.interact as pypy-c -i might be buggy Modified: pypy/dist/demo/distribution/fileclient.py ============================================================================== --- pypy/dist/demo/distribution/fileclient.py (original) +++ pypy/dist/demo/distribution/fileclient.py Tue Oct 2 12:02:06 2007 @@ -1,14 +1,25 @@ """ This is sample client for a server based in fileserver.py, not counting -initialization and __doc__ has just 2 lines. Usage: +initialization, code.interact and __doc__ has just 2 lines! Usage: -pypy-c -i fileclient.py +pypy-c fileclient.py + +The file_opener is a proxy for remote file object. Which means you can +perform same operations as locally, like file_opener('/etc/passwd').read() +or file_opener('/tmp/x', 'w').write('x') + +pypy-c needs to be compiled with --allworkingmodules in order to have socket +working. """ + HOST = '127.0.0.1' PORT = 12221 from distributed.socklayer import connect file_opener = connect((HOST, PORT)).open -# now you can do for example file_opener('/etc/passwd').read() to -# read remote /etc/passwd +import code +code.interact(local=locals()) +# The file_opener is a proxy for remote file object. Which means you can +# perform same operations as locally, like file_opener('/etc/passwd').read() +# or file_opener('/tmp/x', 'w').write('x') Modified: pypy/dist/demo/distribution/fileserver.py ============================================================================== --- pypy/dist/demo/distribution/fileserver.py (original) +++ pypy/dist/demo/distribution/fileserver.py Tue Oct 2 12:02:06 2007 @@ -3,8 +3,11 @@ fully operational file server, sample client which is in fileclient.py is included as well. -Note that you must run it with pypy-c compiled with transparent proxy -and allworkingmodules (or at least socket and select) +run by: +pypy-c fileserver.py + +pypy-c needs to be compiled with --allworkingmodules in order to have socket +working. """ HOST = '127.0.0.1' # defaults to localhost, not to export your files From fijal at codespeak.net Tue Oct 2 12:06:52 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 2 Oct 2007 12:06:52 +0200 (CEST) Subject: [pypy-svn] r47092 - pypy/dist/demo/distribution Message-ID: <20071002100652.A249180A4@code0.codespeak.net> Author: fijal Date: Tue Oct 2 12:06:51 2007 New Revision: 47092 Modified: pypy/dist/demo/distribution/fileserver.py Log: Improve docstring Modified: pypy/dist/demo/distribution/fileserver.py ============================================================================== --- pypy/dist/demo/distribution/fileserver.py (original) +++ pypy/dist/demo/distribution/fileserver.py Tue Oct 2 12:06:51 2007 @@ -1,4 +1,6 @@ -""" This is sample demo about how flexible pypy distribution is. +""" This is a sample demo showcasing file server, done by the pypy +distriobution library. + Not counting __doc__ and initialization this is 2 line, fully operational file server, sample client which is in fileclient.py is included as well. From fijal at codespeak.net Tue Oct 2 12:40:23 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 2 Oct 2007 12:40:23 +0200 (CEST) Subject: [pypy-svn] r47095 - pypy/dist/pypy/lib/distributed Message-ID: <20071002104023.B1F6F80A7@code0.codespeak.net> Author: fijal Date: Tue Oct 2 12:40:22 2007 New Revision: 47095 Modified: pypy/dist/pypy/lib/distributed/socklayer.py pypy/dist/pypy/lib/distributed/support.py Log: unobfusticate support routines Modified: pypy/dist/pypy/lib/distributed/socklayer.py ============================================================================== --- pypy/dist/pypy/lib/distributed/socklayer.py (original) +++ pypy/dist/pypy/lib/distributed/socklayer.py Tue Oct 2 12:40:22 2007 @@ -64,9 +64,9 @@ return SenderWrapper(s).send, ReceiverWrapper(s).receive def connect(address, socket=socket): - from distributed.support import get_remote_view + from distributed.support import RemoteView from distributed import RemoteProtocol - return get_remote_view(RemoteProtocol(*socket_connecter(address, socket))) + return RemoteView(RemoteProtocol(*socket_connecter(address, socket))) def spawn_remote_side(code, gw): """ A very simple wrapper around greenexecnet to allow Modified: pypy/dist/pypy/lib/distributed/support.py ============================================================================== --- pypy/dist/pypy/lib/distributed/support.py (original) +++ pypy/dist/pypy/lib/distributed/support.py Tue Oct 2 12:40:22 2007 @@ -2,13 +2,16 @@ """ Some random support functions """ -def get_remote_view(protocol): - # this is dynamic to provide needed level of laziness - class RemoteView(object): - pass +from distributed.protocol import ObjectNotFound - for key in protocol.remote_keys(): - getter = lambda self: protocol.get_remote(key) - setattr(RemoteView, key, property(getter)) +class RemoteView(object): + def __init__(self, protocol): + self.__dict__['__protocol'] = protocol - return RemoteView() + def __getattr__(self, name): + if name == '__dict__': + return super(RemoteView, self).__getattr__(name) + try: + return self.__dict__['__protocol'].get_remote(name) + except ObjectNotFound: + raise AttributeError(name) From fijal at codespeak.net Tue Oct 2 12:41:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 2 Oct 2007 12:41:04 +0200 (CEST) Subject: [pypy-svn] r47096 - pypy/dist/demo/distribution Message-ID: <20071002104104.EA0FA80AA@code0.codespeak.net> Author: fijal Date: Tue Oct 2 12:41:04 2007 New Revision: 47096 Added: pypy/dist/demo/distribution/client.py (contents, props changed) pypy/dist/demo/distribution/server.py (contents, props changed) Log: Intermediate version of some demos, will fix later (battery dead) Added: pypy/dist/demo/distribution/client.py ============================================================================== --- (empty file) +++ pypy/dist/demo/distribution/client.py Tue Oct 2 12:41:04 2007 @@ -0,0 +1,16 @@ +""" This a sample client, suitable for use with server.py from this +directory + +run by: +pypy-c client.py +""" + +HOST = '127.0.0.1' +PORT = 12222 + +from distributed.socklayer import connect +remote_handle = connect((HOST, PORT)) + +import code +code.interact(local=locals()) + Added: pypy/dist/demo/distribution/server.py ============================================================================== --- (empty file) +++ pypy/dist/demo/distribution/server.py Tue Oct 2 12:41:04 2007 @@ -0,0 +1,35 @@ +""" This is a demo exposing all globals from the current process over +socket, to be accessible remotely. + +run by: +pypy-c server.py + +pypy-c needs to be compiled with --allworkingmodules in order to have socket +working. +""" + +# things to export +# function +def f(x): + return x + 3 + +# class +class X: + def __init__(self): + self.slot = 3 + + def meth(self, f, arg): + """ Method eating callable and calling it with an argument + """ + assert callable(f) + return f(arg) + +# object +x = X() + +# constants +HOST = '127.0.0.1' +PORT = 12222 + +from distributed.socklayer import socket_loop +socket_loop((HOST, PORT), globals()) From fijal at codespeak.net Tue Oct 2 12:51:02 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 2 Oct 2007 12:51:02 +0200 (CEST) Subject: [pypy-svn] r47097 - pypy/dist/demo/distribution Message-ID: <20071002105102.51DE480AC@code0.codespeak.net> Author: fijal Date: Tue Oct 2 12:51:01 2007 New Revision: 47097 Modified: pypy/dist/demo/distribution/client.py pypy/dist/demo/distribution/server.py Log: Improve comments. Modified: pypy/dist/demo/distribution/client.py ============================================================================== --- pypy/dist/demo/distribution/client.py (original) +++ pypy/dist/demo/distribution/client.py Tue Oct 2 12:51:01 2007 @@ -14,3 +14,22 @@ import code code.interact(local=locals()) +""" Things that can be done: 1. remote object access + +x = remote_handle.x +assert type(x) is remote_handle.X # typecheck +x.meth(lambda x: x + 10, 6) # remote call, with callback localy +x.meth(remote_handle.f, 3) # remote call, remote callback +remote_handle.sys._getframe(2).f_locals['x'] # remote frame access +# XXX should be 'is x' and shouldn't need (2) argument + +# XXX next one does not work, while it should. Too much mangling with remote +# traceback frames probably +try: + x.meth() # wrong argument numbers +except: + import sys + e, c, tb = sys.exc_info() + import pdb + pdb.post_mortem(tb) +""" Modified: pypy/dist/demo/distribution/server.py ============================================================================== --- pypy/dist/demo/distribution/server.py (original) +++ pypy/dist/demo/distribution/server.py Tue Oct 2 12:51:01 2007 @@ -27,6 +27,9 @@ # object x = X() +# module +import sys + # constants HOST = '127.0.0.1' PORT = 12222 From arigo at codespeak.net Tue Oct 2 16:22:30 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 2 Oct 2007 16:22:30 +0200 (CEST) Subject: [pypy-svn] r47101 - pypy/dist/demo Message-ID: <20071002142230.D0E6C8122@code0.codespeak.net> Author: arigo Date: Tue Oct 2 16:22:29 2007 New Revision: 47101 Modified: pypy/dist/demo/bpnn.py Log: Avoid float **. Modified: pypy/dist/demo/bpnn.py ============================================================================== --- pypy/dist/demo/bpnn.py (original) +++ pypy/dist/demo/bpnn.py Tue Oct 2 16:22:29 2007 @@ -131,7 +131,8 @@ # calculate error error = 0.0 for k in range(len(targets)): - error = error + 0.5*(targets[k]-self.ao[k])**2 + delta = targets[k]-self.ao[k] + error = error + 0.5*delta*delta return error From cfbolz at codespeak.net Tue Oct 2 20:38:21 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 2 Oct 2007 20:38:21 +0200 (CEST) Subject: [pypy-svn] r47102 - pypy/branch/rdict-index-based2 Message-ID: <20071002183821.57F528134@code0.codespeak.net> Author: cfbolz Date: Tue Oct 2 20:38:19 2007 New Revision: 47102 Added: pypy/branch/rdict-index-based2/ - copied from r47101, pypy/dist/ Log: create a new branch to merge in the work of the rdict-index-based branch. From cfbolz at codespeak.net Tue Oct 2 20:45:01 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 2 Oct 2007 20:45:01 +0200 (CEST) Subject: [pypy-svn] r47103 - in pypy/branch/rdict-index-based2/pypy/rpython: lltypesystem test Message-ID: <20071002184501.45DC88124@code0.codespeak.net> Author: cfbolz Date: Tue Oct 2 20:45:00 2007 New Revision: 47103 Modified: pypy/branch/rdict-index-based2/pypy/rpython/lltypesystem/rdict.py pypy/branch/rdict-index-based2/pypy/rpython/test/test_rdict.py Log: manually merge the changes of the rdict-index-based branch (which itself has the code from the more-gckinds branch, it seems). The work was mostly done by mwh, I only adapted it to todays rdict. The idea is to make rdict not use pointers to entries across function boundaries too much, as this makes the life of a moving GC very hard. Instead, the pair of (dict pointer, index) is used. Modified: pypy/branch/rdict-index-based2/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/branch/rdict-index-based2/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/branch/rdict-index-based2/pypy/rpython/lltypesystem/rdict.py Tue Oct 2 20:45:00 2007 @@ -75,7 +75,7 @@ # compute the shape of the DICTENTRY structure entryfields = [] - entrymeths = { + adtmeths = { 'must_clear_key': (isinstance(self.DICTKEY, lltype.Ptr) and self.DICTKEY._needsgc()), 'must_clear_value': (isinstance(self.DICTVALUE, lltype.Ptr) @@ -98,23 +98,25 @@ s_value) # * the state of the entry - trying to encode it as dummy objects + adtmeths['entry_has_f_everused'] = 0 + adtmeths['entry_has_f_valid'] = 0 if nullkeymarker and dummykeyobj: # all the state can be encoded in the key - entrymeths['everused'] = ll_everused_from_key - entrymeths['dummy_obj'] = dummykeyobj - entrymeths['valid'] = ll_valid_from_key - entrymeths['mark_deleted'] = ll_mark_deleted_in_key + adtmeths['entry_everused'] = ll_everused_from_key + adtmeths['dummy_obj'] = dummykeyobj + adtmeths['entry_valid'] = ll_valid_from_key + adtmeths['mark_entry_deleted'] = ll_mark_deleted_in_key # the key is overwritten by 'dummy' when the entry is deleted - entrymeths['must_clear_key'] = False + adtmeths['must_clear_key'] = False elif nullvaluemarker and dummyvalueobj: # all the state can be encoded in the value - entrymeths['everused'] = ll_everused_from_value - entrymeths['dummy_obj'] = dummyvalueobj - entrymeths['valid'] = ll_valid_from_value - entrymeths['mark_deleted'] = ll_mark_deleted_in_value + adtmeths['entry_everused'] = ll_everused_from_value + adtmeths['dummy_obj'] = dummyvalueobj + adtmeths['entry_valid'] = ll_valid_from_value + adtmeths['mark_entry_deleted'] = ll_mark_deleted_in_value # value is overwritten by 'dummy' when entry is deleted - entrymeths['must_clear_value'] = False + adtmeths['must_clear_value'] = False else: # we need a flag to know if the entry was ever used @@ -122,25 +124,27 @@ # the key and value will be reset to NULL to clear their # reference) entryfields.append(("f_everused", lltype.Bool)) - entrymeths['everused'] = ll_everused_from_flag + adtmeths['entry_everused'] = ll_everused_from_flag + adtmeths['entry_has_f_everused'] = 1 # can we still rely on a dummy obj to mark deleted entries? if dummykeyobj: - entrymeths['dummy_obj'] = dummykeyobj - entrymeths['valid'] = ll_valid_from_key - entrymeths['mark_deleted'] = ll_mark_deleted_in_key + adtmeths['dummy_obj'] = dummykeyobj + adtmeths['entry_valid'] = ll_valid_from_key + adtmeths['mark_entry_deleted'] = ll_mark_deleted_in_key # key is overwritten by 'dummy' when entry is deleted - entrymeths['must_clear_key'] = False + adtmeths['must_clear_key'] = False elif dummyvalueobj: - entrymeths['dummy_obj'] = dummyvalueobj - entrymeths['valid'] = ll_valid_from_value - entrymeths['mark_deleted'] = ll_mark_deleted_in_value + adtmeths['dummy_obj'] = dummyvalueobj + adtmeths['entry_valid'] = ll_valid_from_value + adtmeths['mark_entry_deleted'] = ll_mark_deleted_in_value # value is overwritten by 'dummy' when entry is deleted - entrymeths['must_clear_value'] = False + adtmeths['must_clear_value'] = False else: entryfields.append(("f_valid", lltype.Bool)) - entrymeths['valid'] = ll_valid_from_flag - entrymeths['mark_deleted'] = ll_mark_deleted_in_flag + adtmeths['entry_has_f_valid'] = 1 + adtmeths['entry_valid'] = ll_valid_from_flag + adtmeths['mark_entry_deleted'] = ll_mark_deleted_in_flag # * the value entryfields.append(("value", self.DICTVALUE)) @@ -150,15 +154,19 @@ fasthashfn = None else: fasthashfn = self.key_repr.get_ll_fasthash_function() + + adtmeths['entry_has_f_hash'] = 0 if fasthashfn is None: entryfields.append(("f_hash", lltype.Signed)) - entrymeths['hash'] = ll_hash_from_cache + adtmeths['entry_has_f_hash'] = 1 + # XXX entry_hash, maybe? + adtmeths['hash'] = ll_hash_from_cache else: - entrymeths['hash'] = ll_hash_recomputed - entrymeths['fasthashfn'] = fasthashfn + adtmeths['hash'] = ll_hash_recomputed + adtmeths['fasthashfn'] = fasthashfn # Build the lltype data structures - self.DICTENTRY = lltype.Struct("dictentry", adtmeths=entrymeths, + self.DICTENTRY = lltype.Struct("dictentry", *entryfields) self.DICTENTRYARRAY = lltype.GcArray(self.DICTENTRY) fields = [ ("num_items", lltype.Signed), @@ -168,13 +176,13 @@ self.r_rdict_eqfn, self.r_rdict_hashfn = self._custom_eq_hash_repr() fields.extend([ ("fnkeyeq", self.r_rdict_eqfn.lowleveltype), ("fnkeyhash", self.r_rdict_hashfn.lowleveltype) ]) - adtmeths = { + adtmeths.update({ 'keyhash': ll_keyhash_custom, 'keyeq': ll_keyeq_custom, 'r_rdict_eqfn': self.r_rdict_eqfn, 'r_rdict_hashfn': self.r_rdict_hashfn, 'paranoia': True, - } + }) else: # figure out which functions must be used to hash and compare ll_keyhash = self.key_repr.get_ll_hash_function() @@ -182,11 +190,11 @@ ll_keyhash = lltype.staticAdtMethod(ll_keyhash) if ll_keyeq is not None: ll_keyeq = lltype.staticAdtMethod(ll_keyeq) - adtmeths = { + adtmeths.update({ 'keyhash': ll_keyhash, 'keyeq': ll_keyeq, 'paranoia': False, - } + }) adtmeths['KEY'] = self.DICTKEY adtmeths['VALUE'] = self.DICTVALUE self.DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths, @@ -350,47 +358,47 @@ # be direct_call'ed from rtyped flow graphs, which means that they will # get flowed and annotated, mostly with SomePtr. -def ll_everused_from_flag(entry): - return entry.f_everused +def ll_everused_from_flag(d, i): + return d.entries[i].f_everused + +def ll_everused_from_key(d, i): + return bool(d.entries[i].key) + +def ll_everused_from_value(d, i): + return bool(d.entries[i].value) + +def ll_valid_from_flag(d, i): + return d.entries[i].f_valid + +def ll_mark_deleted_in_flag(d, i): + d.entries[i].f_valid = False + +def ll_valid_from_key(d, i): + DICT = lltype.typeOf(d).TO + dummy = DICT.dummy_obj.ll_dummy_value + return d.entry_everused(i) and d.entries[i].key != dummy + +def ll_mark_deleted_in_key(d, i): + DICT = lltype.typeOf(d).TO + dummy = DICT.dummy_obj.ll_dummy_value + d.entries[i].key = dummy -def ll_everused_from_key(entry): - return bool(entry.key) +def ll_valid_from_value(d, i): + DICT = lltype.typeOf(d).TO + dummy = DICT.dummy_obj.ll_dummy_value + return d.entry_everused(i) and d.entries[i].value != dummy -def ll_everused_from_value(entry): - return bool(entry.value) +def ll_mark_deleted_in_value(d, i): + DICT = lltype.typeOf(d).TO + dummy = DICT.dummy_obj.ll_dummy_value + d.entries[i].value = dummy -def ll_valid_from_flag(entry): - return entry.f_valid - -def ll_mark_deleted_in_flag(entry): - entry.f_valid = False - -def ll_valid_from_key(entry): - ENTRY = lltype.typeOf(entry).TO - dummy = ENTRY.dummy_obj.ll_dummy_value - return entry.everused() and entry.key != dummy - -def ll_mark_deleted_in_key(entry): - ENTRY = lltype.typeOf(entry).TO - dummy = ENTRY.dummy_obj.ll_dummy_value - entry.key = dummy - -def ll_valid_from_value(entry): - ENTRY = lltype.typeOf(entry).TO - dummy = ENTRY.dummy_obj.ll_dummy_value - return entry.everused() and entry.value != dummy - -def ll_mark_deleted_in_value(entry): - ENTRY = lltype.typeOf(entry).TO - dummy = ENTRY.dummy_obj.ll_dummy_value - entry.value = dummy - -def ll_hash_from_cache(entry): - return entry.f_hash - -def ll_hash_recomputed(entry): - ENTRY = lltype.typeOf(entry).TO - return ENTRY.fasthashfn(entry.key) +def ll_hash_from_cache(d, i): + return d.entries[i].f_hash + +def ll_hash_recomputed(d, i): + DICT = lltype.typeOf(d).TO + return DICT.fasthashfn(d.entries[i].key) def ll_keyhash_custom(d, key): DICT = lltype.typeOf(d).TO @@ -408,9 +416,9 @@ return bool(d) and d.num_items != 0 def ll_dict_getitem(d, key): - entry = ll_dict_lookup(d, key, d.keyhash(key)) - if entry.valid(): - return entry.value + i = ll_dict_lookup(d, key, d.keyhash(key)) + if d.entry_valid(i): + return d.entries[i].value else: raise KeyError ll_dict_getitem.oopspec = 'dict.getitem(d, key)' @@ -418,20 +426,21 @@ def ll_dict_setitem(d, key, value): hash = d.keyhash(key) - entry = ll_dict_lookup(d, key, hash) - everused = entry.everused() - valid = entry.valid() + i = ll_dict_lookup(d, key, hash) + entry = d.entries[i] + everused = d.entry_everused(i) + valid = d.entry_valid(i) # set up the new entry - ENTRY = lltype.typeOf(entry).TO + DICT = lltype.typeOf(d).TO entry.value = value if valid: return entry.key = key - if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash - if hasattr(ENTRY, 'f_valid'): entry.f_valid = True + if DICT.entry_has_f_hash: entry.f_hash = hash + if DICT.entry_has_f_valid: entry.f_valid = True d.num_items += 1 if not everused: - if hasattr(ENTRY, 'f_everused'): entry.f_everused = True + if DICT.entry_has_f_everused: entry.f_everused = True d.num_pristine_entries -= 1 if d.num_pristine_entries <= len(d.entries) / 3: ll_dict_resize(d) @@ -443,35 +452,38 @@ # the dict contains no deleted entries. This routine has the advantage # of never calling d.keyhash() and d.keyeq(), so it cannot call back # to user code. ll_dict_insertclean() doesn't resize the dict, either. - entry = ll_dict_lookup_clean(d, hash) - ENTRY = lltype.typeOf(entry).TO + i = ll_dict_lookup(d, key, hash) + entry = d.entries[i] + DICT = lltype.typeOf(d).TO entry.value = value entry.key = key - if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash - if hasattr(ENTRY, 'f_valid'): entry.f_valid = True - if hasattr(ENTRY, 'f_everused'): entry.f_everused = True + if DICT.entry_has_f_hash: entry.f_hash = hash + if DICT.entry_has_f_valid: entry.f_valid = True + if DICT.entry_has_f_everused: entry.f_everused = True d.num_items += 1 d.num_pristine_entries -= 1 def ll_dict_delitem(d, key): - entry = ll_dict_lookup(d, key, d.keyhash(key)) - if not entry.valid(): + i = ll_dict_lookup(d, key, d.keyhash(key)) + if not d.entry_valid(i): raise KeyError - entry.mark_deleted() + d.mark_entry_deleted(i) d.num_items -= 1 # clear the key and the value if they are GC pointers - ENTRY = lltype.typeOf(entry).TO - if ENTRY.must_clear_key: + DICT = lltype.typeOf(d).TO + entry = d.entries[i] + if DICT.must_clear_key: key = entry.key # careful about destructor side effects: # keep key alive until entry.value has also # been zeroed (if it must be) - entry.key = lltype.nullptr(ENTRY.key.TO) - if ENTRY.must_clear_value: - entry.value = lltype.nullptr(ENTRY.value.TO) + entry.key = lltype.nullptr(DICT.entries.TO.OF.key.TO) + if DICT.must_clear_value: + entry.value = lltype.nullptr(DICT.entries.TO.OF.value.TO) num_entries = len(d.entries) if num_entries > DICT_INITSIZE and d.num_items < num_entries / 4: ll_dict_resize(d) + def ll_dict_resize(d): old_entries = d.entries old_size = len(old_entries) @@ -480,15 +492,34 @@ new_size = old_size * 2 while new_size > DICT_INITSIZE and d.num_items < new_size / 4: new_size /= 2 - d.entries = lltype.malloc(lltype.typeOf(old_entries).TO, new_size, zero=True) - d.num_items = 0 - d.num_pristine_entries = new_size + new_entries = lltype.malloc(lltype.typeOf(old_entries).TO, new_size, zero=True) + new_num_items = 0 + new_num_pristine_entries = new_size i = 0 while i < old_size: entry = old_entries[i] - if entry.valid(): - ll_dict_insertclean(d, entry.key, entry.value, entry.hash()) + if d.entry_valid(i): + hash = d.hash(i) + # AAAAAAAAA XXX :-( + old_entries = d.entries + old_num_items = d.num_items + old_num_pristine_entries = d.num_pristine_entries + d.entries = new_entries + d.num_items = new_num_items + d.num_pristine_entries = new_num_pristine_entries + + ll_dict_insertclean(d, entry.key, entry.value, hash) + + new_entries = d.entries + new_num_items = d.num_items + new_num_pristine_entries = d.num_pristine_entries + d.entries = old_entries + d.num_items = old_num_items + d.num_pristine_entries = old_num_pristine_entries i += 1 + d.entries = new_entries + d.num_items = new_num_items + d.num_pristine_entries = new_num_pristine_entries # ------- a port of CPython's dictobject.c's lookdict implementation ------- PERTURB_SHIFT = 5 @@ -500,26 +531,28 @@ i = r_uint(hash & mask) # do the first try before any looping entry = entries[i] - if entry.valid(): + found_freeslot = False + freeslot_index = r_uint(0) + if d.entry_valid(i): checkingkey = entry.key if checkingkey == key: - return entry # found the entry - if d.keyeq is not None and entry.hash() == hash: + return i # found the entry + if d.keyeq is not None and d.hash(i) == hash: # correct hash, maybe the key is e.g. a different pointer to # an equal object found = d.keyeq(checkingkey, key) if DICT.paranoia: if (entries != d.entries or - not entry.valid() or entry.key != checkingkey): + not d.entry_valid(i) or entry.key != checkingkey): # the compare did major nasty stuff to the dict: start over return ll_dict_lookup(d, key, hash) if found: - return entry # found the entry - freeslot = lltype.nullptr(lltype.typeOf(entry).TO) - elif entry.everused(): - freeslot = entry + return i # found the entry + elif d.entry_everused(i): + freeslot_index = i + found_freeslot = True else: - return entry # pristine entry -- lookup failed + return i # pristine entry -- lookup failed # In the loop, a deleted entry (everused and not valid) is by far # (factor of 100s) the least likely outcome, so test for that last. @@ -527,26 +560,30 @@ while 1: i = ((i << 2) + i + perturb + 1) & mask entry = entries[i] - if not entry.everused(): - return freeslot or entry - elif entry.valid(): + if not d.entry_everused(i): + if found_freeslot: + return freeslot_index + else: + return i + elif d.entry_valid(i): checkingkey = entry.key if checkingkey == key: - return entry - if d.keyeq is not None and entry.hash() == hash: + return i + if d.keyeq is not None and d.hash(i) == hash: # correct hash, maybe the key is e.g. a different pointer to # an equal object found = d.keyeq(checkingkey, key) if DICT.paranoia: if (entries != d.entries or - not entry.valid() or entry.key != checkingkey): + not d.entry_valid(i) or entry.key != checkingkey): # the compare did major nasty stuff to the dict: # start over return ll_dict_lookup(d, key, hash) if found: - return entry # found the entry - elif not freeslot: - freeslot = entry + return i # found the entry + elif not found_freeslot: + freeslot_index = i + found_freeslot = True perturb >>= PERTURB_SHIFT def ll_dict_lookup_clean(d, hash): @@ -556,13 +593,11 @@ entries = d.entries mask = len(entries) - 1 i = r_uint(hash & mask) - entry = entries[i] perturb = r_uint(hash) - while entry.everused(): + while d.entry_everused(i): i = ((i << 2) + i + perturb + 1) & mask - entry = entries[i] perturb >>= PERTURB_SHIFT - return entry + return i # ____________________________________________________________ # @@ -638,8 +673,9 @@ entries_len = len(entries) while index < entries_len: entry = entries[index] + i = index index = index + 1 - if entry.valid(): + if dict.entry_valid(i): iter.index = index if RETURNTYPE is lltype.Void: return None @@ -660,16 +696,16 @@ # methods def ll_get(dict, key, default): - entry = ll_dict_lookup(dict, key, dict.keyhash(key)) - if entry.valid(): - return entry.value + i = ll_dict_lookup(dict, key, dict.keyhash(key)) + if dict.entry_valid(i): + return dict.entries[i].value else: return default def ll_setdefault(dict, key, default): - entry = ll_dict_lookup(dict, key, dict.keyhash(key)) - if entry.valid(): - return entry.value + i = ll_dict_lookup(dict, key, dict.keyhash(key)) + if dict.entry_valid(i): + return dict.entries[i].value else: ll_dict_setitem(dict, key, default) return default @@ -689,10 +725,10 @@ entry = dict.entries[i] ENTRY = lltype.typeOf(entry).TO d_entry.key = entry.key - if hasattr(ENTRY, 'f_valid'): d_entry.f_valid = entry.f_valid - if hasattr(ENTRY, 'f_everused'): d_entry.f_everused = entry.f_everused + if DICT.entry_has_f_valid : d_entry.f_valid = entry.f_valid + if DICT.entry_has_f_everused: d_entry.f_everused = entry.f_everused d_entry.value = entry.value - if hasattr(ENTRY, 'f_hash'): d_entry.f_hash = entry.f_hash + if DICT.entry_has_f_hash: d_entry.f_hash = entry.f_hash i += 1 return d @@ -709,8 +745,8 @@ d2len = len(entries) i = 0 while i < d2len: - entry = entries[i] - if entry.valid(): + if dic2.entry_valid(i): + entry = entries[i] ll_dict_setitem(dic1, entry.key, entry.value) i += 1 @@ -733,8 +769,8 @@ i = 0 p = 0 while i < dlen: - entry = entries[i] - if entry.valid(): + if dic.entry_valid(i): + entry = entries[i] ELEM = lltype.typeOf(items).TO.OF if ELEM is not lltype.Void: if func is dum_items: @@ -751,7 +787,7 @@ return res def ll_contains(d, key): - entry = ll_dict_lookup(d, key, d.keyhash(key)) - return entry.valid() + i = ll_dict_lookup(d, key, d.keyhash(key)) + return d.entry_valid(i) ll_contains.oopspec = 'dict.contains(d, key)' ll_contains.oopargcheck = lambda d, key: bool(d) Modified: pypy/branch/rdict-index-based2/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/branch/rdict-index-based2/pypy/rpython/test/test_rdict.py (original) +++ pypy/branch/rdict-index-based2/pypy/rpython/test/test_rdict.py Tue Oct 2 20:45:00 2007 @@ -548,7 +548,7 @@ res = self.interpret(func2, [ord(x), ord(y)]) for i in range(len(res.entries)): - assert not (res.entries[i].everused() and not res.entries[i].valid()) + assert not (res.entry_everused(i) and not res.entry_valid(i)) def func3(c0, c1, c2, c3, c4, c5, c6, c7): d = {} @@ -568,7 +568,7 @@ for i in range(rdict.DICT_INITSIZE)]) count_frees = 0 for i in range(len(res.entries)): - if not res.entries[i].everused(): + if not res.entry_everused(i): count_frees += 1 assert count_frees >= 3 @@ -610,8 +610,8 @@ res = self.interpret(f, []) assert res.item0 == True DICT = lltype.typeOf(res.item1).TO - assert not hasattr(DICT.entries.TO.OF, 'f_everused')# non-None string keys - assert not hasattr(DICT.entries.TO.OF, 'f_valid') # strings have a dummy + assert not DICT.entry_has_f_everused# non-None string keys + assert not DICT.entry_has_f_valid # strings have a dummy def test_opt_nullvaluemarker(self): def f(n): @@ -621,8 +621,8 @@ res = self.interpret(f, [-5]) assert res.item0 == 4 DICT = lltype.typeOf(res.item1).TO - assert not hasattr(DICT.entries.TO.OF, 'f_everused')# non-None str values - assert not hasattr(DICT.entries.TO.OF, 'f_valid') # strs have a dummy + assert not DICT.entry_has_f_everused# non-None string keys + assert not DICT.entry_has_f_valid # strings have a dummy def test_opt_nonullmarker(self): class A: @@ -638,8 +638,8 @@ res = self.interpret(f, [-5]) assert res.item0 == -5441 DICT = lltype.typeOf(res.item1).TO - assert hasattr(DICT.entries.TO.OF, 'f_everused') # can-be-None A instances - assert not hasattr(DICT.entries.TO.OF, 'f_valid')# with a dummy A instance + assert DICT.entry_has_f_everused # can-be-None A instances + assert not DICT.entry_has_f_valid# with a dummy A instance res = self.interpret(f, [6]) assert res.item0 == -5441 @@ -654,8 +654,8 @@ assert res.item0 == 1 assert res.item1 == 24 DICT = lltype.typeOf(res.item2).TO - assert hasattr(DICT.entries.TO.OF, 'f_everused') # all ints can be zero - assert not hasattr(DICT.entries.TO.OF, 'f_valid')# nonneg int: dummy -1 + assert DICT.entry_has_f_everused # all ints can be zero + assert not DICT.entry_has_f_valid# nonneg int: dummy -1 def test_opt_no_dummy(self): def f(n): @@ -667,8 +667,8 @@ assert res.item0 == 1 assert res.item1 == -24 DICT = lltype.typeOf(res.item2).TO - assert hasattr(DICT.entries.TO.OF, 'f_everused') # all ints can be zero - assert hasattr(DICT.entries.TO.OF, 'f_valid') # no dummy available + assert DICT.entry_has_f_everused # all ints can be zero + assert DICT.entry_has_f_valid # no dummy available def test_opt_multiple_identical_dicts(self): def f(n): From cfbolz at codespeak.net Tue Oct 2 20:47:45 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 2 Oct 2007 20:47:45 +0200 (CEST) Subject: [pypy-svn] r47104 - pypy/branch/rdict-index-based Message-ID: <20071002184745.44D838124@code0.codespeak.net> Author: cfbolz Date: Tue Oct 2 20:47:44 2007 New Revision: 47104 Removed: pypy/branch/rdict-index-based/ Log: kill the rdict-index-based branch, since its ideas were put into the rdict-index-based2 branch. From cfbolz at codespeak.net Tue Oct 2 22:48:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 2 Oct 2007 22:48:53 +0200 (CEST) Subject: [pypy-svn] r47107 - pypy/branch/kill-keepalives-again Message-ID: <20071002204853.513738144@code0.codespeak.net> Author: cfbolz Date: Tue Oct 2 22:48:53 2007 New Revision: 47107 Added: pypy/branch/kill-keepalives-again/ - copied from r47106, pypy/dist/ Log: an experimental branch (that will probably be scrapped) to experiment with the stuff from the old kill-keepalive branch. From cfbolz at codespeak.net Wed Oct 3 00:51:41 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 3 Oct 2007 00:51:41 +0200 (CEST) Subject: [pypy-svn] r47109 - in pypy/branch/kill-keepalives-again/pypy: annotation rpython rpython/lltypesystem rpython/test Message-ID: <20071002225141.73C1F813B@code0.codespeak.net> Author: cfbolz Date: Wed Oct 3 00:51:40 2007 New Revision: 47109 Modified: pypy/branch/kill-keepalives-again/pypy/annotation/bookkeeper.py pypy/branch/kill-keepalives-again/pypy/annotation/builtin.py pypy/branch/kill-keepalives-again/pypy/annotation/model.py pypy/branch/kill-keepalives-again/pypy/annotation/unaryop.py pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lloperation.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/opimpl.py pypy/branch/kill-keepalives-again/pypy/rpython/rptr.py pypy/branch/kill-keepalives-again/pypy/rpython/test/test_rptr.py Log: first step in reviving the kill-keepalive branch: make the new way of rtyping the access of interior pointers work again. rptr tests pass. Modified: pypy/branch/kill-keepalives-again/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/kill-keepalives-again/pypy/annotation/bookkeeper.py Wed Oct 3 00:51:40 2007 @@ -11,7 +11,7 @@ SomeInteger, SomeExternalObject, SomeOOInstance, TLS, SomeAddress, \ SomeUnicodeCodePoint, SomeOOStaticMeth, s_None, s_ImpossibleValue, \ SomeLLADTMeth, SomeBool, SomeTuple, SomeOOClass, SomeImpossibleValue, \ - SomeList, SomeObject, HarmlesslyBlocked, SomeWeakRef + SomeList, SomeObject, HarmlesslyBlocked, SomeWeakRef, lltype_to_annotation from pypy.annotation.classdef import ClassDef, InstanceSource from pypy.annotation.listdef import ListDef, MOST_GENERAL_LISTDEF from pypy.annotation.dictdef import DictDef, MOST_GENERAL_DICTDEF @@ -239,7 +239,7 @@ if isinstance(s_callable, SomeLLADTMeth): adtmeth = s_callable s_callable = self.immutablevalue(adtmeth.func) - args_s = [SomePtr(adtmeth.ll_ptrtype)] + args_s + args_s = [lltype_to_annotation(adtmeth.ll_ptrtype)] + args_s if isinstance(s_callable, SomePBC): s_result = binding(call_op.result, s_ImpossibleValue) self.consider_call_site_for_pbc(s_callable, Modified: pypy/branch/kill-keepalives-again/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/annotation/builtin.py (original) +++ pypy/branch/kill-keepalives-again/pypy/annotation/builtin.py Wed Oct 3 00:51:40 2007 @@ -628,7 +628,6 @@ # memory address from pypy.rpython.memory import lladdress -from pypy.rpython.lltypesystem import llmemory def raw_malloc(s_size): assert isinstance(s_size, SomeInteger) #XXX add noneg...? @@ -669,7 +668,6 @@ #_________________________________ # offsetof/sizeof -from pypy.rpython.lltypesystem import llmemory def offsetof(TYPE, fldname): return SomeInteger() Modified: pypy/branch/kill-keepalives-again/pypy/annotation/model.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/annotation/model.py (original) +++ pypy/branch/kill-keepalives-again/pypy/annotation/model.py Wed Oct 3 00:51:40 2007 @@ -551,6 +551,11 @@ def can_be_none(self): return False +class SomeInteriorPtr(SomePtr): + def __init__(self, ll_ptrtype): + assert isinstance(ll_ptrtype, lltype.InteriorPtr) + self.ll_ptrtype = ll_ptrtype + class SomeLLADTMeth(SomeObject): immutable = True def __init__(self, ll_ptrtype, func): @@ -597,6 +602,13 @@ return s_val.ootype if isinstance(s_val, SomeOOStaticMeth): return s_val.method + if isinstance(s_val, SomeInteriorPtr): + p = s_val.ll_ptrtype + if 0 in p.offsets: + assert list(p.offsets).count(0) == 1 + return lltype.Ptr(lltype.Ptr(p.PARENTTYPE)._interior_ptr_type_with_index(p.TO)) + else: + return lltype.Ptr(p.PARENTTYPE) if isinstance(s_val, SomePtr): return s_val.ll_ptrtype for witness, T in annotation_to_ll_map: @@ -630,6 +642,8 @@ return SomeOOClass(ootype.ROOT) elif isinstance(T, ExternalType): return SomeExternalInstance(T._class_) + elif isinstance(T, lltype.InteriorPtr): + return SomeInteriorPtr(T) else: return SomePtr(T) else: @@ -642,13 +656,19 @@ return s_None if isinstance(v, MethodType): ll_ptrtype = lltype.typeOf(v.im_self) - assert isinstance(ll_ptrtype, lltype.Ptr) + assert isinstance(ll_ptrtype, (lltype.Ptr, lltype.InteriorPtr)) return SomeLLADTMeth(ll_ptrtype, v.im_func) if isinstance(v, FunctionType): # this case should only be for staticmethod instances used in # adtmeths: the getattr() result is then a plain FunctionType object. from pypy.annotation.bookkeeper import getbookkeeper return getbookkeeper().immutablevalue(v) + if isinstance(v, lltype._interior_ptr): + ob = v._parent + if ob is None: + raise RuntimeError + T = lltype.InteriorPtr(lltype.typeOf(ob), v._T, v._offsets) + return SomeInteriorPtr(T) return lltype_to_annotation(lltype.typeOf(v)) # ____________________________________________________________ Modified: pypy/branch/kill-keepalives-again/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/annotation/unaryop.py (original) +++ pypy/branch/kill-keepalives-again/pypy/annotation/unaryop.py Wed Oct 3 00:51:40 2007 @@ -691,7 +691,7 @@ def call(adtmeth, args): bookkeeper = getbookkeeper() s_func = bookkeeper.immutablevalue(adtmeth.func) - return s_func.call(args.prepend(SomePtr(adtmeth.ll_ptrtype))) + return s_func.call(args.prepend(lltype_to_annotation(adtmeth.ll_ptrtype))) from pypy.rpython.ootypesystem import ootype class __extend__(SomeOOInstance): Modified: pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py Wed Oct 3 00:51:40 2007 @@ -558,6 +558,31 @@ result = self.op_direct_call(write_barrier, *args) op_bare_setfield = op_setfield + def op_getinteriorfield(self, obj, *offsets): + checkptr(obj) + ob = obj + for o in offsets: + if isinstance(o, str): + ob = getattr(ob, o) + else: + ob = ob[o] + assert not isinstance(ob, lltype._interior_ptr) + return ob + + def op_setinteriorfield(self, obj, *fieldnamesval): + prefields, finalfield, fieldvalue = ( + fieldnamesval[:-2], fieldnamesval[-2], fieldnamesval[-1]) + for o in prefields: + if isinstance(o, str): + obj = getattr(obj, o) + else: + obj = obj[o] + if isinstance(finalfield, str): + setattr(obj, finalfield, fieldvalue) + else: + obj[finalfield] = fieldvalue + op_bare_setinteriorfield = op_setinteriorfield + def op_getarrayitem(self, array, index): return array[index] Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lloperation.py Wed Oct 3 00:51:40 2007 @@ -320,6 +320,9 @@ 'getarrayitem': LLOp(sideeffects=False, canrun=True), 'getarraysize': LLOp(canfold=True), 'getsubstruct': LLOp(canfold=True), + 'getinteriorfield': LLOp(sideeffects=False, canrun=True), + 'getinteriorarraysize': LLOp(canfold=True), + 'setinteriorfield': LLOp(), 'getarraysubstruct': LLOp(canfold=True), 'setfield': LLOp(), 'bare_setfield': LLOp(), Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py Wed Oct 3 00:51:40 2007 @@ -631,6 +631,34 @@ o = self.TO._container_example() return _ptr(self, o, solid=True) + def _interior_ptr_type_with_index(self, TO): + assert self.TO._gckind == 'gc' + if isinstance(TO, Struct): + R = GcStruct("Interior", ('ptr', self), ('index', Signed), + hints={'interior_ptr_type':True}, + adtmeths=TO._adtmeths) + else: + R = GcStruct("Interior", ('ptr', self), ('index', Signed), + hints={'interior_ptr_type':True}) + return R + +class InteriorPtr(LowLevelType): + def __init__(self, PARENTTYPE, TO, offsets): + self.PARENTTYPE = PARENTTYPE + self.TO = TO + self.offsets = tuple(offsets) + def __str__(self): + return '%s (%s).%s'%(self.__class__.__name__, + self.PARENTTYPE._short_name(), + '.'.join(map(str, self.offsets))) + def _example(self): + ob = Ptr(self.PARENTTYPE)._example() + for o in self.offsets: + if isinstance(o, str): + ob = getattr(ob, o) + else: + ob = ob[0] + return ob # ____________________________________________________________ @@ -667,6 +695,12 @@ return val.lltype() raise TypeError("typeOf(%r object)" % (tp.__name__,)) +def rawTypeOf(val): + if isinstance(val, _interior_ptr): + return Ptr(val._T) + else: + return typeOf(val) + _to_primitive = { Char: chr, UniChar: unichr, @@ -850,13 +884,6 @@ parent, base = parentlink(ptr._obj) return _subarray._makeptr(parent, base + n, ptr._solid) -def _expose(val, solid=False): - """XXX A nice docstring here""" - T = typeOf(val) - if isinstance(T, ContainerType): - val = _ptr(Ptr(T), val, solid=solid) - return val - def parentlink(container): parent = container._parentstructure() if parent is not None: @@ -903,26 +930,14 @@ class UninitializedMemoryAccess(Exception): pass -class _ptr(object): - __slots__ = ('_TYPE', '_T', - '_weak', '_solid', - '_obj0', '__weakref__') +class _abstract_ptr(object): + __slots__ = ('_T',) - def _set_TYPE(self, TYPE): - _ptr._TYPE.__set__(self, TYPE) + # assumes one can access _TYPE, _expose and _obj def _set_T(self, T): _ptr._T.__set__(self, T) - def _set_weak(self, weak): - _ptr._weak.__set__(self, weak) - - def _set_solid(self, solid): - _ptr._solid.__set__(self, solid) - - def _set_obj0(self, obj): - _ptr._obj0.__set__(self, obj) - def _togckind(self): return self._T._gckind @@ -930,19 +945,8 @@ # XXX deprecated interface return self._TYPE._needsgc() # xxx other rules? - def __init__(self, TYPE, pointing_to, solid=False): - self._set_TYPE(TYPE) - self._set_T(TYPE.TO) - self._set_weak(False) - self._setobj(pointing_to, solid) - - def _become(self, other): - assert self._TYPE == other._TYPE - assert not self._weak - self._setobj(other._obj, other._solid) - def __eq__(self, other): - if not isinstance(other, _ptr): + if type(self) is not type(other): raise TypeError("comparing pointer with %r object" % ( type(other).__name__,)) if self._TYPE != other._TYPE: @@ -997,7 +1001,7 @@ if isinstance(self._T, Struct): if field_name in self._T._flds: o = self._obj._getattr(field_name) - return _expose(o, self._solid) + return self._expose(field_name, o) if isinstance(self._T, ContainerType): try: adtmeth = self._T._adtmeths[field_name] @@ -1013,20 +1017,6 @@ raise AttributeError("%r instance has no field %r" % (self._T, field_name)) - #def _setfirst(self, p): - # if isinstance(self._T, Struct) and self._T._names: - # if not isinstance(p, _ptr) or not isinstance(p._obj, _struct): - # raise InvalidCast(typeOf(p), typeOf(self)) - # field_name = self._T._names[0] - # T1 = self._T._flds[field_name] - # T2 = typeOf(p._obj) - # if T1 != T2: - # raise InvalidCast(typeOf(p), typeOf(self)) - # setattr(self._obj, field_name, p._obj) - # p._obj._setparentstructure(self._obj, 0) - # return - # raise TypeError("%r instance has no first field" % (self._T,)) - def __setattr__(self, field_name, val): if isinstance(self._T, Struct): if field_name in self._T._flds: @@ -1050,7 +1040,7 @@ raise TypeError("array slicing not supported") raise IndexError("array index out of bounds") o = self._obj.getitem(i) - return _expose(o, self._solid) + return self._expose(i, o) raise TypeError("%r instance is not an array" % (self._T,)) def __setitem__(self, i, val): @@ -1124,6 +1114,34 @@ return callb(*args) raise TypeError("%r instance is not a function" % (self._T,)) +class _ptr(_abstract_ptr): + __slots__ = ('_TYPE', + '_weak', '_solid', + '_obj0', '__weakref__') + + def _set_TYPE(self, TYPE): + _ptr._TYPE.__set__(self, TYPE) + + def _set_weak(self, weak): + _ptr._weak.__set__(self, weak) + + def _set_solid(self, solid): + _ptr._solid.__set__(self, solid) + + def _set_obj0(self, obj): + _ptr._obj0.__set__(self, obj) + + def __init__(self, TYPE, pointing_to, solid=False): + self._set_TYPE(TYPE) + self._set_T(TYPE.TO) + self._set_weak(False) + self._setobj(pointing_to, solid) + + def _become(self, other): + assert self._TYPE == other._TYPE + assert not self._weak + self._setobj(other._obj, other._solid) + def _cast_to(self, PTRTYPE): CURTYPE = self._TYPE down_or_up = castable(PTRTYPE, CURTYPE) @@ -1188,8 +1206,67 @@ def _as_obj(self): return self._obj + def _expose(self, offset, val): + """XXX A nice docstring here""" + T = typeOf(val) + if isinstance(T, ContainerType): + if self._T._gckind == 'gc' and T._gckind == 'raw' and not isinstance(T, OpaqueType): + val = _interior_ptr(T, self._obj, [offset]) + else: + val = _ptr(Ptr(T), val, solid=self._solid) + return val + assert not '__dict__' in dir(_ptr) +class _interior_ptr(_abstract_ptr): + __slots__ = ('_parent', '_offsets') + def _set_parent(self, _parent): + _interior_ptr._parent.__set__(self, _parent) + def _set_offsets(self, _offsets): + _interior_ptr._offsets.__set__(self, _offsets) + + def __init__(self, _T, _parent, _offsets): + self._set_T(_T) + #self._set_parent(weakref.ref(_parent)) + self._set_parent(_parent) + self._set_offsets(_offsets) + + def __nonzero__(self): + raise RuntimeError, "do not test an interior pointer for nullity" + + def _get_obj(self): + ob = self._parent + if ob is None: + raise RuntimeError + if isinstance(ob, _container): + ob._check() + for o in self._offsets: + if isinstance(o, str): + ob = ob._getattr(o) + else: + ob = ob.getitem(o) + return ob + _obj = property(_get_obj) + + def _get_TYPE(self): + ob = self._parent + if ob is None: + raise RuntimeError + return InteriorPtr(typeOf(ob), self._T, self._offsets) +## _TYPE = property(_get_TYPE) + + def _expose(self, offset, val): + """XXX A nice docstring here""" + T = typeOf(val) + if isinstance(T, ContainerType): + assert T._gckind == 'raw' + val = _interior_ptr(T, self._parent, self._offsets + [offset]) + return val + + + +assert not '__dict__' in dir(_interior_ptr) + class _container(object): __slots__ = () def _parentstructure(self): @@ -1261,7 +1338,8 @@ if parent is None: break T = typeOf(parent) - if not isinstance(T, Struct) or T._first_struct()[0] != index: + if (not isinstance(T, Struct) or T._first_struct()[0] != index + or isinstance(T, FixedSizeArray)): break container = parent return container Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/opimpl.py Wed Oct 3 00:51:40 2007 @@ -120,6 +120,31 @@ # the diff between op_getarrayitem and op_getarraysubstruct # is the same as between op_getfield and op_getsubstruct +def op_getinteriorarraysize(obj, *offsets): + checkptr(obj) + ob = obj + for o in offsets: + if isinstance(o, str): + ob = getattr(ob, o) + else: + ob = ob[o] + return len(ob) + +def op_getinteriorfield(obj, *offsets): + checkptr(obj) + ob = obj + T = lltype.typeOf(obj).TO + for o in offsets: + if not T._hints.get('immutable'): + raise TypeError("cannot fold getinteriorfield on mutable struct") + if isinstance(o, str): + ob = getattr(ob, o) + T = getattr(T, o) + else: + raise TypeError("cannot fold getfield on mutable struct") + assert not isinstance(ob, lltype._interior_ptr) + return ob + def op_getarraysize(array): checkptr(array) return len(array) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rptr.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/rptr.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/rptr.py Wed Oct 3 00:51:40 2007 @@ -19,6 +19,10 @@ ## else: return self.__class__, self.ll_ptrtype +class __extend__(annmodel.SomeInteriorPtr): + def rtyper_makerepr(self, rtyper): + return InteriorPtrRepr(self.ll_ptrtype) + class PtrRepr(Repr): @@ -37,7 +41,14 @@ return hop.inputarg(hop.r_result, arg=0) FIELD_TYPE = getattr(self.lowleveltype.TO, attr) if isinstance(FIELD_TYPE, lltype.ContainerType): - newopname = 'getsubstruct' + if (attr, FIELD_TYPE) == self.lowleveltype.TO._first_struct(): + return hop.genop('cast_pointer', [hop.inputarg(self, 0)], + resulttype=hop.r_result.lowleveltype) + elif isinstance(hop.r_result, InteriorPtrRepr): + return hop.genop('same_as', [hop.inputarg(self, 0)], + resulttype=self.lowleveltype) + else: + newopname = 'getsubstruct' else: newopname = 'getfield' vlist = hop.inputargs(self, lltype.Void) @@ -98,7 +109,21 @@ ARRAY = r_ptr.lowleveltype.TO ITEM_TYPE = ARRAY.OF if isinstance(ITEM_TYPE, lltype.ContainerType): - newopname = 'getarraysubstruct' + if isinstance(hop.r_result, InteriorPtrRepr): + v_array, v_index = hop.inputargs(r_ptr, lltype.Signed) + INTERIOR_PTR_TYPE = r_ptr.lowleveltype._interior_ptr_type_with_index(ITEM_TYPE) + cflags = hop.inputconst(lltype.Void, {'flavor': 'gc'}) + args = [flowmodel.Constant(INTERIOR_PTR_TYPE, lltype.Void), + cflags] + v_interior_ptr = hop.genop('malloc', args, + resulttype=lltype.Ptr(INTERIOR_PTR_TYPE)) + hop.genop('setfield', + [v_interior_ptr, flowmodel.Constant('ptr', lltype.Void), v_array]) + hop.genop('setfield', + [v_interior_ptr, flowmodel.Constant('index', lltype.Void), v_index]) + return v_interior_ptr + else: + newopname = 'getarraysubstruct' else: newopname = 'getarrayitem' vlist = hop.inputargs(r_ptr, lltype.Signed) @@ -159,23 +184,26 @@ class __extend__(annmodel.SomeLLADTMeth): def rtyper_makerepr(self, rtyper): - return LLADTMethRepr(self) + return LLADTMethRepr(self, rtyper) def rtyper_makekey(self): return self.__class__, self.ll_ptrtype, self.func class LLADTMethRepr(Repr): - def __init__(self, adtmeth): + def __init__(self, adtmeth, rtyper): self.func = adtmeth.func self.lowleveltype = adtmeth.ll_ptrtype - + self.ll_ptrtype = adtmeth.ll_ptrtype + self.lowleveltype = rtyper.getrepr(annmodel.lltype_to_annotation(adtmeth.ll_ptrtype)).lowleveltype + def rtype_simple_call(self, hop): hop2 = hop.copy() func = self.func s_func = hop.rtyper.annotator.bookkeeper.immutablevalue(func) v_ptr = hop2.args_v[0] hop2.r_s_popfirstarg() - hop2.v_s_insertfirstarg(v_ptr, annmodel.SomePtr(self.lowleveltype)) + hop2.v_s_insertfirstarg( + v_ptr, annmodel.lltype_to_annotation(self.ll_ptrtype)) hop2.v_s_insertfirstarg(flowmodel.Constant(func), s_func) return hop2.dispatch() @@ -186,4 +214,121 @@ return v return NotImplemented - +class InteriorPtrRepr(Repr): + def __init__(self, ptrtype): + assert isinstance(ptrtype, lltype.InteriorPtr) + self.v_offsets = [] + numitemoffsets = 0 + for i, offset in enumerate(ptrtype.offsets): + if isinstance(offset, int): + numitemoffsets += 1 + self.v_offsets.append(None) + else: + assert isinstance(offset, str) + self.v_offsets.append(flowmodel.Constant(offset, lltype.Void)) + self.parentptrtype = lltype.Ptr(ptrtype.PARENTTYPE) + self.resulttype = lltype.Ptr(ptrtype.TO) + assert numitemoffsets <= 1 + if numitemoffsets > 0: + self.lowleveltype = lltype.Ptr(self.parentptrtype._interior_ptr_type_with_index(self.resulttype.TO)) + else: + self.lowleveltype = self.parentptrtype + + def getinteriorfieldargs(self, hop, v_self): + vlist = [] + if None in self.v_offsets: + INTERIOR_TYPE = v_self.concretetype.TO + nameiter = iter(INTERIOR_TYPE._names) + name = nameiter.next() + vlist.append( + hop.genop('getfield', + [v_self, flowmodel.Constant(name, lltype.Void)], + resulttype=INTERIOR_TYPE._flds[name])) + else: + vlist.append(v_self) + for v_offset in self.v_offsets: + if v_offset is None: + name = nameiter.next() + vlist.append( + hop.genop('getfield', + [v_self, flowmodel.Constant(name, lltype.Void)], + resulttype=INTERIOR_TYPE._flds[name])) + else: + vlist.append(v_offset) + if None in self.v_offsets: + try: + nameiter.next() + except StopIteration: + pass + else: + assert False + return vlist + + def rtype_len(self, hop): + v_self, = hop.inputargs(self) + vlist = self.getinteriorfieldargs(hop, v_self) + return hop.genop('getinteriorarraysize', vlist, + resulttype=lltype.Signed) + + def rtype_getattr(self, hop): + attr = hop.args_s[1].const + if isinstance(hop.s_result, annmodel.SomeLLADTMeth): + return hop.inputarg(hop.r_result, arg=0) + FIELD_TYPE = getattr(self.resulttype.TO, attr) + if isinstance(FIELD_TYPE, lltype.ContainerType): + return hop.genop('same_as', [hop.inputarg(self, 0)], + resulttype=self.lowleveltype) + else: + v_self, v_attr = hop.inputargs(self, lltype.Void) + vlist = self.getinteriorfieldargs(hop, v_self) + [v_attr] + return hop.genop('getinteriorfield', vlist, + resulttype=hop.r_result.lowleveltype) + + def rtype_setattr(self, hop): + attr = hop.args_s[1].const + FIELD_TYPE = getattr(self.resulttype.TO, attr) + assert not isinstance(FIELD_TYPE, lltype.ContainerType) + v_self, v_fieldname, v_value = hop.inputargs(self, lltype.Void, hop.args_r[2]) + vlist = self.getinteriorfieldargs(hop, v_self) + [v_fieldname, v_value] + return hop.genop('setinteriorfield', vlist) + + + + +class __extend__(pairtype(InteriorPtrRepr, IntegerRepr)): + def rtype_getitem((r_ptr, r_item), hop): + ARRAY = r_ptr.resulttype.TO + ITEM_TYPE = ARRAY.OF + if isinstance(ITEM_TYPE, lltype.ContainerType): + v_array, v_index = hop.inputargs(r_ptr, lltype.Signed) + INTERIOR_PTR_TYPE = r_ptr.lowleveltype._interior_ptr_type_with_index(ITEM_TYPE) + cflags = hop.inputconst(lltype.Void, {'flavor': 'gc'}) + args = [flowmodel.Constant(INTERIOR_PTR_TYPE, lltype.Void), cflags] + v_interior_ptr = hop.genop('malloc', args, + resulttype=lltype.Ptr(INTERIOR_PTR_TYPE)) + hop.genop('setfield', + [v_interior_ptr, flowmodel.Constant('ptr', lltype.Void), v_array]) + hop.genop('setfield', + [v_interior_ptr, flowmodel.Constant('index', lltype.Void), v_index]) + return v_interior_ptr + else: + v_self, v_index = hop.inputargs(r_ptr, lltype.Signed) + vlist = r_ptr.getinteriorfieldargs(hop, v_self) + [v_index] + return hop.genop('getinteriorfield', vlist, + resulttype=ITEM_TYPE) + + def rtype_setitem((r_ptr, r_index), hop): + ARRAY = r_ptr.resulttype.TO + ITEM_TYPE = ARRAY.OF + assert not isinstance(ITEM_TYPE, lltype.ContainerType) + v_self, v_index, v_value = hop.inputargs(r_ptr, lltype.Signed, hop.args_r[2]) + vlist = r_ptr.getinteriorfieldargs(hop, v_self) + [v_index, v_value] + hop.genop('setinteriorfield', vlist) + +class __extend__(pairtype(InteriorPtrRepr, LLADTMethRepr)): + + def convert_from_to((r_from, r_to), v, llops): + if r_from.lowleveltype == r_to.lowleveltype: + return v + return NotImplemented + Modified: pypy/branch/kill-keepalives-again/pypy/rpython/test/test_rptr.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/test/test_rptr.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/test/test_rptr.py Wed Oct 3 00:51:40 2007 @@ -234,3 +234,82 @@ res = interpret(f, []) assert res.chars[0] == '0' assert res.chars[1] == 'x' + + +def test_first_subfield_access_is_cast_pointer(): + B = GcStruct("B", ('x', Signed)) + C = GcStruct("C", ('super', B), ('y', Signed)) + def f(): + c = malloc(C) + c.super.x = 1 + c.y = 2 + return c.super.x + c.y + s, t = ll_rtype(f, []) + from pypy.translator.translator import graphof + from pypy.objspace.flow.model import summary + graph = graphof(t, f) + graphsum = summary(graph) + assert 'getsubstruct' not in graphsum + assert 'cast_pointer' in graphsum + + + +def test_interior_ptr(): + S = Struct("S", ('x', Signed)) + T = GcStruct("T", ('s', S)) + def f(): + t = malloc(T) + t.s.x = 1 + return t.s.x + res = interpret(f, []) + assert res == 1 + +def test_interior_ptr_with_index(): + S = Struct("S", ('x', Signed)) + T = GcArray(S) + def f(): + t = malloc(T, 1) + t[0].x = 1 + return t[0].x + res = interpret(f, []) + assert res == 1 + +def test_interior_ptr_with_field_and_index(): + S = Struct("S", ('x', Signed)) + T = GcStruct("T", ('items', Array(S))) + def f(): + t = malloc(T, 1) + t.items[0].x = 1 + return t.items[0].x + res = interpret(f, []) + assert res == 1 + +def test_interior_ptr_with_index_and_field(): + S = Struct("S", ('x', Signed)) + T = Struct("T", ('s', S)) + U = GcArray(T) + def f(): + u = malloc(U, 1) + u[0].s.x = 1 + return u[0].s.x + res = interpret(f, []) + assert res == 1 + +def test_interior_ptr_len(): + S = Struct("S", ('x', Signed)) + T = GcStruct("T", ('items', Array(S))) + def f(): + t = malloc(T, 1) + return len(t.items) + res = interpret(f, []) + assert res == 1 + +def test_interior_ptr_with_setitem(): + T = GcStruct("T", ('s', Array(Signed))) + def f(): + t = malloc(T, 1) + t.s[0] = 1 + return t.s[0] + res = interpret(f, []) + assert res == 1 + From cfbolz at codespeak.net Wed Oct 3 01:07:33 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 3 Oct 2007 01:07:33 +0200 (CEST) Subject: [pypy-svn] r47110 - pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test Message-ID: <20071002230733.EAF9D813B@code0.codespeak.net> Author: cfbolz Date: Wed Oct 3 01:07:32 2007 New Revision: 47110 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_lltype.py Log: fix test_lltype Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_lltype.py Wed Oct 3 01:07:32 2007 @@ -1,7 +1,11 @@ from pypy.rpython.lltypesystem.lltype import * def isweak(p, T): - return p._weak and typeOf(p).TO == T + try: + typeOf(p) + except TypeError: + return True + return False def test_basics(): S0 = GcStruct("s0", ('a', Signed), ('b', Signed)) @@ -202,6 +206,7 @@ py.test.raises(RuntimeError, "cast_pointer(Ptr(S1), p3)") def test_best_effort_gced_parent_detection(): + py.test.skip("test not relevant any more") S2 = Struct("s2", ('a', Signed)) S1 = GcStruct("s1", ('sub1', S2), ('sub2', S2), ('tail', Array(('e', Signed)))) p1 = malloc(S1, 1) @@ -216,6 +221,7 @@ py.test.raises(RuntimeError, "p3[0]") def test_best_effort_gced_parent_for_arrays(): + py.test.skip("test not relevant any more") A1 = GcArray(('v', Signed)) p1 = malloc(A1, 10) p1[5].v=3 @@ -422,6 +428,8 @@ assert runtime_type_info(s1.sub) == getRuntimeTypeInfo(S1) def test_flavor_malloc(): + def isweak(p, T): + return p._weak and typeOf(p).TO == T S = Struct('s', ('x', Signed)) py.test.raises(TypeError, malloc, S) p = malloc(S, flavor="raw") From niko at codespeak.net Wed Oct 3 06:31:50 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Wed, 3 Oct 2007 06:31:50 +0200 (CEST) Subject: [pypy-svn] r47111 - in pypy/dist/pypy/translator/jvm: . src/pypy test Message-ID: <20071003043150.ED6838120@code0.codespeak.net> Author: niko Date: Wed Oct 3 06:31:49 2007 New Revision: 47111 Added: pypy/dist/pypy/translator/jvm/src/pypy/RecordStringString.java Modified: pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java pypy/dist/pypy/translator/jvm/test/test_builtin.py pypy/dist/pypy/translator/jvm/typesystem.py Log: fix a few of the builtin functions Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Wed Oct 3 06:31:49 2007 @@ -473,6 +473,11 @@ ootype.CustomDict: jvmtype.jPyPyCustomDict, ootype.WeakReference: jvmtype.jPyPyWeakRef, ll_os.STAT_RESULT: jvmtype.jPyPyStatResult, + + # These are some configured records that are generated by Java + # code. + ootype.Record({"item0": ootype.String, "item1": ootype.String}): + jvmtype.jPyPyRecordStringString, } def lltype_to_cts(self, OOT): Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Wed Oct 3 06:31:49 2007 @@ -269,7 +269,11 @@ return result; } - // Used in testing: + // Used in testing the JVM backend: + // + // A series of methods which serve a similar purpose to repr() in Python: + // they create strings that can be exec'd() to rebuild data structures. + // Also methods for writing to System.out. public static void dump(String text) { System.out.println(text); @@ -343,6 +347,17 @@ public static String serializeObject(Object o) { if (o == null) return "None"; + if (o instanceof ArrayList) { + StringBuffer sb = new StringBuffer(); + sb.append("["); + for (Object obj : (ArrayList)o) + sb.append(serializeObject(obj)).append(","); + sb.append("]"); + return sb.toString(); + } + else if (o instanceof String) { + return escaped_string((String)o); + } return o.toString(); } @@ -800,8 +815,20 @@ public static double ll_time_time() { return System.currentTimeMillis()/1000.0; } - - + + public static void ll_time_sleep(double seconds) + { + double startTime = ll_time_time(); + double endTime = startTime + seconds; + do { + try { + Thread.sleep((int)((endTime-startTime)*1000)); + return; + } catch (InterruptedException exc) {} + startTime = ll_time_time(); + } while (startTime < endTime); + } + public static String ll_join(String a, String b) { return a + "/" + b; // XXX Added: pypy/dist/pypy/translator/jvm/src/pypy/RecordStringString.java ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/src/pypy/RecordStringString.java Wed Oct 3 06:31:49 2007 @@ -0,0 +1,10 @@ +package pypy; + +public final class RecordStringString { + public String item0, item1; + public RecordStringString() { + } + public RecordStringString(String i0, String i1) { + item0 = i0; item1 = i1; + } +} Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Wed Oct 3 06:31:49 2007 @@ -4,6 +4,9 @@ import java.util.HashMap; import java.util.ArrayList; import java.util.Map; +import java.util.Set; +import java.util.Iterator; +import java.util.Arrays; abstract class FileWrapper { @@ -323,10 +326,52 @@ else return false; } + + public static String ll_os_getenv(String key) + { + return System.getenv(key); + } + + public static void ll_os_putenv(String key, String value) + { + //System.setenv(key, value); + // it appears that there is no such method??!! + } + + public static ArrayList ll_os_envkeys() + { + Map variables = System.getenv(); + Set variableNames = variables.keySet(); + return new ArrayList(variableNames); + } public static ArrayList ll_os_envitems() { - return new ArrayList(); // XXX + Map variables = System.getenv(); + Set variableNames = variables.keySet(); + Iterator nameIterator = variableNames.iterator(); + ArrayList result = new ArrayList(); + + for (int index = 0; index < variableNames.size(); index++) + { + String name = (String) nameIterator.next(); + String value = (String) variables.get(name); + result.add(new RecordStringString(name, value)); + } + + return result; + } + + public static ArrayList ll_os_listdir(String path) + { + if (path == "") + throwOSError(PyPy.ENOENT, "No such file or directory: ''"); + + File f = new File(path); + if (!f.exists() || !f.isDirectory()) + throwOSError(PyPy.ENOENT, "No such file or directory: '"+path+"'"); + + return new ArrayList(Arrays.asList(f.list())); } public static String ll_os_getcwd() Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Wed Oct 3 06:31:49 2007 @@ -23,14 +23,8 @@ def test_os_dup(self): py.test.skip("not implemented") - def test_environ_items(self): - py.test.skip('fixme!') - def test_environ(self): - py.test.skip('fixme!') - - def test_os_listdir(self): - py.test.skip('fixme!') + py.test.skip('fixme! how to set environment variables in Java?') def test_os_read_binary_crlf(self): py.test.skip('fixme!') @@ -39,6 +33,5 @@ class TestJvmTime(JvmTest, BaseTestTime): - def test_time_sleep(self): - py.test.skip('fixme!') + pass Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Wed Oct 3 06:31:49 2007 @@ -179,6 +179,7 @@ jPyPyStatResult = JvmClassType('pypy.StatResult') jPyPyWeakRef = JvmClassType('pypy.PyPyWeakRef') jll_os = JvmClassType('pypy.ll_os') +jPyPyRecordStringString = JvmClassType('pypy.RecordStringString') jArithmeticException = JvmClassType('java.lang.ArithmeticException', throwable=True) From pypy-svn at codespeak.net Wed Oct 3 14:32:31 2007 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Wed, 3 Oct 2007 14:32:31 +0200 (CEST) Subject: [pypy-svn] Tips of the Week Message-ID: <20071003072957.13351.qmail@ppp91-122-141-35.pppoe.avangard-dsl.ru> An HTML attachment was scrubbed... URL: From mwh at codespeak.net Wed Oct 3 14:45:34 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 3 Oct 2007 14:45:34 +0200 (CEST) Subject: [pypy-svn] r47112 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20071003124534.1688E8155@code0.codespeak.net> Author: mwh Date: Wed Oct 3 14:45:32 2007 New Revision: 47112 Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: the beginnings of fixes for 64-bit support Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Wed Oct 3 14:45:32 2007 @@ -188,6 +188,10 @@ # ULONGLONG r_ulonglong # SIZE_T r_size_t # -------------------------------------------------------------------- +# Note that rffi.r_int is not necessarily the same as +# rarithmetic.r_int, etc! rffi.INT/r_int correspond to the C-level +# 'int' type, whereas rarithmetic.r_int corresponds to the +# Python-level int type (which is a C long). Fun. def CStruct(name, *fields, **kwds): """ A small helper to create external C structure, not the Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Wed Oct 3 14:45:32 2007 @@ -29,9 +29,9 @@ res = ctypes2lltype(lltype.SingleFloat, ctypes.c_float(-3.5)) assert isinstance(res, rffi.r_singlefloat) assert float(res) == -3.5 - assert lltype2ctypes(rffi.r_uint(-1)) == sys.maxint * 2 + 1 + assert lltype2ctypes(rffi.r_ulong(-1)) == sys.maxint * 2 + 1 res = ctypes2lltype(lltype.Unsigned, sys.maxint * 2 + 1) - assert (res, type(res)) == (rffi.r_uint(-1), rffi.r_uint) + assert (res, type(res)) == (rffi.r_ulong(-1), rffi.r_ulong) res = lltype2ctypes(llmemory.sizeof(lltype.Signed)) assert res == struct.calcsize("l") @@ -377,7 +377,7 @@ p1 = rffi.cast(SIGNEDPTR, p1) p2 = rffi.cast(SIGNEDPTR, p2) print 'my_compar:', p1[0], p2[0] - return cmp(p1[0], p2[0]) + return rffi.cast(rffi.INT, cmp(p1[0], p2[0])) qsort(rffi.cast(rffi.VOIDP, a), rffi.cast(rffi.SIZE_T, 10), From mwh at codespeak.net Wed Oct 3 15:02:43 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 3 Oct 2007 15:02:43 +0200 (CEST) Subject: [pypy-svn] r47113 - pypy/dist/pypy/rpython/lltypesystem/test Message-ID: <20071003130243.4525F8155@code0.codespeak.net> Author: mwh Date: Wed Oct 3 15:02:42 2007 New Revision: 47113 Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Log: more precision about int types in the tests Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Wed Oct 3 15:02:42 2007 @@ -131,12 +131,12 @@ def f(): struct = lltype.malloc(TP.TO, flavor='raw') - struct.c_one = 3 + struct.c_one = cast(INT, 3) struct.c_two = '\x33' - struct.c_three = 5 + struct.c_three = cast(INT, 5) result = z(struct) lltype.free(struct, flavor='raw') - return result + return cast(LONG, result) fn = compile(f, [], backendopt=False) assert fn() == 8 @@ -321,16 +321,16 @@ def test_struct_create(): X = CStruct('xx', ('one', INT)) def f(): - p = make(X, c_one=3) + p = make(X, c_one=cast(INT, 3)) res = p.c_one lltype.free(p, flavor='raw') - return res + return cast(LONG, res) assert f() == 3 assert interpret(f, []) == 3 def test_structcopy(): - X2 = lltype.Struct('X2', ('x', INT)) - X1 = lltype.Struct('X1', ('a', INT), ('x2', X2), ('p', lltype.Ptr(X2))) + X2 = lltype.Struct('X2', ('x', LONG)) + X1 = lltype.Struct('X1', ('a', LONG), ('x2', X2), ('p', lltype.Ptr(X2))) def f(): p2 = make(X2, x=123) p1 = make(X1, a=5, p=p2) @@ -374,9 +374,9 @@ def test_stringpolicy1(): - strlen = llexternal('strlen', [CCHARP], INT, includes=['string.h']) + strlen = llexternal('strlen', [CCHARP], SIZE_T, includes=['string.h']) def f(): - return strlen("Xxx") + return cast(LONG, strlen("Xxx")) assert interpret(f, [], backendopt=True) == 3 def test_stringpolicy2(): @@ -398,14 +398,14 @@ assert interpret(f, [], backendopt=True) == 3 def test_stringpolicy_mixed(): - strlen = llexternal('strlen', [CCHARP], INT, + strlen = llexternal('strlen', [CCHARP], SIZE_T, includes=['string.h']) def f(): res1 = strlen("abcd") ll_str = str2charp("Xxx") res2 = strlen(ll_str) lltype.free(ll_str, flavor='raw') - return res1*10 + res2 + return cast(LONG, res1*10 + res2) assert interpret(f, [], backendopt=True) == 43 From mwh at codespeak.net Wed Oct 3 15:18:46 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 3 Oct 2007 15:18:46 +0200 (CEST) Subject: [pypy-svn] r47114 - pypy/dist/pypy/rpython/module Message-ID: <20071003131846.68A578158@code0.codespeak.net> Author: mwh Date: Wed Oct 3 15:18:44 2007 New Revision: 47114 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py Log: more INT precision Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Wed Oct 3 15:18:44 2007 @@ -195,7 +195,7 @@ try: if arg_is_path: arg = rffi.str2charp(arg) - error = os_mystat(arg, stresult) + error = rffi.cast(rffi.LONG, os_mystat(arg, stresult)) if arg_is_path: rffi.free_charp(arg) if error != 0: From mwh at codespeak.net Wed Oct 3 15:42:58 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 3 Oct 2007 15:42:58 +0200 (CEST) Subject: [pypy-svn] r47115 - pypy/dist/pypy/rpython/module Message-ID: <20071003134258.570678160@code0.codespeak.net> Author: mwh Date: Wed Oct 3 15:42:56 2007 New Revision: 47115 Modified: pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/module/ll_time.py Log: more INT->LONG casting. not sure how this approach will scale... Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Wed Oct 3 15:42:56 2007 @@ -71,7 +71,7 @@ def extdef_for_os_function_returning_int(self, name, **kwds): c_func = self.llexternal(name, [], rffi.INT, **kwds) def c_func_llimpl(): - res = c_func() + res = rffi.cast(rffi.LONG, c_func()) if res == -1: raise OSError(rffi.get_errno(), "%s failed" % name) return res @@ -355,7 +355,7 @@ rffi.INT) def os_open_llimpl(path, flags, mode): - result = os_open(path, flags, mode) + result = rffi.cast(rffi.LONG, os_open(path, flags, mode)) if result == -1: raise OSError(rffi.get_errno(), "os_open failed") return result @@ -488,8 +488,9 @@ [rffi.INT, rffi.LONGLONG], rffi.INT) def ftruncate_llimpl(fd, length): - res = os_ftruncate(rffi.cast(rffi.INT, fd), - rffi.cast(rffi.LONGLONG, length)) + res = rffi.cast(rffi.LONG, + os_ftruncate(rffi.cast(rffi.INT, fd), + rffi.cast(rffi.LONGLONG, length))) if res < 0: raise OSError(rffi.get_errno(), "os_lseek failed") @@ -781,7 +782,7 @@ os_isatty = self.llexternal(underscore_on_windows+'isatty', [rffi.INT], rffi.INT) def isatty_llimpl(fd): - res = os_isatty(rffi.cast(rffi.INT, fd)) + res = rffi.cast(rffi.LONG, os_isatty(rffi.cast(rffi.INT, fd))) return res != 0 return extdef([int], bool, llimpl=isatty_llimpl, Modified: pypy/dist/pypy/rpython/module/ll_time.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_time.py (original) +++ pypy/dist/pypy/rpython/module/ll_time.py Wed Oct 3 15:42:56 2007 @@ -82,11 +82,11 @@ t = lltype.malloc(self.TIMEVAL, flavor='raw') if self.GETTIMEOFDAY_NO_TZ: - if c_gettimeofday(t) == 0: + if rffi.cast(rffi.LONG, c_gettimeofday(t)) == 0: result = float(t.c_tv_sec) + \ float(t.c_tv_usec) * 0.000001 else: - if c_gettimeofday(t, void) == 0: + if rffi.cast(rffi.LONG, c_gettimeofday(t, void)) == 0: result = float(t.c_tv_sec) + \ float(t.c_tv_usec) * 0.000001 lltype.free(t, flavor='raw') @@ -162,7 +162,7 @@ frac = math.fmod(secs, 1.0) t.c_tv_sec = int(secs) t.c_tv_usec = int(frac*1000000.0) - if c_select(0, void, void, void, t) != 0: + if rffi.cast(rffi.LONG, c_select(0, void, void, void, t)) != 0: errno = rffi.get_errno() if errno != EINTR: raise OSError(rffi.get_errno(), "Select failed") From cfbolz at codespeak.net Wed Oct 3 19:34:29 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 3 Oct 2007 19:34:29 +0200 (CEST) Subject: [pypy-svn] r47116 - pypy/dist/pypy/doc Message-ID: <20071003173429.BC8458172@code0.codespeak.net> Author: cfbolz Date: Wed Oct 3 19:34:28 2007 New Revision: 47116 Added: pypy/dist/pypy/doc/sandbox.txt Modified: pypy/dist/pypy/doc/index.txt Log: condense the various mails on pypy-dev into some documentation about the sandbox features. Modified: pypy/dist/pypy/doc/index.txt ============================================================================== --- pypy/dist/pypy/doc/index.txt (original) +++ pypy/dist/pypy/doc/index.txt Wed Oct 3 19:34:28 2007 @@ -24,6 +24,7 @@ * `What PyPy can do for your objects`_ * `Stackless and coroutines`_ * `JIT Generation in PyPy`_ + * `Sandboxing Python code`_ `extension compiler`_ describes the (in-progress) tool that can be used to write modules in PyPy's style and compile them into regular CPython @@ -177,6 +178,7 @@ .. _`Nightly builds and benchmarks`: http://tuatara.cs.uni-duesseldorf.de/benchmark.html .. _`directory reference`: .. _`rlib`: rlib.html +.. _`Sandboxing Python code`: sandbox.html PyPy directory cross-reference ------------------------------ Added: pypy/dist/pypy/doc/sandbox.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/sandbox.txt Wed Oct 3 19:34:28 2007 @@ -0,0 +1,88 @@ +PyPy's sandboxing features +========================== + +Overview +-------- + +One of PyPy's translation aspect is a sandboxing feature. It's "sandboxing" as +in "full virtualization", but done in normal C with no OS support at all. It's +a two-processes model: we can translate PyPy to a special "pypy-c-sandbox" +executable, which is safe in the sense that it doesn't do any library or system +call - instead, whenever it would like to perform such an operation, it +marshals the operation name and the arguments to its stdout and it waits for +the marshalled result on its stdin. This pypy-c-sandbox process is meant to be +run by an outer "controller" program that answers to these operation requests. + +The pypy-c-sandbox program is obtained by adding a transformation during +translation, which turns all RPython-level external function calls into +stubs that do the marshalling/waiting/unmarshalling. An attacker that +tries to escape the sandbox is stuck within a C program that contains no +external function call at all except to write to stdout and read from +stdin. (It's still attackable, e.g. by exploiting segfault-like +situations, but as far as I can tell - unlike CPython - any RPython +program is really robust against this kind of attack, at least if we +enable the extra checks that all RPython list and string indexing are in +range. Alternatively, on Linux there is a lightweight OS-level +sandboxing technique available by default - google for 'seccomp'.) + +The outer controller is a plain Python program that can run in CPython +or a regular PyPy. It can perform any virtualization it likes, by +giving the subprocess any custom view on its world. For example, while +the subprocess thinks it's using file handles, in reality the numbers +are created by the controller process and so they need not be (and +probably should not be) real OS-level file handles at all. In the demo +controller I've implemented there is simply a mapping from numbers to +file-like objects. The controller answers to the "os_open" operation by +translating the requested path to some file or file-like object in some +virtual and completely custom directory hierarchy. The file-like object +is put in the mapping with any unused number >= 3 as a key, and the +latter is returned to the subprocess. The "os_read" operation works by +mapping the pseudo file handle given by the subprocess back to a +file-like object in the controller, and reading from the file-like +object. + +Translating an RPython program with sandboxing enables also uses a special flag +that enables all sorts of C-level assertions against index-out-of-bounds +accesses. + +By the way, as you should have realized, it's really independent from +the fact that it's PyPy that we are translating. Any RPython program +should do. I've successfully tried it on the JS interpreter. The +controller is only called "pypy_interact" because it emulates a file +hierarchy that makes pypy-c-sandbox happy - it contains (read-only) +virtual directories like /bin/lib-python and /bin/pypy/lib and it +pretends that the executable is /bin/pypy-c. + +Howto +----- + + +In pypy/translator/goal:: + + ./translate.py --sandbox --source targetpypystandalone.py --withoutmod-gc + --withoutmod-_weakref + + (the gc and _weakref modules are disabled because they are a bit + too unsafe, in the sense that they could allow bogus memory + accesses) + +To run it, use the tools in the pypy/translator/sandbox directory:: + + ./pypy_interact.py /some/path/pypy-c-sandbox [args...] + +Just like pypy-c, if you pass no argument you get the interactive +prompt. In theory it's impossible to do anything bad or read a random +file on the machine from this prompt. (There is no protection against +using all the RAM or CPU yet.) To pass a script as an argument you need +to put it in a directory along with all its dependencies, and ask +pypy_interact to export this directory (read-only) to the subprocess' +virtual /tmp directory with the --tmp=DIR option. + +Not all operations are supported; e.g. if you type os.readlink('...'), +the controller crashes with an exception and the subprocess is killed. +Other operations make the subprocess die directly with a "Fatal RPython +error". None of this is a security hole; it just means that if you try +to run some random program, it risks getting killed depending on the +Python built-in functions it tries to call. + + From fijal at codespeak.net Wed Oct 3 19:38:58 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 3 Oct 2007 19:38:58 +0200 (CEST) Subject: [pypy-svn] r47117 - pypy/dist/pypy/doc Message-ID: <20071003173858.058418172@code0.codespeak.net> Author: fijal Date: Wed Oct 3 19:38:58 2007 New Revision: 47117 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: rctime is more or less done, not counting windows testing Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Wed Oct 3 19:38:58 2007 @@ -39,7 +39,6 @@ - _ssl - fcntl - mmap - - rctime - readline - signal - rsdl From cfbolz at codespeak.net Wed Oct 3 19:43:51 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 3 Oct 2007 19:43:51 +0200 (CEST) Subject: [pypy-svn] r47118 - pypy/dist/pypy/doc Message-ID: <20071003174351.5E2CF8172@code0.codespeak.net> Author: cfbolz Date: Wed Oct 3 19:43:51 2007 New Revision: 47118 Modified: pypy/dist/pypy/doc/sandbox.txt Log: some fixes (thanks armin) Modified: pypy/dist/pypy/doc/sandbox.txt ============================================================================== --- pypy/dist/pypy/doc/sandbox.txt (original) +++ pypy/dist/pypy/doc/sandbox.txt Wed Oct 3 19:43:51 2007 @@ -59,12 +59,8 @@ In pypy/translator/goal:: - ./translate.py --sandbox --source targetpypystandalone.py --withoutmod-gc - --withoutmod-_weakref + ./translate.py --sandbox targetpypystandalone.py - (the gc and _weakref modules are disabled because they are a bit - too unsafe, in the sense that they could allow bogus memory - accesses) To run it, use the tools in the pypy/translator/sandbox directory:: @@ -72,8 +68,7 @@ Just like pypy-c, if you pass no argument you get the interactive prompt. In theory it's impossible to do anything bad or read a random -file on the machine from this prompt. (There is no protection against -using all the RAM or CPU yet.) To pass a script as an argument you need +file on the machine from this prompt. To pass a script as an argument you need to put it in a directory along with all its dependencies, and ask pypy_interact to export this directory (read-only) to the subprocess' virtual /tmp directory with the --tmp=DIR option. From cfbolz at codespeak.net Wed Oct 3 19:46:32 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 3 Oct 2007 19:46:32 +0200 (CEST) Subject: [pypy-svn] r47119 - pypy/dist/pypy/doc Message-ID: <20071003174632.E367C817F@code0.codespeak.net> Author: cfbolz Date: Wed Oct 3 19:46:32 2007 New Revision: 47119 Modified: pypy/dist/pypy/doc/sandbox.txt Log: how to limit CPU time and RAM usage Modified: pypy/dist/pypy/doc/sandbox.txt ============================================================================== --- pypy/dist/pypy/doc/sandbox.txt (original) +++ pypy/dist/pypy/doc/sandbox.txt Wed Oct 3 19:46:32 2007 @@ -73,6 +73,10 @@ pypy_interact to export this directory (read-only) to the subprocess' virtual /tmp directory with the --tmp=DIR option. +To limit the used heapsize, use the ``--heapsize=N`` option to +pypy_interact.py. You can also give a limit to the CPU time (real time) by +using the ``--timeout=N`` option. + Not all operations are supported; e.g. if you type os.readlink('...'), the controller crashes with an exception and the subprocess is killed. Other operations make the subprocess die directly with a "Fatal RPython From arigo at codespeak.net Wed Oct 3 19:58:46 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 3 Oct 2007 19:58:46 +0200 (CEST) Subject: [pypy-svn] r47120 - pypy/dist/pypy/doc Message-ID: <20071003175846.E314D818B@code0.codespeak.net> Author: arigo Date: Wed Oct 3 19:58:46 2007 New Revision: 47120 Modified: pypy/dist/pypy/doc/sandbox.txt Log: Extra introduction, taken from the upcoming pypy-dev e-mail. Modified: pypy/dist/pypy/doc/sandbox.txt ============================================================================== --- pypy/dist/pypy/doc/sandbox.txt (original) +++ pypy/dist/pypy/doc/sandbox.txt Wed Oct 3 19:58:46 2007 @@ -1,6 +1,31 @@ PyPy's sandboxing features ========================== +Introduction +------------ + +It is possible to compile a version of pypy-c that runs +fully "virtualized", i.e. where an external process controls all +input/output. Such a pypy-c is a secure sandbox: it is safe to run +any untrusted Python code with it. The Python code cannot see or +modify any local file except via interaction with the external +process. It is also impossible to do any other I/O or consume more +than some amount of RAM or CPU time or real time. This works with no +OS support at all - just ANSI C code generated in a careful way. It's +the kind of thing you could embed in a browser plug-in, for example +(it would be safe even if it wasn't run as a separate process, +actually). + +For comparison, trying to plug CPython into a special virtualizing C +library is not only OS-specific, but unsafe, because one of the known +ways to segfault CPython could be used by an attacker to trick CPython +into issuing malicious system calls directly. The C code generated by +PyPy is not segfaultable, as long as our code generators are correct - +that's a lower number of lines of code to trust. For the paranoid, in +this case we also generate systematic run-time checks against buffer +overflows. + + Overview -------- From arigo at codespeak.net Wed Oct 3 20:04:56 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 3 Oct 2007 20:04:56 +0200 (CEST) Subject: [pypy-svn] r47121 - pypy/dist/pypy/doc Message-ID: <20071003180456.0B5428173@code0.codespeak.net> Author: arigo Date: Wed Oct 3 20:04:55 2007 New Revision: 47121 Modified: pypy/dist/pypy/doc/sandbox.txt Log: Reduce this long ()ized sentence - it's better explained in the introduction. Modified: pypy/dist/pypy/doc/sandbox.txt ============================================================================== --- pypy/dist/pypy/doc/sandbox.txt (original) +++ pypy/dist/pypy/doc/sandbox.txt Wed Oct 3 20:04:55 2007 @@ -43,12 +43,9 @@ stubs that do the marshalling/waiting/unmarshalling. An attacker that tries to escape the sandbox is stuck within a C program that contains no external function call at all except to write to stdout and read from -stdin. (It's still attackable, e.g. by exploiting segfault-like -situations, but as far as I can tell - unlike CPython - any RPython -program is really robust against this kind of attack, at least if we -enable the extra checks that all RPython list and string indexing are in -range. Alternatively, on Linux there is a lightweight OS-level -sandboxing technique available by default - google for 'seccomp'.) +stdin. (It's still attackable in theory, e.g. by exploiting segfault-like +situations, but as explained in the introduction we think that PyPy is +rather safe against such attacks.) The outer controller is a plain Python program that can run in CPython or a regular PyPy. It can perform any virtualization it likes, by From lac at codespeak.net Wed Oct 3 20:07:36 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Wed, 3 Oct 2007 20:07:36 +0200 (CEST) Subject: [pypy-svn] r47122 - pypy/dist/pypy/doc Message-ID: <20071003180736.5C5FC8173@code0.codespeak.net> Author: lac Date: Wed Oct 3 20:07:35 2007 New Revision: 47122 Modified: pypy/dist/pypy/doc/sandbox.txt Log: minor grammar tweaks and typos fixed Modified: pypy/dist/pypy/doc/sandbox.txt ============================================================================== --- pypy/dist/pypy/doc/sandbox.txt (original) +++ pypy/dist/pypy/doc/sandbox.txt Wed Oct 3 20:07:35 2007 @@ -29,20 +29,20 @@ Overview -------- -One of PyPy's translation aspect is a sandboxing feature. It's "sandboxing" as +One of PyPy's translation aspects is a sandboxing feature. It's "sandboxing" as in "full virtualization", but done in normal C with no OS support at all. It's a two-processes model: we can translate PyPy to a special "pypy-c-sandbox" -executable, which is safe in the sense that it doesn't do any library or system -call - instead, whenever it would like to perform such an operation, it +executable, which is safe in the sense that it doesn't do any library or +system calls - instead, whenever it would like to perform such an operation, it marshals the operation name and the arguments to its stdout and it waits for the marshalled result on its stdin. This pypy-c-sandbox process is meant to be -run by an outer "controller" program that answers to these operation requests. +run by an outer "controller" program that answers these operation requests. The pypy-c-sandbox program is obtained by adding a transformation during translation, which turns all RPython-level external function calls into stubs that do the marshalling/waiting/unmarshalling. An attacker that tries to escape the sandbox is stuck within a C program that contains no -external function call at all except to write to stdout and read from +external function calls at all except for writing to stdout and reading from stdin. (It's still attackable in theory, e.g. by exploiting segfault-like situations, but as explained in the introduction we think that PyPy is rather safe against such attacks.) @@ -88,7 +88,7 @@ ./pypy_interact.py /some/path/pypy-c-sandbox [args...] -Just like pypy-c, if you pass no argument you get the interactive +Just like with pypy-c, if you pass no argument you get the interactive prompt. In theory it's impossible to do anything bad or read a random file on the machine from this prompt. To pass a script as an argument you need to put it in a directory along with all its dependencies, and ask From arigo at codespeak.net Wed Oct 3 20:37:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 3 Oct 2007 20:37:35 +0200 (CEST) Subject: [pypy-svn] r47123 - pypy/dist/pypy/doc Message-ID: <20071003183735.317FD814A@code0.codespeak.net> Author: arigo Date: Wed Oct 3 20:37:34 2007 New Revision: 47123 Modified: pypy/dist/pypy/doc/index.txt Log: A quick status paragraph. Modified: pypy/dist/pypy/doc/index.txt ============================================================================== --- pypy/dist/pypy/doc/index.txt (original) +++ pypy/dist/pypy/doc/index.txt Wed Oct 3 20:37:34 2007 @@ -73,6 +73,12 @@ Status =================================== +PyPy can be used to run Python programs on Linux, OS/X, Windows, and +on top of .NET. It is recommended to try out the current Subversion +HEAD, which contains `major improvements`__ since the last release. + +.. __: http://codespeak.net/pipermail/pypy-dev/2007q4/004103.html + PyPy is mainly developed on Linux and Mac OS X. Windows is supported, but platform-specific bugs tend to take longer before we notice and fix them. About 64-bit machines: although support is mostly present, we From rxe at codespeak.net Wed Oct 3 21:32:06 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 3 Oct 2007 21:32:06 +0200 (CEST) Subject: [pypy-svn] r47124 - pypy/dist/pypy/rpython Message-ID: <20071003193206.CF34C818A@code0.codespeak.net> Author: rxe Date: Wed Oct 3 21:32:05 2007 New Revision: 47124 Modified: pypy/dist/pypy/rpython/annlowlevel.py Log: try to get tests running in llvm backend again. the problem seems to be that None crept in to graph.name during typing. not sure i understand as to why, and looks likely it is an oversight. anyways feel free to revert. Modified: pypy/dist/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/dist/pypy/rpython/annlowlevel.py (original) +++ pypy/dist/pypy/rpython/annlowlevel.py Wed Oct 3 21:32:05 2007 @@ -41,7 +41,10 @@ def lowlevelspecialize(funcdesc, args_s, key_for_args): args_s, key, ignored, builder = flatten_star_args(funcdesc, args_s) - key = [key] + if key is not None: + key = [key] + else: + key = [] new_args_s = [] for i, s_obj in enumerate(args_s): if i in key_for_args: From arigo at codespeak.net Wed Oct 3 21:36:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 3 Oct 2007 21:36:09 +0200 (CEST) Subject: [pypy-svn] r47125 - pypy/dist/pypy/doc Message-ID: <20071003193609.ED78F818A@code0.codespeak.net> Author: arigo Date: Wed Oct 3 21:36:09 2007 New Revision: 47125 Modified: pypy/dist/pypy/doc/index.txt Log: Mention Java. Modified: pypy/dist/pypy/doc/index.txt ============================================================================== --- pypy/dist/pypy/doc/index.txt (original) +++ pypy/dist/pypy/doc/index.txt Wed Oct 3 21:36:09 2007 @@ -73,9 +73,10 @@ Status =================================== -PyPy can be used to run Python programs on Linux, OS/X, Windows, and -on top of .NET. It is recommended to try out the current Subversion -HEAD, which contains `major improvements`__ since the last release. +PyPy can be used to run Python programs on Linux, OS/X, +Windows, on top of .NET, and (recently) on top of Java. +It is recommended to try out the current Subversion HEAD, +which contains `major improvements`__ since the last release. .. __: http://codespeak.net/pipermail/pypy-dev/2007q4/004103.html From arigo at codespeak.net Wed Oct 3 21:45:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 3 Oct 2007 21:45:51 +0200 (CEST) Subject: [pypy-svn] r47126 - pypy/dist/pypy/rpython Message-ID: <20071003194551.D5A6D8188@code0.codespeak.net> Author: arigo Date: Wed Oct 3 21:45:51 2007 New Revision: 47126 Modified: pypy/dist/pypy/rpython/annlowlevel.py Log: The key has to be computed in this way - its purpose is to get the correct specialization. I admit that I don't see how a None in this list can lead to a None in graph.name, but the change I'm reverting was dangerous. Modified: pypy/dist/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/dist/pypy/rpython/annlowlevel.py (original) +++ pypy/dist/pypy/rpython/annlowlevel.py Wed Oct 3 21:45:51 2007 @@ -41,10 +41,7 @@ def lowlevelspecialize(funcdesc, args_s, key_for_args): args_s, key, ignored, builder = flatten_star_args(funcdesc, args_s) - if key is not None: - key = [key] - else: - key = [] + key = [key] new_args_s = [] for i, s_obj in enumerate(args_s): if i in key_for_args: From arigo at codespeak.net Wed Oct 3 21:57:19 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 3 Oct 2007 21:57:19 +0200 (CEST) Subject: [pypy-svn] r47127 - pypy/dist/pypy/rpython Message-ID: <20071003195719.257BF8188@code0.codespeak.net> Author: arigo Date: Wed Oct 3 21:57:18 2007 New Revision: 47127 Modified: pypy/dist/pypy/rpython/annlowlevel.py Log: Use a tuple inside a tuple as the specialization key. This allows us to avoid putting an explicit None in the key, which shows up as a substring of graph.name and confuses llvm tests. Modified: pypy/dist/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/dist/pypy/rpython/annlowlevel.py (original) +++ pypy/dist/pypy/rpython/annlowlevel.py Wed Oct 3 21:57:18 2007 @@ -40,8 +40,8 @@ pol.rtyper = rtyper def lowlevelspecialize(funcdesc, args_s, key_for_args): - args_s, key, ignored, builder = flatten_star_args(funcdesc, args_s) - key = [key] + args_s, key1, ignored, builder = flatten_star_args(funcdesc, args_s) + key = [] new_args_s = [] for i, s_obj in enumerate(args_s): if i in key_for_args: @@ -59,7 +59,10 @@ # passing non-low-level types to a ll_* function is allowed # for module/ll_* key.append(s_obj.__class__) - flowgraph = funcdesc.cachedgraph(tuple(key), builder=builder) + key = (tuple(key),) + if key1 is not None: + key += (key1,) + flowgraph = funcdesc.cachedgraph(key, builder=builder) args_s[:] = new_args_s return flowgraph lowlevelspecialize = staticmethod(lowlevelspecialize) From rxe at codespeak.net Wed Oct 3 22:27:57 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 3 Oct 2007 22:27:57 +0200 (CEST) Subject: [pypy-svn] r47132 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071003202757.52C1D816C@code0.codespeak.net> Author: rxe Date: Wed Oct 3 22:27:56 2007 New Revision: 47132 Modified: pypy/dist/pypy/translator/llvm/test/test_typed.py Log: fix test Modified: pypy/dist/pypy/translator/llvm/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_typed.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_typed.py Wed Oct 3 22:27:56 2007 @@ -218,7 +218,7 @@ def func(x, y): z = x + y / 2.1 * x z = math.fmod(z, 60.0) - z = pow(z, 2) + z = math.pow(z, 2) z = -z return int(z) From rxe at codespeak.net Wed Oct 3 22:28:31 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 3 Oct 2007 22:28:31 +0200 (CEST) Subject: [pypy-svn] r47133 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071003202831.DC989816C@code0.codespeak.net> Author: rxe Date: Wed Oct 3 22:28:31 2007 New Revision: 47133 Modified: pypy/dist/pypy/translator/llvm/test/runtest.py Log: yay Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Wed Oct 3 22:28:31 2007 @@ -1,5 +1,4 @@ import py -py.test.skip("llvm is not actively maintained") from pypy.translator.llvm.buildllvm import llvm_is_on_path, llvm_version, gcc_version from pypy.translator.llvm.genllvm import GenLLVM From rxe at codespeak.net Wed Oct 3 22:55:47 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 3 Oct 2007 22:55:47 +0200 (CEST) Subject: [pypy-svn] r47136 - in pypy/dist/pypy: rpython/rctypes/test translator/llvm/test Message-ID: <20071003205547.A8D08818A@code0.codespeak.net> Author: rxe Date: Wed Oct 3 22:55:47 2007 New Revision: 47136 Modified: pypy/dist/pypy/rpython/rctypes/test/test_rarray.py pypy/dist/pypy/rpython/rctypes/test/test_rprimitive.py pypy/dist/pypy/translator/llvm/test/test_lltype.py Log: fwiw, kill any llvm tests that reference rctypes (they never worked and they never will) Modified: pypy/dist/pypy/rpython/rctypes/test/test_rarray.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_rarray.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_rarray.py Wed Oct 3 22:55:47 2007 @@ -16,9 +16,6 @@ c_int_10 = ARRAY(c_int,10) -test_c_compile = True -test_llvm_compile = False - def maketest(): A1 = c_int * 10 A2 = POINTER(c_int) * 10 @@ -346,9 +343,6 @@ class Test_compilation: def setup_class(self): - if not test_c_compile: - py.test.skip("c compilation disabled") - from pypy.translator.c.test.test_genc import compile self.compile = lambda s, x, y : compile(x, y) @@ -408,10 +402,3 @@ fn = self.compile(func, [int]) res = fn(N) assert res == 12345678 - -class Test_compilation_llvm(Test_compilation): - def setup_class(self): - if not test_llvm_compile: - py.test.skip("llvm tests disabled") - from pypy.translator.llvm.test.runtest import compile_function - self.compile = lambda s, x, y : compile_function(x, y) Modified: pypy/dist/pypy/rpython/rctypes/test/test_rprimitive.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_rprimitive.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_rprimitive.py Wed Oct 3 22:55:47 2007 @@ -17,9 +17,6 @@ from ctypes import c_double, c_wchar, c_char_p, pointer, sizeof from ctypes import Structure -test_c_compile = True -test_llvm_compile = False - class Test_annotation: def test_simple(self): res = c_int(42) @@ -469,9 +466,6 @@ class Test_compilation: def setup_class(self): - if not test_c_compile: - py.test.skip("c compilation disabled") - from pypy.translator.c.test.test_genc import compile self.compile = lambda s, x, y : compile(x, y) @@ -615,9 +609,3 @@ fn = self.compile(func, [int]) assert fn(19) == func(19) -class Test_compilation_llvm(Test_compilation): - def setup_class(self): - if not test_llvm_compile: - py.test.skip("llvm compilation disabled") - from pypy.translator.llvm.test.runtest import compile_function - self.compile = lambda s, x, y : compile_function(x, y) Modified: pypy/dist/pypy/translator/llvm/test/test_lltype.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lltype.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lltype.py Wed Oct 3 22:55:47 2007 @@ -442,42 +442,3 @@ fn = compile_function(llf, []) res = fn() assert res == 42 - -def test_rctypes_array_access(): - py.test.skip("this breaks during annotation - ?") - from ctypes import ARRAY, c_int - c_int_1 = ARRAY(c_int, 1) - - my_array = c_int_1() - - def llf(): - my_array[0] = 42 - return my_array[0] - - fn = compile_function(llf, []) - res = fn() - assert res == 42 - -def test_rctypes_char_array_value(): - py.test.skip("this breaks during annotation - ?") - from ctypes import c_char - A = c_char * 3 - - def llf(n): - a = A() - a[n+0] = 'x' - a[n+1] = 'y' - return a.value == 'xy' - - fn = compile_function(llf, [int]) - res = fn(0) - assert res - -def test_rctypes_variants(): - py.test.skip("wip") - from pypy.rpython.rctypes.test import test_rarray - llf, expected = test_rarray.maketest() - - fn = compile_function(llf, []) - res = fn() - assert res == expected From cfbolz at codespeak.net Thu Oct 4 10:16:02 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 4 Oct 2007 10:16:02 +0200 (CEST) Subject: [pypy-svn] r47142 - pypy/branch/kill-keepalives-again/pypy/translator/backendopt/test Message-ID: <20071004081602.A11938173@code0.codespeak.net> Author: cfbolz Date: Thu Oct 4 10:16:00 2007 New Revision: 47142 Modified: pypy/branch/kill-keepalives-again/pypy/translator/backendopt/test/test_constfold.py Log: skip the only non-working constfold test Modified: pypy/branch/kill-keepalives-again/pypy/translator/backendopt/test/test_constfold.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/backendopt/test/test_constfold.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/backendopt/test/test_constfold.py Thu Oct 4 10:16:00 2007 @@ -1,3 +1,4 @@ +import py from pypy.objspace.flow.model import checkgraph, Constant, summary from pypy.translator.translator import TranslationContext, graphof from pypy.rpython.llinterp import LLInterpreter @@ -173,6 +174,7 @@ def test_keepalive_const_substruct(): + py.test.skip("do we want partial folding of getinteriorfield?") S2 = lltype.Struct('S2', ('x', lltype.Signed)) S1 = lltype.GcStruct('S1', ('sub', S2)) s1 = lltype.malloc(S1) @@ -180,7 +182,7 @@ def fn(): return s1.sub.x graph, t = get_graph(fn, []) - assert summary(graph) == {'getsubstruct': 1, 'getfield': 1} + assert summary(graph) == {'getinteriorfield': 1} constant_fold_graph(graph) # kill all references to 's1' From cfbolz at codespeak.net Thu Oct 4 12:59:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 4 Oct 2007 12:59:50 +0200 (CEST) Subject: [pypy-svn] r47146 - in pypy/branch/kill-keepalives-again/pypy: rpython/lltypesystem translator/c/src Message-ID: <20071004105950.AFF228181@code0.codespeak.net> Author: cfbolz Date: Thu Oct 4 12:59:49 2007 New Revision: 47146 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/rstr.py pypy/branch/kill-keepalives-again/pypy/translator/c/src/support.h Log: make rstr not look into internal structures Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/rstr.py Thu Oct 4 12:59:49 2007 @@ -87,10 +87,7 @@ cflags = inputconst(Void, {'flavor': 'gc'}) v_result = llops.genop('malloc_varsize', [cstr, cflags, v_len], resulttype=Ptr(STR)) - cchars = inputconst(Void, "chars") - v_chars = llops.genop('getsubstruct', [v_result, cchars], - resulttype=Ptr(STR.chars)) - llops.gencapicall('PyString_ToLLCharArray', [v, v_chars]) + llops.gencapicall('PyString_ToRPyString', [v, v_result]) string_repr = llops.rtyper.type_system.rstr.string_repr v_result = llops.convertvar(v_result, string_repr, r_to) return v_result @@ -101,15 +98,11 @@ string_repr = llops.rtyper.type_system.rstr.string_repr v = llops.convertvar(v, r_from, string_repr) cchars = inputconst(Void, "chars") - v_chars = llops.genop('getsubstruct', [v, cchars], - resulttype=Ptr(STR.chars)) - v_size = llops.genop('getarraysize', [v_chars], - resulttype=Signed) # xxx put in table - return llops.gencapicall('PyString_FromLLCharArrayAndSize', - [v_chars, v_size], + return llops.gencapicall('PyString_FromRPyString', + [v], resulttype=pyobj_repr, - _callable= lambda chars, sz: pyobjectptr(''.join(chars))) + _callable= lambda v: pyobjectptr(''.join(v.chars))) def mallocstr(length): debug_assert(length >= 0, "negative string length") Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/src/support.h ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/src/support.h (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/src/support.h Thu Oct 4 12:59:49 2007 @@ -18,12 +18,12 @@ #define FAIL_ZER(msg) FAIL_EXCEPTION(PyExc_ZeroDivisionError, msg) #define CFAIL() RPyConvertExceptionFromCPython() -#define PyString_FromLLCharArrayAndSize(itemsarray, size) \ - PyString_FromStringAndSize(itemsarray->items, size) +#define PyString_FromRPyString(rpystr) \ + PyString_FromStringAndSize(RPyString_AsString(rpystr), RPyString_Size(rpystr)) -#define PyString_ToLLCharArray(s, itemsarray) \ - memcpy(itemsarray->items, PyString_AS_STRING(s), \ - itemsarray->length) +#define PyString_ToRPyString(s, rpystr) \ + memcpy(RPyString_AsString(rpystr), PyString_AS_STRING(s), \ + RPyString_Size(rpystr)) /* Extra checks can be enabled with the RPY_ASSERT or RPY_LL_ASSERT * macros. They differ in the level at which the tests are made. From cfbolz at codespeak.net Thu Oct 4 13:11:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 4 Oct 2007 13:11:28 +0200 (CEST) Subject: [pypy-svn] r47147 - pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem Message-ID: <20071004111128.4A1DD8190@code0.codespeak.net> Author: cfbolz Date: Thu Oct 4 13:11:26 2007 New Revision: 47147 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lloperation.py Log: add bare_setinteriorfield here too Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lloperation.py Thu Oct 4 13:11:26 2007 @@ -323,6 +323,7 @@ 'getinteriorfield': LLOp(sideeffects=False, canrun=True), 'getinteriorarraysize': LLOp(canfold=True), 'setinteriorfield': LLOp(), + 'bare_setinteriorfield': LLOp(), 'getarraysubstruct': LLOp(canfold=True), 'setfield': LLOp(), 'bare_setfield': LLOp(), From fijal at codespeak.net Thu Oct 4 14:35:36 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 4 Oct 2007 14:35:36 +0200 (CEST) Subject: [pypy-svn] r47148 - pypy/branch/bz2-module-rffi Message-ID: <20071004123536.A08F68192@code0.codespeak.net> Author: fijal Date: Thu Oct 4 14:35:35 2007 New Revision: 47148 Added: pypy/branch/bz2-module-rffi/ - copied from r47147, pypy/dist/pypy/module/bz2/ Log: Create a new branch for rffi-based bz2 From fijal at codespeak.net Thu Oct 4 14:37:21 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 4 Oct 2007 14:37:21 +0200 (CEST) Subject: [pypy-svn] r47149 - pypy/branch/bz2-module-rffi Message-ID: <20071004123721.426198192@code0.codespeak.net> Author: fijal Date: Thu Oct 4 14:37:20 2007 New Revision: 47149 Modified: pypy/branch/bz2-module-rffi/interp_bz2.py Log: Intermediate commit around moving bz2 module out of rctypes Modified: pypy/branch/bz2-module-rffi/interp_bz2.py ============================================================================== --- pypy/branch/bz2-module-rffi/interp_bz2.py (original) +++ pypy/branch/bz2-module-rffi/interp_bz2.py Thu Oct 4 14:37:20 2007 @@ -1,34 +1,39 @@ -from pypy.rpython.rctypes.tool import ctypes_platform -import pypy.rpython.rctypes.implementation # this defines rctypes magic +from pypy.rpython.tool import rffi_platform as platform +from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem import lltype from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.typedef import interp_attrproperty from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, interp2app from pypy.rlib.streamio import Stream -from ctypes import * -import ctypes.util import sys -from bzlib import bz_stream, BZFILE, FILE - -#libbz2 = cdll.LoadLibrary(ctypes.util.find_library("bz2")) - -c_void = None - class CConfig: - _header_ = """ - #include - #include - #include - """ - - _includes_ = ["bzlib.h"] - libbz2 = ctypes_platform.Library('bz2') - off_t = ctypes_platform.SimpleType("off_t", c_longlong) - size_t = ctypes_platform.SimpleType("size_t", c_ulong) - BUFSIZ = ctypes_platform.ConstantInteger("BUFSIZ") - SEEK_SET = ctypes_platform.ConstantInteger("SEEK_SET") + _includes_ = ['stdio.h', 'sys/types.h', 'bzlib.h'] + _libraries_ = ['bz2'] + calling_conv = 'c' + + off_t = platform.SimpleType("off_t", rffi.LONGLONG) + size_t = platform.SimpleType("size_t", rffi.ULONG) + BUFSIZ = platform.ConstantInteger("BUFSIZ") + _alloc_type = lltype.FuncType([rffi.VOIDP, rffi.INT, rffi.INT], rffi.VOIDP) + _free_type = lltype.FuncType([rffi.VOIDP, rffi.VOIDP], lltype.Void) + SEEK_SET = platform.ConstantInteger("SEEK_SET") + bz_stream = platform.Struct('bz_stream', + [('next_in', rffi.CCHARP), + ('avail_in', rffi.UINT), + ('total_in_lo32', rffi.UINT), + ('total_in_hi32', rffi.UINT), + ('next_out', rffi.CCHARP), + ('avail_out', rffi.UINT), + ('total_out_lo32', rffi.UINT), + ('total_out_hi32', rffi.UINT), + ('state', rffi.VOIDP), + ('bzalloc', lltype.Ptr(_alloc_type)), + ('bzfree', lltype.Ptr(_free_type)), + ('opaque', rffi.VOIDP), + ]) constants = {} constant_names = ['BZ_RUN', 'BZ_FLUSH', 'BZ_FINISH', 'BZ_OK', @@ -37,11 +42,11 @@ 'BZ_DATA_ERROR_MAGIC', 'BZ_IO_ERROR', 'BZ_UNEXPECTED_EOF', 'BZ_OUTBUFF_FULL', 'BZ_CONFIG_ERROR'] for name in constant_names: - setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name)) + setattr(CConfig, name, platform.DefinedConstantInteger(name)) class cConfig: pass -cConfig.__dict__.update(ctypes_platform.configure(CConfig)) +cConfig.__dict__.update(platform.configure(CConfig)) for name in constant_names: value = getattr(cConfig, name) @@ -49,8 +54,8 @@ constants[name] = value locals().update(constants) -libbz2 = cConfig.libbz2 off_t = cConfig.off_t +bz_stream = cConfig.bz_stream BUFSIZ = cConfig.BUFSIZ SEEK_SET = cConfig.SEEK_SET BZ_OK = cConfig.BZ_OK @@ -69,15 +74,15 @@ else: SMALLCHUNK = BUFSIZ -if sizeof(c_int) > 4: +if rffi.sizeof(rffi.INT) > 4: BIGCHUNK = 512 * 32 else: BIGCHUNK = 512 * 1024 - + MAXINT = sys.maxint if BZ_CONFIG_ERROR: - if sizeof(c_long) >= 8 or sizeof(c_longlong) >= 8: + if rffi.sizeof(rffi.LONG) >= 8 or rffi.sizeof(rffi.LONGLONG) >= 8: def _bzs_total_out(bzs): return (bzs.total_out_hi32 << 32) + bzs.total_out_lo32 else: @@ -87,38 +92,35 @@ def _bzs_total_out(bzs): return bzs.total_out -# the least but one parameter should be c_void_p but it's not used +def external(name, args, result): + return rffi.llexternal(name, args, result, includes=CConfig._includes_, + libraries=['bz2']) + +FILE = rffi.COpaquePtr('FILE') +BZFILE = rffi.COpaquePtr('BZFILE') + +# the least but one parameter should be rffi.VOIDP but it's not used # so I trick the compiler to not complain about constanst pointer passed # to void* arg -libbz2.BZ2_bzReadOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, - c_int, POINTER(c_int), c_int] -libbz2.BZ2_bzReadOpen.restype = POINTER(BZFILE) -libbz2.BZ2_bzWriteOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, - c_int, c_int] -libbz2.BZ2_bzWriteOpen.restype = POINTER(BZFILE) -libbz2.BZ2_bzReadClose.argtypes = [POINTER(c_int), POINTER(BZFILE)] -libbz2.BZ2_bzReadClose.restype = c_void -libbz2.BZ2_bzWriteClose.argtypes = [POINTER(c_int), POINTER(BZFILE), - c_int, POINTER(c_uint), POINTER(c_uint)] -libbz2.BZ2_bzWriteClose.restype = c_void -libbz2.BZ2_bzRead.argtypes = [POINTER(c_int), POINTER(BZFILE), POINTER(c_char), c_int] -libbz2.BZ2_bzRead.restype = c_int -libbz2.BZ2_bzWrite.argtypes = [POINTER(c_int), POINTER(BZFILE), c_char_p, c_int] -libbz2.BZ2_bzWrite.restype = c_void - -libbz2.BZ2_bzCompressInit.argtypes = [POINTER(bz_stream), c_int, c_int, c_int] -libbz2.BZ2_bzCompressInit.restype = c_int -libbz2.BZ2_bzCompressEnd.argtypes = [POINTER(bz_stream)] -libbz2.BZ2_bzCompressEnd.restype = c_int -libbz2.BZ2_bzCompress.argtypes = [POINTER(bz_stream), c_int] -libbz2.BZ2_bzCompress.restype = c_int - -libbz2.BZ2_bzDecompressInit.argtypes = [POINTER(bz_stream), c_int, c_int] -libbz2.BZ2_bzDecompressInit.restype = c_int -libbz2.BZ2_bzDecompressEnd.argtypes = [POINTER(bz_stream)] -libbz2.BZ2_bzDecompressEnd.restype = c_int -libbz2.BZ2_bzDecompress.argtypes = [POINTER(bz_stream)] -libbz2.BZ2_bzDecompress.restype = c_int +BZ2_bzReadOpen = external('BZ2_bzReadOpen', [rffi.INTP, FILE, rffi.INT, + rffi.INT, rffi.INTP, rffi.INT], BZFILE) +BZ2_bzWriteOpen = external('BZ2_bzWriteOpen', [rffi.INTP, FILE, rffi.INT, + rffi.INT, rffi.INT], BZFILE) +BZ2_bzReadClose = external('BZ2_bzReadClose', [rffi.INTP, BZFILE], lltype.Void) +BZ2_bzWriteClose = external('BZ2_bzWriteClose', [rffi.INTP, BZFILE, + rffi.INT, rffi.UINTP, rffi.UINTP], lltype.Void) +BZ2_bzRead = external('BZ2_bzRead', [rffi.INTP, BZFILE, rffi.CCHARP, rffi.INT], + rffi.INT) +BZ2_bzWrite = external('BZ2_bzWrite', [rffi.INTP, BZFILE, rffi.CCHARP, + rffi.INT], lltype.Void) +BZ2_bzCompressInit = external('BZ2_bzCompressInit', [bz_stream, rffi.INT, + rffi.INT, rffi.INT], rffi.INT) +BZ2_bzCompressEnd = external('BZ2_bzCompressEnd', [bz_stream], rffi.INT) +BZ2_bzCompress = external('BZ2_bzCompress', [bz_stream, rffi.INT], rffi.INT) +BZ2_bzDecompressInit = external('BZ2_bzDecompressInit', [bz_stream, rffi.INT, + rffi.INT], rffi.INT) +BZ2_bzDecompressEnd = external('BZ2_bzDecompressEnd', [bz_stream], rffi.INT) +BZ2_bzDecompress = external('BZ2_bzDecompress', [bz_stream], rffi.INT) def _catch_bz2_error(space, bzerror): if BZ_CONFIG_ERROR and bzerror == BZ_CONFIG_ERROR: @@ -311,24 +313,27 @@ must be a number between 1 and 9.""" def __init__(self, space, compresslevel): self.space = space - self.bzs = bz_stream() + self.bzs = lltype.malloc(bz_stream, flavor='raw', zero=True) self.running = False + self.initialized = False self._init_bz2comp(compresslevel) + self.initialized = True __init__.unwrap_spec = ['self', ObjSpace, int] def _init_bz2comp(self, compresslevel): if compresslevel < 1 or compresslevel > 9: raise OperationError(self.space.w_ValueError, self.space.wrap("compresslevel must be between 1 and 9")) - - bzerror = libbz2.BZ2_bzCompressInit(byref(self.bzs), compresslevel, 0, 0) + + bzerror = BZ2_bzCompressInit(self.bzs, compresslevel, 0, 0) if bzerror != BZ_OK: _catch_bz2_error(self.space, bzerror) self.running = True def __del__(self): - libbz2.BZ2_bzCompressEnd(byref(self.bzs)) + if self.initialized: + BZ2_bzCompressEnd(self.bzs) def compress(self, data): """compress(data) -> string @@ -354,14 +359,14 @@ in_buf = create_string_buffer(in_bufsize) in_buf.value = data - self.bzs.next_in = cast(in_buf, POINTER(c_char)) + self.bzs.next_in = cast(in_buf, rffi.CCHARP) self.bzs.avail_in = in_bufsize - self.bzs.next_out = cast(out_buf, POINTER(c_char)) + self.bzs.next_out = cast(out_buf, rffi.CCHARP) self.bzs.avail_out = out_bufsize temp = [] while True: - bzerror = libbz2.BZ2_bzCompress(byref(self.bzs), BZ_RUN) + bzerror = BZ2_bzCompress(self.bzs, BZ_RUN) if bzerror != BZ_RUN_OK: _catch_bz2_error(self.space, bzerror) @@ -374,7 +379,7 @@ out_bufsize = _new_buffer_size(out_bufsize) out_buf = create_string_buffer(out_bufsize) - self.bzs.next_out = cast(out_buf, POINTER(c_char)) + self.bzs.next_out = cast(out_buf, rffi.CCHARP) self.bzs.avail_out = out_bufsize if temp: @@ -397,14 +402,14 @@ out_bufsize = SMALLCHUNK out_buf = create_string_buffer(out_bufsize) - self.bzs.next_out = cast(out_buf, POINTER(c_char)) + self.bzs.next_out = cast(out_buf, rffi.CCHARP) self.bzs.avail_out = out_bufsize total_out = _bzs_total_out(self.bzs) temp = [] while True: - bzerror = libbz2.BZ2_bzCompress(byref(self.bzs), BZ_FINISH) + bzerror = BZ2_bzCompress(self.bzs, BZ_FINISH) if bzerror == BZ_STREAM_END: break elif bzerror != BZ_FINISH_OK: @@ -416,7 +421,7 @@ out_bufsize = _new_buffer_size(out_bufsize) out_buf = create_string_buffer(out_bufsize) - self.bzs.next_out = cast(out_buf, POINTER(c_char)) + self.bzs.next_out = cast(out_buf, rffi.CCHARP) self.bzs.avail_out = out_bufsize if temp: @@ -464,17 +469,17 @@ self._init_bz2decomp() def _init_bz2decomp(self): - bzerror = libbz2.BZ2_bzDecompressInit(byref(self.bzs), 0, 0) + bzerror = BZ2_bzDecompressInit(self.bzs, 0, 0) if bzerror != BZ_OK: _catch_bz2_error(self.space, bzerror) self.running = True def __del__(self): - libbz2.BZ2_bzDecompressEnd(byref(self.bzs)) + BZ2_bzDecompressEnd(self.bzs) def decompress(self, data): - """"decompress(data) -> string + """decompress(data) -> string Provide more data to the decompressor object. It will return chunks of decompressed data whenever possible. If you try to decompress data @@ -493,14 +498,14 @@ out_bufsize = SMALLCHUNK out_buf = create_string_buffer(out_bufsize) - self.bzs.next_in = cast(in_buf, POINTER(c_char)) + self.bzs.next_in = cast(in_buf, rffi.CCHARP) self.bzs.avail_in = in_bufsize - self.bzs.next_out = cast(out_buf, POINTER(c_char)) + self.bzs.next_out = cast(out_buf, rffi.CCHARP) self.bzs.avail_out = out_bufsize temp = [] while True: - bzerror = libbz2.BZ2_bzDecompress(byref(self.bzs)) + bzerror = BZ2_bzDecompress(self.bzs) if bzerror == BZ_STREAM_END: if self.bzs.avail_in != 0: unused = [self.bzs.next_in[i] for i in range(self.bzs.avail_in)] @@ -519,7 +524,7 @@ out_bufsize = _new_buffer_size(out_bufsize) out_buf = create_string_buffer(out_bufsize) - self.bzs.next_out = cast(out_buf, POINTER(c_char)) + self.bzs.next_out = cast(out_buf, rffi.CCHARP) self.bzs.avail_out = out_bufsize if temp: @@ -564,23 +569,23 @@ in_buf = create_string_buffer(in_bufsize) in_buf.value = data - bzs.next_in = cast(in_buf, POINTER(c_char)) + bzs.next_in = cast(in_buf, rffi.CCHARP) bzs.avail_in = in_bufsize - bzs.next_out = cast(out_buf, POINTER(c_char)) + bzs.next_out = cast(out_buf, rffi.CCHARP) bzs.avail_out = out_bufsize - bzerror = libbz2.BZ2_bzCompressInit(byref(bzs), compresslevel, 0, 0) + bzerror = BZ2_bzCompressInit(bzs, compresslevel, 0, 0) if bzerror != BZ_OK: _catch_bz2_error(space, bzerror) total_out = _bzs_total_out(bzs) temp = [] while True: - bzerror = libbz2.BZ2_bzCompress(byref(bzs), BZ_FINISH) + bzerror = BZ2_bzCompress(bzs, BZ_FINISH) if bzerror == BZ_STREAM_END: break elif bzerror != BZ_FINISH_OK: - libbz2.BZ2_bzCompressEnd(byref(bzs)) + BZ2_bzCompressEnd(bzs) _catch_bz2_error(space, bzerror) if bzs.avail_out == 0: @@ -589,7 +594,7 @@ out_bufsize = _new_buffer_size(out_bufsize) out_buf = create_string_buffer(out_bufsize) - bzs.next_out = cast(out_buf, POINTER(c_char)) + bzs.next_out = cast(out_buf, rffi.CCHARP) bzs.avail_out = out_bufsize if temp: @@ -602,7 +607,7 @@ total_out = _bzs_total_out(bzs) res = "".join([out_buf[i] for i in range(total_out)]) - libbz2.BZ2_bzCompressEnd(byref(bzs)) + BZ2_bzCompressEnd(bzs) return space.wrap(res) compress.unwrap_spec = [ObjSpace, str, int] @@ -624,26 +629,26 @@ out_bufsize = SMALLCHUNK out_buf = create_string_buffer(out_bufsize) - bzs.next_in = cast(in_buf, POINTER(c_char)) + bzs.next_in = cast(in_buf, rffi.CCHARP) bzs.avail_in = in_bufsize - bzs.next_out = cast(out_buf, POINTER(c_char)) + bzs.next_out = cast(out_buf, rffi.CCHARP) bzs.avail_out = out_bufsize - bzerror = libbz2.BZ2_bzDecompressInit(byref(bzs), 0, 0) + bzerror = BZ2_bzDecompressInit(bzs, 0, 0) if bzerror != BZ_OK: _catch_bz2_error(space, bzerror) temp = [] while True: - bzerror = libbz2.BZ2_bzDecompress(byref(bzs)) + bzerror = BZ2_bzDecompress(bzs) if bzerror == BZ_STREAM_END: break if bzerror != BZ_OK: - libbz2.BZ2_bzDecompressEnd(byref(bzs)) + BZ2_bzDecompressEnd(bzs) _catch_bz2_error(space, bzerror) if bzs.avail_in == 0: - libbz2.BZ2_bzDecompressEnd(byref(bzs)) + BZ2_bzDecompressEnd(bzs) raise OperationError(space.w_ValueError, space.wrap("couldn't find end of stream")) elif bzs.avail_out == 0: @@ -653,7 +658,7 @@ out_bufsize = _new_buffer_size(out_bufsize) out_buf = create_string_buffer(out_bufsize) - bzs.next_out = cast(out_buf, POINTER(c_char)) + bzs.next_out = cast(out_buf, rffi.CCHARP) bzs.avail_out = out_bufsize total_out = _bzs_total_out(bzs) @@ -664,6 +669,6 @@ else: res = "".join([out_buf[i] for i in range(total_out) if out_buf[i] != '\x00']) - libbz2.BZ2_bzDecompressEnd(byref(bzs)) + BZ2_bzDecompressEnd(bzs) return space.wrap(res) decompress.unwrap_spec = [ObjSpace, str] From rxe at codespeak.net Thu Oct 4 22:12:06 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 4 Oct 2007 22:12:06 +0200 (CEST) Subject: [pypy-svn] r47166 - in pypy/dist/pypy/translator: . llvm Message-ID: <20071004201206.23A818199@code0.codespeak.net> Author: rxe Date: Thu Oct 4 22:12:05 2007 New Revision: 47166 Modified: pypy/dist/pypy/translator/llvm/funcnode.py pypy/dist/pypy/translator/unsimplify.py Log: fix an ancient typo Modified: pypy/dist/pypy/translator/llvm/funcnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/funcnode.py (original) +++ pypy/dist/pypy/translator/llvm/funcnode.py Thu Oct 4 22:12:05 2007 @@ -4,7 +4,7 @@ from pypy.translator.llvm.node import LLVMNode, ConstantLLVMNode from pypy.translator.llvm.opwriter import OpWriter from pypy.translator.llvm.log import log -from pypy.translator.unsimplify import remove_double_links, no_links_to_startblack +from pypy.translator.unsimplify import remove_double_links, no_links_to_startblock log = log.funcnode class FuncTypeNode(LLVMNode): @@ -65,7 +65,7 @@ def post_setup_transform(self): remove_double_links(self.db.translator.annotator, self.graph) - no_links_to_startblack(self.graph) + no_links_to_startblock(self.graph) def writedecl(self, codewriter): codewriter.declare(self.getdecl()) Modified: pypy/dist/pypy/translator/unsimplify.py ============================================================================== --- pypy/dist/pypy/translator/unsimplify.py (original) +++ pypy/dist/pypy/translator/unsimplify.py Thu Oct 4 22:12:05 2007 @@ -144,7 +144,7 @@ insert_empty_block(annotator, link) traverse(visit, graph) -def no_links_to_startblack(graph): +def no_links_to_startblock(graph): """Ensure no links to start block.""" links_to_start_block = False for block in graph.iterblocks(): From cfbolz at codespeak.net Thu Oct 4 23:07:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 4 Oct 2007 23:07:28 +0200 (CEST) Subject: [pypy-svn] r47171 - pypy/dist/pypy/doc Message-ID: <20071004210728.8CBF681A5@code0.codespeak.net> Author: cfbolz Date: Thu Oct 4 23:07:26 2007 New Revision: 47171 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: remove rsdl and rcairo since they are not really based on rctypes. Add cleanup of genrffi as a task. Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Thu Oct 4 23:07:26 2007 @@ -23,6 +23,7 @@ timeshifter) - unicode strings in RPython - finish rctypes removal + - move genrffi to not use rctypes stuff any more - think about approaches to id, especially concerning boehm, where the id will keep the object alive and concerning a moving GC @@ -41,7 +42,5 @@ - mmap - readline - signal - - rsdl - - rcairo - rewrite or kill the extcompiler From rxe at codespeak.net Fri Oct 5 00:20:44 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 5 Oct 2007 00:20:44 +0200 (CEST) Subject: [pypy-svn] r47173 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071004222044.0FA6581AE@code0.codespeak.net> Author: rxe Date: Fri Oct 5 00:20:43 2007 New Revision: 47173 Modified: pypy/dist/pypy/translator/llvm/test/runtest.py Log: disable tests until things stablize a bit. Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Fri Oct 5 00:20:43 2007 @@ -1,4 +1,5 @@ import py +py.test.skip("llvm is a state of flux") from pypy.translator.llvm.buildllvm import llvm_is_on_path, llvm_version, gcc_version from pypy.translator.llvm.genllvm import GenLLVM From cfbolz at codespeak.net Fri Oct 5 00:36:21 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 5 Oct 2007 00:36:21 +0200 (CEST) Subject: [pypy-svn] r47174 - pypy/branch/kill-keepalives-again/pypy/translator/backendopt/test Message-ID: <20071004223621.B0C9D81AE@code0.codespeak.net> Author: cfbolz Date: Fri Oct 5 00:36:21 2007 New Revision: 47174 Modified: pypy/branch/kill-keepalives-again/pypy/translator/backendopt/test/test_malloc.py Log: add some backend removal tests about the removal of the intermediate small structs that are passed around. Works out of a box. Also add a test about getinteriorptr and setinteriorptr. Fails, because the malloc remover does not know these ops. I am not sure how imporant it is to add them, since these ops are hopefully kind of rare. I guess it would be nice for completeness. Modified: pypy/branch/kill-keepalives-again/pypy/translator/backendopt/test/test_malloc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/backendopt/test/test_malloc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/backendopt/test/test_malloc.py Fri Oct 5 00:36:21 2007 @@ -330,6 +330,45 @@ [link] = entrymap[graph.returnblock] assert link.prevblock.operations[-1].opname == 'keepalive' + def test_interior_ptr(self): + py.test.skip("fails") + S = lltype.Struct("S", ('x', lltype.Signed)) + T = lltype.GcStruct("T", ('s', S)) + def f(x): + t = lltype.malloc(T) + t.s.x = x + return t.s.x + graph = self.check(f, [int], [42], 42) + + def test_interior_ptr_with_index(self): + S = lltype.Struct("S", ('x', lltype.Signed)) + T = lltype.GcArray(S) + def f(x): + t = lltype.malloc(T, 1) + t[0].x = x + return t[0].x + graph = self.check(f, [int], [42], 42) + + def test_interior_ptr_with_field_and_index(self): + S = lltype.Struct("S", ('x', lltype.Signed)) + T = lltype.GcStruct("T", ('items', lltype.Array(S))) + def f(x): + t = lltype.malloc(T, 1) + t.items[0].x = x + return t.items[0].x + graph = self.check(f, [int], [42], 42) + + def test_interior_ptr_with_index_and_field(self): + S = lltype.Struct("S", ('x', lltype.Signed)) + T = lltype.Struct("T", ('s', S)) + U = lltype.GcArray(T) + def f(x): + u = lltype.malloc(U, 1) + u[0].s.x = x + return u[0].s.x + graph = self.check(f, [int], [42], 42) + + class TestOOTypeMallocRemoval(BaseMallocRemovalTest): type_system = 'ootype' MallocRemover = OOTypeMallocRemover From rxe at codespeak.net Fri Oct 5 01:16:07 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 5 Oct 2007 01:16:07 +0200 (CEST) Subject: [pypy-svn] r47175 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071004231607.5215681A1@code0.codespeak.net> Author: rxe Date: Fri Oct 5 01:16:05 2007 New Revision: 47175 Modified: pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/extfuncnode.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/node.py pypy/dist/pypy/translator/llvm/test/test_rffi.py Log: some exploration into rffi - test_basic() seems to work. Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Fri Oct 5 01:16:05 2007 @@ -167,7 +167,7 @@ includestr += "-I %s " % ii return includestr -def generate_llfile(db, extern_decls, entrynode, standalone): +def generate_llfile(db, extern_decls, entrynode, c_include, c_sources, standalone): ccode = [] function_names = [] @@ -211,6 +211,10 @@ # ask gcpolicy for any code needed ccode.append('%s\n' % db.gcpolicy.genextern_code()) + for c_source in c_sources: + for l in c_source: + ccode.append(l) + # append our source file ccode.append(open(get_module_file('genexterns.c')).read()) Modified: pypy/dist/pypy/translator/llvm/extfuncnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/extfuncnode.py (original) +++ pypy/dist/pypy/translator/llvm/extfuncnode.py Fri Oct 5 01:16:05 2007 @@ -33,6 +33,19 @@ self.value = value self.ref = "%" + value._name + def external_c_source(self): + # return a list of unique includes and sources in C + c_includes = [] + c_sources = [] + + if hasattr(self.value, 'includes'): + c_includes = list(self.value.includes) + + if hasattr(self.value, 'sources'): + c_sources = list(self.value.sources) + + return c_includes, c_sources + def writeglobalconstants(self, codewriter): pass Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Fri Oct 5 01:16:05 2007 @@ -175,10 +175,22 @@ return c.value._obj def generate_ll_externs(self): + c_includes = {} + c_sources = {} + + for node in self.db.getnodes(): + includes, sources = node.external_c_source() + for include in includes: + c_includes[include] = True + for source in sources: + c_sources[source] = True + self.llexterns_header, self.llexterns_functions = \ generate_llfile(self.db, self.extern_decls, self.entrynode, + c_includes, + c_sources, self.standalone) def create_codewriter(self): Modified: pypy/dist/pypy/translator/llvm/node.py ============================================================================== --- pypy/dist/pypy/translator/llvm/node.py (original) +++ pypy/dist/pypy/translator/llvm/node.py Fri Oct 5 01:16:05 2007 @@ -25,6 +25,12 @@ def setup(self): pass + # ______________________________________________________________________ + + def external_c_source(self): + # return a list of unique includes and sources in C + return [], [] + # __________________ before "implementation" ____________________ Modified: pypy/dist/pypy/translator/llvm/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rffi.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rffi.py Fri Oct 5 01:16:05 2007 @@ -5,10 +5,29 @@ from pypy.tool.udir import udir from pypy.rlib.rarithmetic import r_uint -py.test.skip("Extfunc support in llvm needs refactoring (!!??!)") +from pypy.rpython.lltypesystem.rffi import * +from pypy.rpython.lltypesystem.lltype import Signed, Ptr, Char, malloc +from pypy.rpython.lltypesystem import lltype from pypy.translator.llvm.test.runtest import * +def test_basic(): + c_source = py.code.Source(""" + int z(int x) + { + return (x + 3); + } + """) + z = llexternal('z', [Signed], Signed, sources=[c_source]) + + def f(): + return z(8) + + xf = compile_function(f, []) + assert xf() == 8+3 + +#py.test.skip("Extfunc support in llvm needs refactoring (!!??!)" + def test_external_function_ll_os_dup(): def fn(): return os.dup(0) From fijal at codespeak.net Fri Oct 5 11:10:53 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 5 Oct 2007 11:10:53 +0200 (CEST) Subject: [pypy-svn] r47179 - pypy/dist/pypy/module/readline Message-ID: <20071005091053.9A4F0815C@code0.codespeak.net> Author: fijal Date: Fri Oct 5 11:10:52 2007 New Revision: 47179 Modified: pypy/dist/pypy/module/readline/c_readline.py pypy/dist/pypy/module/readline/interp_readline.py Log: Move readline from rctypes to rffi Modified: pypy/dist/pypy/module/readline/c_readline.py ============================================================================== --- pypy/dist/pypy/module/readline/c_readline.py (original) +++ pypy/dist/pypy/module/readline/c_readline.py Fri Oct 5 11:10:52 2007 @@ -1,42 +1,27 @@ -from ctypes import * -from pypy.rpython.rctypes.tool.ctypes_platform import configure, Library +from pypy.rpython.tool import rffi_platform as platform +from pypy.rpython.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, interp2app -#------------------------------------------------------------ -# configuration for binding to external readline library -# through rctypes -# -class CConfig: - _header_ = "" - _includes_ = ["readline/readline.h", "readline/history.h"] - readline = Library('readline') - -cconfig = configure(CConfig) -libreadline = cconfig['readline'] +includes = ["readline/readline.h", "readline/history.h"] +libs = ['readline'] +def external(name, args, result): + return rffi.llexternal(name, args, result, includes=includes, + libraries=libs) # get a binding to c library functions and define their args and return types # char *readline(char *) -c_readline = libreadline.readline -c_readline.argtypes = [c_char_p] -c_readline.restype = c_char_p +c_readline = external('readline', [rffi.CCHARP], rffi.CCHARP) # void rl_initiliaze(void) -c_rl_initialize = libreadline.rl_initialize -c_rl_initialize.argtypes = [] -c_rl_initialize.restype = None +c_rl_initialize = external('rl_initialize', [], lltype.Void) # void using_history(void) -c_using_history = libreadline.using_history -c_using_history.argtypes = [] -c_using_history.restype = None +c_using_history = external('using_history', [], lltype.Void) # void add_history(const char *) -c_add_history = libreadline.add_history -c_add_history.argtypes = [c_char_p] -c_add_history.restype = None - +c_add_history = external('add_history', [rffi.CCHARP], lltype.Void) #------------------------------------------------------------ # special initialization of readline @@ -54,9 +39,10 @@ space.wrap(app_readline_func)) def readline_func(space, prompt): - res = c_readline(prompt) - if res is None: + ll_res = c_readline(prompt) + if not ll_res: raise OperationError(space.w_EOFError, space.w_None) + res = rffi.charp2str(ll_res) if res and res != readlinestate.lastline: readlinestate.lastline = res c_add_history(res) Modified: pypy/dist/pypy/module/readline/interp_readline.py ============================================================================== --- pypy/dist/pypy/module/readline/interp_readline.py (original) +++ pypy/dist/pypy/module/readline/interp_readline.py Fri Oct 5 11:10:52 2007 @@ -3,13 +3,14 @@ from pypy.interpreter.baseobjspace import ObjSpace -from pypy.module.readline import c_readline +from pypy.module.readline import c_readline +from pypy.rpython.lltypesystem import rffi #------------------------------------------------------------ # exported API (see interpleveldefs in __init__.py) # def readline(space, prompt): - return space.wrap(c_readline.c_readline(prompt)) + return space.wrap(rffi.charp2str(c_readline.c_readline(prompt))) readline.unwrap_spec = [ObjSpace, str] def setcompleter(space, w_callback): From fijal at codespeak.net Fri Oct 5 11:11:33 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 5 Oct 2007 11:11:33 +0200 (CEST) Subject: [pypy-svn] r47180 - pypy/dist/pypy/module/fcntl Message-ID: <20071005091133.3CB7B815C@code0.codespeak.net> Author: fijal Date: Fri Oct 5 11:11:33 2007 New Revision: 47180 Modified: pypy/dist/pypy/module/fcntl/interp_fcntl.py Log: move fcntl from rctypes to rffi Modified: pypy/dist/pypy/module/fcntl/interp_fcntl.py ============================================================================== --- pypy/dist/pypy/module/fcntl/interp_fcntl.py (original) +++ pypy/dist/pypy/module/fcntl/interp_fcntl.py Fri Oct 5 11:11:33 2007 @@ -1,22 +1,16 @@ -from pypy.rpython.rctypes.tool import ctypes_platform -from pypy.rpython.rctypes.tool.libc import libc -import pypy.rpython.rctypes.implementation # this defines rctypes magic -from pypy.rpython.rctypes.aerrno import geterrno +from pypy.rpython.tool import rffi_platform as platform +from pypy.rpython.lltypesystem import rffi, lltype from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import W_Root, ObjSpace -from ctypes import * import sys class CConfig: - _header_ = """ - #include - #include - #include - """ - flock = ctypes_platform.Struct("struct flock", - [('l_start', c_longlong), ('l_len', c_longlong), - ('l_pid', c_long), ('l_type', c_short), - ('l_whence', c_short)]) + _includes_ = ['fcntl.h', 'sys/file.h', 'sys/ioctl.h'] + flock = platform.Struct("struct flock", + [('l_start', rffi.LONGLONG), ('l_len', rffi.LONGLONG), + ('l_pid', rffi.LONG), ('l_type', rffi.SHORT), + ('l_whence', rffi.SHORT)]) + has_flock = platform.Has('flock') # constants, look in fcntl.h and platform docs for the meaning # some constants are linux only so they will be correctly exposed outside @@ -35,12 +29,12 @@ 'I_PUNLINK', 'I_FLUSHBAND', 'I_CKBAND', 'I_GETBAND', 'I_ATMARK', 'I_SETCLTIME', 'I_GETCLTIME', 'I_CANPUT'] for name in constant_names: - setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name)) + setattr(CConfig, name, platform.DefinedConstantInteger(name)) class cConfig: pass -cConfig.__dict__.update(ctypes_platform.configure(CConfig)) +cConfig.__dict__.update(platform.configure(CConfig)) cConfig.flock.__name__ = "_flock" if "linux" in sys.platform: @@ -56,40 +50,24 @@ constants[name] = value locals().update(constants) -_flock = cConfig.flock -libc.strerror.restype = c_char_p -libc.strerror.argtypes = [c_int] - -fcntl_int = libc['fcntl'] -fcntl_int.argtypes = [c_int, c_int, c_int] -fcntl_int.restype = c_int - -fcntl_str = libc['fcntl'] -fcntl_str.argtypes = [c_int, c_int, c_char_p] -fcntl_str.restype = c_int - -fcntl_flock = libc['fcntl'] -fcntl_flock.argtypes = [c_int, c_int, POINTER(_flock)] -fcntl_flock.restype = c_int - -ioctl_int = libc['ioctl'] -ioctl_int.argtypes = [c_int, c_int, c_int] -ioctl_int.restype = c_int - -ioctl_str = libc['ioctl'] -ioctl_str.argtypes = [c_int, c_int, c_char_p] -ioctl_str.restype = c_int - - -has_flock = False -if hasattr(libc, "flock"): - libc.flock.argtypes = [c_int, c_int] - libc.flock.restype = c_int - has_flock = True +def external(name, args, result): + return rffi.llexternal(name, args, result, includes=CConfig._includes_) + +_flock = lltype.Ptr(cConfig.flock) +strerror = external('strerror', [rffi.INT], rffi.CCHARP) +fcntl_int = external('fcntl', [rffi.INT, rffi.INT, rffi.INT], rffi.INT) +fcntl_str = external('fcntl', [rffi.INT, rffi.INT, rffi.CCHARP], rffi.INT) +fcntl_flock = external('fcntl', [rffi.INT, rffi.INT, _flock], rffi.INT) +ioctl_int = external('ioctl', [rffi.INT, rffi.INT, rffi.INT], rffi.INT) +ioctl_str = external('ioctl', [rffi.INT, rffi.INT, rffi.CCHARP], rffi.INT) + +has_flock = cConfig.has_flock +if has_flock: + c_flock = external('flock', [rffi.INT, rffi.INT], rffi.INT) def _get_error_msg(): - errno = geterrno() - return libc.strerror(errno) + errno = rffi.get_errno() + return rffi.charp2str(strerror(errno)) def _get_module_object(space, obj_name): w_module = space.getbuiltinmodule('fcntl') @@ -102,17 +80,18 @@ return space.int_w(w_fd) def _check_flock_op(space, op): - l = _flock() if op == LOCK_UN: - l.l_type = F_UNLCK + l_type = F_UNLCK elif op & LOCK_SH: - l.l_type = F_RDLCK + l_type = F_RDLCK elif op & LOCK_EX: - l.l_type = F_WRLCK + l_type = F_WRLCK else: raise OperationError(space.w_ValueError, space.wrap("unrecognized flock argument")) + l = lltype.malloc(_flock.TO, flavor='raw') + l.c_l_type = rffi.cast(rffi.SHORT, l_type) return l def fcntl(space, w_fd, op, w_arg=0): @@ -161,15 +140,16 @@ fd = _conv_descriptor(space, w_fd) if has_flock: - rv = libc.flock(fd, op) + rv = c_flock(fd, op) if rv < 0: raise OperationError(space.w_IOError, space.wrap(_get_error_msg())) else: l = _check_flock_op(space, op) - l.l_whence = l.l_start = l.l_len = 0 + l.c_l_whence = l.c_l_start = l.c_l_len = 0 op = [F_SETLKW, F_SETLK][op & LOCK_NB] - fcntl_flock(fd, op, byref(l)) + fcntl_flock(fd, op, l) + lltype.free(l, flavor='raw') flock.unwrap_spec = [ObjSpace, W_Root, int] def lockf(space, w_fd, op, length=0, start=0, whence=0): @@ -200,20 +180,23 @@ fd = _conv_descriptor(space, w_fd) l = _check_flock_op(space, op) - l.l_start = l.l_len = 0 + l.c_l_start = l.c_l_len = 0 if start: - l.l_start = int(start) + l.c_l_start = int(start) if len: - l.l_len = int(length) - l.l_whence = whence + l.c_l_len = int(length) + l.c_l_whence = rffi.cast(rffi.SHORT, whence) try: - op = [F_SETLKW, F_SETLK][op & LOCK_NB] - except IndexError: - raise OperationError(space.w_ValueError, - space.wrap("invalid value for operation")) - fcntl_flock(fd, op, byref(l)) + try: + op = [F_SETLKW, F_SETLK][op & LOCK_NB] + except IndexError: + raise OperationError(space.w_ValueError, + space.wrap("invalid value for operation")) + fcntl_flock(fd, op, l) + finally: + lltype.free(l, flavor='raw') lockf.unwrap_spec = [ObjSpace, W_Root, int, int, int, int] def ioctl(space, w_fd, op, w_arg=0, mutate_flag=True): @@ -248,7 +231,7 @@ C code.""" fd = _conv_descriptor(space, w_fd) # Python turns number > sys.maxint into long, we need the signed C value - op = c_int(op).value + op = rffi.cast(rffi.INT, op) IOCTL_BUFSZ = 1024 @@ -265,13 +248,11 @@ raise OperationError(space.w_ValueError, space.wrap("ioctl string arg too long")) - buf = create_string_buffer(len(arg)) - - rv = ioctl_str(fd, op, buf) + rv = ioctl_str(fd, op, arg) if rv < 0: raise OperationError(space.w_IOError, space.wrap(_get_error_msg())) - return space.wrap(buf.raw) + return space.wrap(arg) else: raise OperationError(space.w_TypeError, space.wrap("an integer or a buffer required")) From fijal at codespeak.net Fri Oct 5 11:15:33 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 5 Oct 2007 11:15:33 +0200 (CEST) Subject: [pypy-svn] r47181 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20071005091533.6A298815D@code0.codespeak.net> Author: fijal Date: Fri Oct 5 11:15:32 2007 New Revision: 47181 Added: pypy/dist/pypy/module/bz2/test/__init__.py (contents, props changed) pypy/dist/pypy/module/bz2/test/support.py (contents, props changed) Removed: pypy/dist/pypy/module/bz2/bz2module.c pypy/dist/pypy/module/bz2/bzlib.py Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py pypy/dist/pypy/module/bz2/test/test_bz2_file.py Log: * move bz2 module from rctypes to rffi * check allocation at the end of every function, unfortunately this makes py.test choke when C-c Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Fri Oct 5 11:15:32 2007 @@ -1,34 +1,39 @@ -from pypy.rpython.rctypes.tool import ctypes_platform -import pypy.rpython.rctypes.implementation # this defines rctypes magic +from pypy.rpython.tool import rffi_platform as platform +from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem import lltype from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.typedef import interp_attrproperty from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, interp2app from pypy.rlib.streamio import Stream -from ctypes import * -import ctypes.util import sys -from bzlib import bz_stream, BZFILE, FILE - -#libbz2 = cdll.LoadLibrary(ctypes.util.find_library("bz2")) - -c_void = None - class CConfig: - _header_ = """ - #include - #include - #include - """ - - _includes_ = ["bzlib.h"] - libbz2 = ctypes_platform.Library('bz2') - off_t = ctypes_platform.SimpleType("off_t", c_longlong) - size_t = ctypes_platform.SimpleType("size_t", c_ulong) - BUFSIZ = ctypes_platform.ConstantInteger("BUFSIZ") - SEEK_SET = ctypes_platform.ConstantInteger("SEEK_SET") + _includes_ = ['stdio.h', 'sys/types.h', 'bzlib.h'] + _libraries_ = ['bz2'] + calling_conv = 'c' + + off_t = platform.SimpleType("off_t", rffi.LONGLONG) + size_t = platform.SimpleType("size_t", rffi.ULONG) + BUFSIZ = platform.ConstantInteger("BUFSIZ") + _alloc_type = lltype.FuncType([rffi.VOIDP, rffi.INT, rffi.INT], rffi.VOIDP) + _free_type = lltype.FuncType([rffi.VOIDP, rffi.VOIDP], lltype.Void) + SEEK_SET = platform.ConstantInteger("SEEK_SET") + bz_stream = platform.Struct('bz_stream', + [('next_in', rffi.CCHARP), + ('avail_in', rffi.UINT), + ('total_in_lo32', rffi.UINT), + ('total_in_hi32', rffi.UINT), + ('next_out', rffi.CCHARP), + ('avail_out', rffi.UINT), + ('total_out_lo32', rffi.UINT), + ('total_out_hi32', rffi.UINT), + ('state', rffi.VOIDP), + ('bzalloc', lltype.Ptr(_alloc_type)), + ('bzfree', lltype.Ptr(_free_type)), + ('opaque', rffi.VOIDP), + ]) constants = {} constant_names = ['BZ_RUN', 'BZ_FLUSH', 'BZ_FINISH', 'BZ_OK', @@ -37,11 +42,11 @@ 'BZ_DATA_ERROR_MAGIC', 'BZ_IO_ERROR', 'BZ_UNEXPECTED_EOF', 'BZ_OUTBUFF_FULL', 'BZ_CONFIG_ERROR'] for name in constant_names: - setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name)) + setattr(CConfig, name, platform.DefinedConstantInteger(name)) class cConfig: pass -cConfig.__dict__.update(ctypes_platform.configure(CConfig)) +cConfig.__dict__.update(platform.configure(CConfig)) for name in constant_names: value = getattr(cConfig, name) @@ -49,8 +54,8 @@ constants[name] = value locals().update(constants) -libbz2 = cConfig.libbz2 off_t = cConfig.off_t +bz_stream = lltype.Ptr(cConfig.bz_stream) BUFSIZ = cConfig.BUFSIZ SEEK_SET = cConfig.SEEK_SET BZ_OK = cConfig.BZ_OK @@ -69,56 +74,53 @@ else: SMALLCHUNK = BUFSIZ -if sizeof(c_int) > 4: +if rffi.sizeof(rffi.INT) > 4: BIGCHUNK = 512 * 32 else: BIGCHUNK = 512 * 1024 - + MAXINT = sys.maxint if BZ_CONFIG_ERROR: - if sizeof(c_long) >= 8 or sizeof(c_longlong) >= 8: + if rffi.sizeof(rffi.LONG) >= 8 or rffi.sizeof(rffi.LONGLONG) >= 8: def _bzs_total_out(bzs): - return (bzs.total_out_hi32 << 32) + bzs.total_out_lo32 + return (bzs.c_total_out_hi32 << 32) + bzs.c_total_out_lo32 else: def _bzs_total_out(bzs): - return bzs.total_out_lo32 + return bzs.c_total_out_lo32 else: def _bzs_total_out(bzs): return bzs.total_out -# the least but one parameter should be c_void_p but it's not used +def external(name, args, result): + return rffi.llexternal(name, args, result, includes=CConfig._includes_, + libraries=['bz2']) + +FILE = rffi.COpaquePtr('FILE') +BZFILE = rffi.COpaquePtr('BZFILE') + +# the least but one parameter should be rffi.VOIDP but it's not used # so I trick the compiler to not complain about constanst pointer passed # to void* arg -libbz2.BZ2_bzReadOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, - c_int, POINTER(c_int), c_int] -libbz2.BZ2_bzReadOpen.restype = POINTER(BZFILE) -libbz2.BZ2_bzWriteOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, - c_int, c_int] -libbz2.BZ2_bzWriteOpen.restype = POINTER(BZFILE) -libbz2.BZ2_bzReadClose.argtypes = [POINTER(c_int), POINTER(BZFILE)] -libbz2.BZ2_bzReadClose.restype = c_void -libbz2.BZ2_bzWriteClose.argtypes = [POINTER(c_int), POINTER(BZFILE), - c_int, POINTER(c_uint), POINTER(c_uint)] -libbz2.BZ2_bzWriteClose.restype = c_void -libbz2.BZ2_bzRead.argtypes = [POINTER(c_int), POINTER(BZFILE), POINTER(c_char), c_int] -libbz2.BZ2_bzRead.restype = c_int -libbz2.BZ2_bzWrite.argtypes = [POINTER(c_int), POINTER(BZFILE), c_char_p, c_int] -libbz2.BZ2_bzWrite.restype = c_void - -libbz2.BZ2_bzCompressInit.argtypes = [POINTER(bz_stream), c_int, c_int, c_int] -libbz2.BZ2_bzCompressInit.restype = c_int -libbz2.BZ2_bzCompressEnd.argtypes = [POINTER(bz_stream)] -libbz2.BZ2_bzCompressEnd.restype = c_int -libbz2.BZ2_bzCompress.argtypes = [POINTER(bz_stream), c_int] -libbz2.BZ2_bzCompress.restype = c_int - -libbz2.BZ2_bzDecompressInit.argtypes = [POINTER(bz_stream), c_int, c_int] -libbz2.BZ2_bzDecompressInit.restype = c_int -libbz2.BZ2_bzDecompressEnd.argtypes = [POINTER(bz_stream)] -libbz2.BZ2_bzDecompressEnd.restype = c_int -libbz2.BZ2_bzDecompress.argtypes = [POINTER(bz_stream)] -libbz2.BZ2_bzDecompress.restype = c_int +BZ2_bzReadOpen = external('BZ2_bzReadOpen', [rffi.INTP, FILE, rffi.INT, + rffi.INT, rffi.INTP, rffi.INT], BZFILE) +BZ2_bzWriteOpen = external('BZ2_bzWriteOpen', [rffi.INTP, FILE, rffi.INT, + rffi.INT, rffi.INT], BZFILE) +BZ2_bzReadClose = external('BZ2_bzReadClose', [rffi.INTP, BZFILE], lltype.Void) +BZ2_bzWriteClose = external('BZ2_bzWriteClose', [rffi.INTP, BZFILE, + rffi.INT, rffi.UINTP, rffi.UINTP], lltype.Void) +BZ2_bzRead = external('BZ2_bzRead', [rffi.INTP, BZFILE, rffi.CCHARP, rffi.INT], + rffi.INT) +BZ2_bzWrite = external('BZ2_bzWrite', [rffi.INTP, BZFILE, rffi.CCHARP, + rffi.INT], lltype.Void) +BZ2_bzCompressInit = external('BZ2_bzCompressInit', [bz_stream, rffi.INT, + rffi.INT, rffi.INT], rffi.INT) +BZ2_bzCompressEnd = external('BZ2_bzCompressEnd', [bz_stream], rffi.INT) +BZ2_bzCompress = external('BZ2_bzCompress', [bz_stream, rffi.INT], rffi.INT) +BZ2_bzDecompressInit = external('BZ2_bzDecompressInit', [bz_stream, rffi.INT, + rffi.INT], rffi.INT) +BZ2_bzDecompressEnd = external('BZ2_bzDecompressEnd', [bz_stream], rffi.INT) +BZ2_bzDecompress = external('BZ2_bzDecompress', [bz_stream], rffi.INT) def _catch_bz2_error(space, bzerror): if BZ_CONFIG_ERROR and bzerror == BZ_CONFIG_ERROR: @@ -258,7 +260,6 @@ raise OperationError(self.space.w_IOError, self.space.wrap("file is not ready for writing")) - class WriteBZ2Filter(Stream): """Standard I/O stream filter that compresses the stream with bz2.""" @@ -294,7 +295,6 @@ def try_to_find_file_descriptor(self): return self.stream.try_to_find_file_descriptor() - def descr_compressor__new__(space, w_subtype, compresslevel=9): x = space.allocate_instance(W_BZ2Compressor, w_subtype) x = space.interp_w(W_BZ2Compressor, x) @@ -311,7 +311,7 @@ must be a number between 1 and 9.""" def __init__(self, space, compresslevel): self.space = space - self.bzs = bz_stream() + self.bzs = lltype.malloc(bz_stream.TO, flavor='raw', zero=True) self.running = False self._init_bz2comp(compresslevel) __init__.unwrap_spec = ['self', ObjSpace, int] @@ -320,15 +320,16 @@ if compresslevel < 1 or compresslevel > 9: raise OperationError(self.space.w_ValueError, self.space.wrap("compresslevel must be between 1 and 9")) - - bzerror = libbz2.BZ2_bzCompressInit(byref(self.bzs), compresslevel, 0, 0) + + bzerror = BZ2_bzCompressInit(self.bzs, compresslevel, 0, 0) if bzerror != BZ_OK: _catch_bz2_error(self.space, bzerror) self.running = True def __del__(self): - libbz2.BZ2_bzCompressEnd(byref(self.bzs)) + BZ2_bzCompressEnd(self.bzs) + lltype.free(self.bzs, flavor='raw') def compress(self, data): """compress(data) -> string @@ -348,44 +349,54 @@ self.space.wrap("this object was already flushed")) out_bufsize = SMALLCHUNK - out_buf = create_string_buffer(out_bufsize) + out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw', + zero=True) in_bufsize = datasize - in_buf = create_string_buffer(in_bufsize) - in_buf.value = data - - self.bzs.next_in = cast(in_buf, POINTER(c_char)) - self.bzs.avail_in = in_bufsize - self.bzs.next_out = cast(out_buf, POINTER(c_char)) - self.bzs.avail_out = out_bufsize - - temp = [] - while True: - bzerror = libbz2.BZ2_bzCompress(byref(self.bzs), BZ_RUN) - if bzerror != BZ_RUN_OK: - _catch_bz2_error(self.space, bzerror) + in_buf = lltype.malloc(rffi.CCHARP.TO, in_bufsize, flavor='raw') + for i in range(datasize): + in_buf[i] = data[i] + + try: + + self.bzs.c_next_in = in_buf + self.bzs.c_avail_in = rffi.cast(rffi.UINT, in_bufsize) + self.bzs.c_next_out = out_buf + self.bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize) + + temp = [] + while True: + bzerror = BZ2_bzCompress(self.bzs, BZ_RUN) + if bzerror != BZ_RUN_OK: + _catch_bz2_error(self.space, bzerror) - if self.bzs.avail_in == 0: - break - elif self.bzs.avail_out == 0: + if self.bzs.c_avail_in == 0: + break + elif self.bzs.c_avail_out == 0: + total_out = _bzs_total_out(self.bzs) + data = "".join([out_buf[i] for i in range(total_out)]) + temp.append(data) + + out_bufsize = _new_buffer_size(out_bufsize) + lltype.free(out_buf, flavor='raw') + out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, + flavor='raw', zero=True) + self.bzs.c_next_out = out_buf + self.bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize) + + if temp: total_out = _bzs_total_out(self.bzs) - data = "".join([out_buf[i] for i in range(total_out)]) + data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))]) temp.append(data) - - out_bufsize = _new_buffer_size(out_bufsize) - out_buf = create_string_buffer(out_bufsize) - self.bzs.next_out = cast(out_buf, POINTER(c_char)) - self.bzs.avail_out = out_bufsize + return self.space.wrap("".join(temp)) - if temp: total_out = _bzs_total_out(self.bzs) - data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))]) - temp.append(data) - return self.space.wrap("".join(temp)) + res = "".join([out_buf[i] for i in range(total_out)]) + return self.space.wrap(res) + finally: + lltype.free(in_buf, flavor='raw') + lltype.free(out_buf, flavor='raw') - total_out = _bzs_total_out(self.bzs) - res = "".join([out_buf[i] for i in range(total_out)]) - return self.space.wrap(res) compress.unwrap_spec = ['self', str] def flush(self): @@ -395,41 +406,47 @@ self.running = False out_bufsize = SMALLCHUNK - out_buf = create_string_buffer(out_bufsize) + out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw', + zero=True) + + try: - self.bzs.next_out = cast(out_buf, POINTER(c_char)) - self.bzs.avail_out = out_bufsize - - total_out = _bzs_total_out(self.bzs) + self.bzs.c_next_out = out_buf + self.bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize) - temp = [] - while True: - bzerror = libbz2.BZ2_bzCompress(byref(self.bzs), BZ_FINISH) - if bzerror == BZ_STREAM_END: - break - elif bzerror != BZ_FINISH_OK: - _catch_bz2_error(self.space, bzerror) - - if self.bzs.avail_out == 0: - data = "".join([out_buf[i] for i in range(_bzs_total_out(self.bzs))]) - temp.append(data) + total_out = _bzs_total_out(self.bzs) + + temp = [] + while True: + bzerror = BZ2_bzCompress(self.bzs, BZ_FINISH) + if bzerror == BZ_STREAM_END: + break + elif bzerror != BZ_FINISH_OK: + _catch_bz2_error(self.space, bzerror) - out_bufsize = _new_buffer_size(out_bufsize) - out_buf = create_string_buffer(out_bufsize) - self.bzs.next_out = cast(out_buf, POINTER(c_char)) - self.bzs.avail_out = out_bufsize + if self.bzs.c_avail_out == 0: + data = "".join([out_buf[i] for i in range(_bzs_total_out(self.bzs))]) + temp.append(data) + + out_bufsize = _new_buffer_size(out_bufsize) + out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, + flavor='raw', zero=True) + self.bzs.c_next_out = out_buf + self.bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize) - if temp: - return self.space.wrap("".join(temp)) + if temp: + return self.space.wrap("".join(temp)) - if self.bzs.avail_out: - size = _bzs_total_out(self.bzs) - total_out - res = "".join([out_buf[i] for i in range(size)]) - return self.space.wrap(res) + if self.bzs.c_avail_out: + size = _bzs_total_out(self.bzs) - total_out + res = "".join([out_buf[i] for i in range(size)]) + return self.space.wrap(res) - total_out = _bzs_total_out(self.bzs) - res = "".join([out_buf[i] for i in range(total_out)]) - return self.space.wrap(res) + total_out = _bzs_total_out(self.bzs) + res = "".join([out_buf[i] for i in range(total_out)]) + return self.space.wrap(res) + finally: + lltype.free(out_buf, flavor='raw') flush.unwrap_spec = ['self'] W_BZ2Compressor.typedef = TypeDef("BZ2Compressor", @@ -456,25 +473,26 @@ def __init__(self, space): self.space = space - - self.bzs = bz_stream() + + self.bzs = lltype.malloc(bz_stream.TO, flavor='raw', zero=True) self.running = False self.unused_data = "" self._init_bz2decomp() def _init_bz2decomp(self): - bzerror = libbz2.BZ2_bzDecompressInit(byref(self.bzs), 0, 0) + bzerror = BZ2_bzDecompressInit(self.bzs, 0, 0) if bzerror != BZ_OK: _catch_bz2_error(self.space, bzerror) self.running = True def __del__(self): - libbz2.BZ2_bzDecompressEnd(byref(self.bzs)) + BZ2_bzDecompressEnd(self.bzs) + lltype.free(self.bzs, flavor='raw') def decompress(self, data): - """"decompress(data) -> string + """decompress(data) -> string Provide more data to the decompressor object. It will return chunks of decompressed data whenever possible. If you try to decompress data @@ -487,50 +505,59 @@ self.space.wrap("end of stream was already found")) in_bufsize = len(data) - in_buf = create_string_buffer(in_bufsize) - in_buf.value = data + in_buf = lltype.malloc(rffi.CCHARP.TO, in_bufsize, flavor='raw') + for i in range(in_bufsize): + in_buf[i] = data[i] out_bufsize = SMALLCHUNK - out_buf = create_string_buffer(out_bufsize) - - self.bzs.next_in = cast(in_buf, POINTER(c_char)) - self.bzs.avail_in = in_bufsize - self.bzs.next_out = cast(out_buf, POINTER(c_char)) - self.bzs.avail_out = out_bufsize + out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw', + zero=True) + + try: - temp = [] - while True: - bzerror = libbz2.BZ2_bzDecompress(byref(self.bzs)) - if bzerror == BZ_STREAM_END: - if self.bzs.avail_in != 0: - unused = [self.bzs.next_in[i] for i in range(self.bzs.avail_in)] - self.unused_data = "".join(unused) - self.running = False - break - if bzerror != BZ_OK: - _catch_bz2_error(self.space, bzerror) + self.bzs.c_next_in = in_buf + self.bzs.c_avail_in = rffi.cast(rffi.UINT, in_bufsize) + self.bzs.c_next_out = out_buf + self.bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize) + + temp = [] + while True: + bzerror = BZ2_bzDecompress(self.bzs) + if bzerror == BZ_STREAM_END: + if self.bzs.c_avail_in != 0: + unused = [self.bzs.c_next_in[i] for i in range(self.bzs.c_avail_in)] + self.unused_data = "".join(unused) + self.running = False + break + if bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) - if self.bzs.avail_in == 0: - break - elif self.bzs.avail_out == 0: + if self.bzs.c_avail_in == 0: + break + elif self.bzs.c_avail_out == 0: + total_out = _bzs_total_out(self.bzs) + data = "".join([out_buf[i] for i in range(total_out)]) + temp.append(data) + + lltype.free(out_buf, flavor='raw') + out_bufsize = _new_buffer_size(out_bufsize) + out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw') + self.bzs.c_next_out = out_buf + self.bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize) + + if temp: total_out = _bzs_total_out(self.bzs) - data = "".join([out_buf[i] for i in range(total_out)]) + data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))]) temp.append(data) - - out_bufsize = _new_buffer_size(out_bufsize) - out_buf = create_string_buffer(out_bufsize) - self.bzs.next_out = cast(out_buf, POINTER(c_char)) - self.bzs.avail_out = out_bufsize - - if temp: + return self.space.wrap("".join(temp)) + total_out = _bzs_total_out(self.bzs) - data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))]) - temp.append(data) - return self.space.wrap("".join(temp)) + res = "".join([out_buf[i] for i in range(total_out) if out_buf[i] != '\x00']) + return self.space.wrap(res) + finally: + lltype.free(in_buf, flavor='raw') + lltype.free(out_buf, flavor='raw') - total_out = _bzs_total_out(self.bzs) - res = "".join([out_buf[i] for i in range(total_out) if out_buf[i] != '\x00']) - return self.space.wrap(res) decompress.unwrap_spec = ['self', str] @@ -553,57 +580,65 @@ raise OperationError(space.w_ValueError, space.wrap("compresslevel must be between 1 and 9")) - bzs = bz_stream() - + bzs = lltype.malloc(bz_stream.TO, flavor='raw', zero=True) in_bufsize = len(data) # conforming to bz2 manual, this is large enough to fit compressed - # data in one shot. We will check it later anyway. + # data in one shot. We will check it later anyway. out_bufsize = in_bufsize + (in_bufsize / 100 + 1) + 600 - out_buf = create_string_buffer(out_bufsize) - in_buf = create_string_buffer(in_bufsize) - in_buf.value = data - - bzs.next_in = cast(in_buf, POINTER(c_char)) - bzs.avail_in = in_bufsize - bzs.next_out = cast(out_buf, POINTER(c_char)) - bzs.avail_out = out_bufsize - - bzerror = libbz2.BZ2_bzCompressInit(byref(bzs), compresslevel, 0, 0) - if bzerror != BZ_OK: - _catch_bz2_error(space, bzerror) - - total_out = _bzs_total_out(bzs) - temp = [] - while True: - bzerror = libbz2.BZ2_bzCompress(byref(bzs), BZ_FINISH) - if bzerror == BZ_STREAM_END: - break - elif bzerror != BZ_FINISH_OK: - libbz2.BZ2_bzCompressEnd(byref(bzs)) + out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw', + zero=True) + in_buf = lltype.malloc(rffi.CCHARP.TO, in_bufsize, flavor='raw') + for i in range(in_bufsize): + in_buf[i] = data[i] + + try: + bzs.c_next_in = in_buf + bzs.c_avail_in = rffi.cast(rffi.UINT, in_bufsize) + bzs.c_next_out = out_buf + bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize) + + bzerror = BZ2_bzCompressInit(bzs, compresslevel, 0, 0) + if bzerror != BZ_OK: _catch_bz2_error(space, bzerror) - - if bzs.avail_out == 0: - data = "".join([out_buf[i] for i in range(_bzs_total_out(bzs))]) - temp.append(data) - - out_bufsize = _new_buffer_size(out_bufsize) - out_buf = create_string_buffer(out_bufsize) - bzs.next_out = cast(out_buf, POINTER(c_char)) - bzs.avail_out = out_bufsize - - if temp: - res = "".join(temp) - - if bzs.avail_out: - size = _bzs_total_out(bzs) - total_out - res = "".join([out_buf[i] for i in range(size)]) - else: + total_out = _bzs_total_out(bzs) - res = "".join([out_buf[i] for i in range(total_out)]) + temp = [] + while True: + bzerror = BZ2_bzCompress(bzs, BZ_FINISH) + if bzerror == BZ_STREAM_END: + break + elif bzerror != BZ_FINISH_OK: + BZ2_bzCompressEnd(bzs) + _catch_bz2_error(space, bzerror) + + if bzs.c_avail_out == 0: + data = "".join([out_buf[i] for i in range(_bzs_total_out(bzs))]) + temp.append(data) + + lltype.free(out_buf, flavor='raw') + out_bufsize = _new_buffer_size(out_bufsize) + out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, + flavor='raw', zero=True) + bzs.c_next_out = out_buf + bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize) - libbz2.BZ2_bzCompressEnd(byref(bzs)) - return space.wrap(res) + if temp: + res = "".join(temp) + + if bzs.c_avail_out: + size = _bzs_total_out(bzs) - total_out + res = "".join([out_buf[i] for i in range(size)]) + else: + total_out = _bzs_total_out(bzs) + res = "".join([out_buf[i] for i in range(total_out)]) + + BZ2_bzCompressEnd(bzs) + return space.wrap(res) + finally: + lltype.free(bzs, flavor='raw') + lltype.free(in_buf, flavor='raw') + lltype.free(out_buf, flavor='raw') compress.unwrap_spec = [ObjSpace, str, int] def decompress(space, data): @@ -615,55 +650,64 @@ in_bufsize = len(data) if in_bufsize == 0: return space.wrap("") - - bzs = bz_stream() - - in_buf = create_string_buffer(in_bufsize) - in_buf.value = data + + bzs = lltype.malloc(bz_stream.TO, flavor='raw', zero=True) + in_buf = lltype.malloc(rffi.CCHARP.TO, in_bufsize, flavor='raw') + for i in range(in_bufsize): + in_buf[i] = data[i] out_bufsize = SMALLCHUNK - out_buf = create_string_buffer(out_bufsize) + out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw', + zero=True) + try: + - bzs.next_in = cast(in_buf, POINTER(c_char)) - bzs.avail_in = in_bufsize - bzs.next_out = cast(out_buf, POINTER(c_char)) - bzs.avail_out = out_bufsize - - bzerror = libbz2.BZ2_bzDecompressInit(byref(bzs), 0, 0) - if bzerror != BZ_OK: - _catch_bz2_error(space, bzerror) - - temp = [] - while True: - bzerror = libbz2.BZ2_bzDecompress(byref(bzs)) - if bzerror == BZ_STREAM_END: - break + bzs.c_next_in = in_buf + bzs.c_avail_in = rffi.cast(rffi.UINT, in_bufsize) + bzs.c_next_out = out_buf + bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize) + + bzerror = BZ2_bzDecompressInit(bzs, 0, 0) if bzerror != BZ_OK: - libbz2.BZ2_bzDecompressEnd(byref(bzs)) _catch_bz2_error(space, bzerror) - if bzs.avail_in == 0: - libbz2.BZ2_bzDecompressEnd(byref(bzs)) - raise OperationError(space.w_ValueError, - space.wrap("couldn't find end of stream")) - elif bzs.avail_out == 0: - total_out = _bzs_total_out(bzs) - data = "".join([out_buf[i] for i in range(total_out)]) + temp = [] + while True: + bzerror = BZ2_bzDecompress(bzs) + if bzerror == BZ_STREAM_END: + break + if bzerror != BZ_OK: + BZ2_bzDecompressEnd(bzs) + _catch_bz2_error(space, bzerror) + + if bzs.c_avail_in == 0: + BZ2_bzDecompressEnd(bzs) + raise OperationError(space.w_ValueError, + space.wrap("couldn't find end of stream")) + elif bzs.c_avail_out == 0: + total_out = _bzs_total_out(bzs) + data = "".join([out_buf[i] for i in range(total_out)]) + temp.append(data) + + lltype.free(out_buf, flavor='raw') + out_bufsize = _new_buffer_size(out_bufsize) + out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, + flavor='raw', zero=True) + bzs.c_next_out = out_buf + bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize) + + total_out = _bzs_total_out(bzs) + if temp: + data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))]) temp.append(data) - - out_bufsize = _new_buffer_size(out_bufsize) - out_buf = create_string_buffer(out_bufsize) - bzs.next_out = cast(out_buf, POINTER(c_char)) - bzs.avail_out = out_bufsize - - total_out = _bzs_total_out(bzs) - if temp: - data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))]) - temp.append(data) - res = "".join(temp) - else: - res = "".join([out_buf[i] for i in range(total_out) if out_buf[i] != '\x00']) + res = "".join(temp) + else: + res = "".join([out_buf[i] for i in range(total_out) if out_buf[i] != '\x00']) - libbz2.BZ2_bzDecompressEnd(byref(bzs)) - return space.wrap(res) + BZ2_bzDecompressEnd(bzs) + return space.wrap(res) + finally: + lltype.free(bzs, flavor='raw') + lltype.free(out_buf, flavor='raw') + lltype.free(in_buf, flavor='raw') decompress.unwrap_spec = [ObjSpace, str] Added: pypy/dist/pypy/module/bz2/test/__init__.py ============================================================================== Added: pypy/dist/pypy/module/bz2/test/support.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/bz2/test/support.py Fri Oct 5 11:15:32 2007 @@ -0,0 +1,8 @@ +class CheckAllocation: + def teardown_method(self, fun): + from pypy.rpython.lltypesystem import ll2ctypes + import gc + gc.collect() + gc.collect() + gc.collect() # to make sure we disallocate buffers + assert not ll2ctypes.ALLOCATED Modified: pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py Fri Oct 5 11:15:32 2007 @@ -1,4 +1,5 @@ from pypy.conftest import gettestobjspace +from pypy.module.bz2.test.support import CheckAllocation import os if os.name == "nt": @@ -28,8 +29,7 @@ if os.path.exists("foo"): os.unlink("foo") - -class AppTestBZ2Compressor: +class AppTestBZ2Compressor(CheckAllocation): def setup_class(cls): space = gettestobjspace(usemodules=('bz2',)) cls.space = space @@ -46,7 +46,7 @@ BZ2Compressor(9) def test_compress(self): - from bz2 import BZ2Compressor + from bz2 import BZ2Compressor bz2c = BZ2Compressor() raises(TypeError, bz2c.compress) @@ -79,7 +79,7 @@ data = "%s%s" % (data, bz2c.flush()) assert self.decompress(data) == self.TEXT -class AppTestBZ2Decompressor: +class AppTestBZ2Decompressor(CheckAllocation): def setup_class(cls): space = gettestobjspace(usemodules=('bz2',)) cls.space = space @@ -139,7 +139,7 @@ bz2d.decompress(self.DATA) raises(EOFError, bz2d.decompress, "foo") -class AppTestBZ2ModuleFunctions: +class AppTestBZ2ModuleFunctions(CheckAllocation): def setup_class(cls): space = gettestobjspace(usemodules=('bz2',)) cls.space = space Modified: pypy/dist/pypy/module/bz2/test/test_bz2_file.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2_file.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2_file.py Fri Oct 5 11:15:32 2007 @@ -1,5 +1,6 @@ import py from pypy.conftest import gettestobjspace +from pypy.module.bz2.test.support import CheckAllocation import os if os.name == "nt": @@ -34,7 +35,9 @@ mod.create_temp_file = create_temp_file mod.decompress = decompress -class AppTestBZ2File: +class AppTestBZ2File: #(CheckAllocation): + # XXX for unknown reasons, we cannot do allocation checks, as sth is + # keeping those objects alive (BZ2File objects) def setup_class(cls): space = gettestobjspace(usemodules=('bz2',)) cls.space = space @@ -92,6 +95,7 @@ bz2f = BZ2File(self.temppath, mode='w') pos = bz2f.tell() + bz2f.close() assert pos == 0 def test_seek(self): From fijal at codespeak.net Fri Oct 5 11:18:03 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 5 Oct 2007 11:18:03 +0200 (CEST) Subject: [pypy-svn] r47182 - pypy/dist/pypy/doc Message-ID: <20071005091803.2C497816B@code0.codespeak.net> Author: fijal Date: Fri Oct 5 11:18:02 2007 New Revision: 47182 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: update cleanup Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Fri Oct 5 11:18:02 2007 @@ -35,12 +35,13 @@ - rewrite the following rctypes modules using rffi: - - bz2 - _demo - _ssl - - fcntl - mmap - - readline - - signal + + - move the following from using old-style C-api + + - signal module + - strtod (RPyton level) - rewrite or kill the extcompiler From fijal at codespeak.net Fri Oct 5 11:21:01 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 5 Oct 2007 11:21:01 +0200 (CEST) Subject: [pypy-svn] r47183 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20071005092101.A0495816C@code0.codespeak.net> Author: fijal Date: Fri Oct 5 11:21:00 2007 New Revision: 47183 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Use lib[func_name] instead of lib.func_name to be safe regarding double function gets (ie different signatures), needed for fcntl Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Fri Oct 5 11:21:00 2007 @@ -512,6 +512,14 @@ # ____________________________________________ def get_ctypes_callable(funcptr, calling_conv): + def get_on_lib(lib, elem): + """ Wrapper to always use lib[func] instead of lib.func + """ + try: + return lib[elem] + except AttributeError: + pass + if getattr(funcptr._obj, 'sources', None): # give up - for tests with an inlined bit of C code raise NotImplementedError("cannot call a C function defined in " @@ -520,10 +528,10 @@ funcname = funcptr._obj._name libraries = getattr(funcptr._obj, 'libraries', None) if not libraries: - cfunc = getattr(standard_c_lib, funcname, None) + cfunc = get_on_lib(standard_c_lib, funcname) # XXX magic: on Windows try to load the function from 'kernel32' too if cfunc is None and hasattr(ctypes, 'windll'): - cfunc = getattr(ctypes.windll.kernel32, funcname, None) + cfunc = get_on_lib(ctypes.windll.kernel32, funcname) else: cfunc = None for libname in libraries: @@ -533,7 +541,7 @@ if libpath: dllclass = getattr(ctypes, calling_conv + 'dll') clib = dllclass.LoadLibrary(libpath) - cfunc = getattr(clib, funcname, None) + cfunc = get_on_lib(clib, funcname) if cfunc is not None: break Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Fri Oct 5 11:21:00 2007 @@ -665,3 +665,13 @@ assert res == pycrypt("pass", "ab") rffi.free_charp(r) assert not ALLOCATED + + def test_different_signatures(self): + fcntl_int = rffi.llexternal('fcntl', [rffi.INT, rffi.INT, rffi.INT], + rffi.INT) + fcntl_str = rffi.llexternal('fcntl', [rffi.INT, rffi.INT, rffi.CCHARP], + rffi.INT) + fcntl_int(12345, 1, 0) + fcntl_str(12345, 3, "xxx") + fcntl_int(12345, 1, 0) + From arigo at codespeak.net Fri Oct 5 11:40:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 11:40:34 +0200 (CEST) Subject: [pypy-svn] r47184 - pypy/branch/rdict-index-based2/pypy/rpython/lltypesystem Message-ID: <20071005094034.81DDC818F@code0.codespeak.net> Author: arigo Date: Fri Oct 5 11:40:34 2007 New Revision: 47184 Modified: pypy/branch/rdict-index-based2/pypy/rpython/lltypesystem/rdict.py Log: Probably a typo during the conversion - potentially very bad :-( Modified: pypy/branch/rdict-index-based2/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/branch/rdict-index-based2/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/branch/rdict-index-based2/pypy/rpython/lltypesystem/rdict.py Fri Oct 5 11:40:34 2007 @@ -452,7 +452,7 @@ # the dict contains no deleted entries. This routine has the advantage # of never calling d.keyhash() and d.keyeq(), so it cannot call back # to user code. ll_dict_insertclean() doesn't resize the dict, either. - i = ll_dict_lookup(d, key, hash) + i = ll_dict_lookup_clean(d, hash) entry = d.entries[i] DICT = lltype.typeOf(d).TO entry.value = value From arigo at codespeak.net Fri Oct 5 12:54:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 12:54:23 +0200 (CEST) Subject: [pypy-svn] r47185 - in pypy/branch/kill-keepalives-again/pypy/rpython: lltypesystem test Message-ID: <20071005105423.ECA3F8120@code0.codespeak.net> Author: arigo Date: Fri Oct 5 12:54:22 2007 New Revision: 47185 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/rdict.py pypy/branch/kill-keepalives-again/pypy/rpython/test/test_rdict.py Log: Another version of mwh's rdict refactoring. This one moves the ADT methods of the ENTRY to the GcArray of entries instead of to the parent DICT. This allows the diff to be slightly smaller - in particular we don't need the "# AAAAAAAAA XXX :-(" in dict_resize(). Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/rdict.py Fri Oct 5 12:54:22 2007 @@ -4,7 +4,7 @@ from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\ rtype_newdict, dum_variant, dum_keys, dum_values, dum_items from pypy.rpython.lltypesystem import lltype -from pypy.rlib.rarithmetic import r_uint +from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.objectmodel import hlinvoke from pypy.rpython import robject from pypy.rlib import objectmodel @@ -158,9 +158,9 @@ entrymeths['fasthashfn'] = fasthashfn # Build the lltype data structures - self.DICTENTRY = lltype.Struct("dictentry", adtmeths=entrymeths, - *entryfields) - self.DICTENTRYARRAY = lltype.GcArray(self.DICTENTRY) + self.DICTENTRY = lltype.Struct("dictentry", *entryfields) + self.DICTENTRYARRAY = lltype.GcArray(self.DICTENTRY, + adtmeths=entrymeths) fields = [ ("num_items", lltype.Signed), ("num_pristine_entries", lltype.Signed), ("entries", lltype.Ptr(self.DICTENTRYARRAY)) ] @@ -350,47 +350,47 @@ # be direct_call'ed from rtyped flow graphs, which means that they will # get flowed and annotated, mostly with SomePtr. -def ll_everused_from_flag(entry): - return entry.f_everused +def ll_everused_from_flag(entries, i): + return entries[i].f_everused -def ll_everused_from_key(entry): - return bool(entry.key) +def ll_everused_from_key(entries, i): + return bool(entries[i].key) -def ll_everused_from_value(entry): - return bool(entry.value) +def ll_everused_from_value(entries, i): + return bool(entries[i].value) -def ll_valid_from_flag(entry): - return entry.f_valid - -def ll_mark_deleted_in_flag(entry): - entry.f_valid = False - -def ll_valid_from_key(entry): - ENTRY = lltype.typeOf(entry).TO - dummy = ENTRY.dummy_obj.ll_dummy_value - return entry.everused() and entry.key != dummy - -def ll_mark_deleted_in_key(entry): - ENTRY = lltype.typeOf(entry).TO - dummy = ENTRY.dummy_obj.ll_dummy_value - entry.key = dummy - -def ll_valid_from_value(entry): - ENTRY = lltype.typeOf(entry).TO - dummy = ENTRY.dummy_obj.ll_dummy_value - return entry.everused() and entry.value != dummy - -def ll_mark_deleted_in_value(entry): - ENTRY = lltype.typeOf(entry).TO - dummy = ENTRY.dummy_obj.ll_dummy_value - entry.value = dummy - -def ll_hash_from_cache(entry): - return entry.f_hash - -def ll_hash_recomputed(entry): - ENTRY = lltype.typeOf(entry).TO - return ENTRY.fasthashfn(entry.key) +def ll_valid_from_flag(entries, i): + return entries[i].f_valid + +def ll_mark_deleted_in_flag(entries, i): + entries[i].f_valid = False + +def ll_valid_from_key(entries, i): + ENTRIES = lltype.typeOf(entries).TO + dummy = ENTRIES.dummy_obj.ll_dummy_value + return entries.everused(i) and entries[i].key != dummy + +def ll_mark_deleted_in_key(entries, i): + ENTRIES = lltype.typeOf(entries).TO + dummy = ENTRIES.dummy_obj.ll_dummy_value + entries[i].key = dummy + +def ll_valid_from_value(entries, i): + ENTRIES = lltype.typeOf(entries).TO + dummy = ENTRIES.dummy_obj.ll_dummy_value + return entries.everused(i) and entries[i].value != dummy + +def ll_mark_deleted_in_value(entries, i): + ENTRIES = lltype.typeOf(entries).TO + dummy = ENTRIES.dummy_obj.ll_dummy_value + entries[i].value = dummy + +def ll_hash_from_cache(entries, i): + return entries[i].f_hash + +def ll_hash_recomputed(entries, i): + ENTRIES = lltype.typeOf(entries).TO + return ENTRIES.fasthashfn(entries[i].key) def ll_keyhash_custom(d, key): DICT = lltype.typeOf(d).TO @@ -408,9 +408,10 @@ return bool(d) and d.num_items != 0 def ll_dict_getitem(d, key): - entry = ll_dict_lookup(d, key, d.keyhash(key)) - if entry.valid(): - return entry.value + i = ll_dict_lookup(d, key, d.keyhash(key)) + entries = d.entries + if entries.valid(i): + return entries[i].value else: raise KeyError ll_dict_getitem.oopspec = 'dict.getitem(d, key)' @@ -418,11 +419,12 @@ def ll_dict_setitem(d, key, value): hash = d.keyhash(key) - entry = ll_dict_lookup(d, key, hash) - everused = entry.everused() - valid = entry.valid() + i = ll_dict_lookup(d, key, hash) + everused = d.entries.everused(i) + valid = d.entries.valid(i) # set up the new entry - ENTRY = lltype.typeOf(entry).TO + ENTRY = lltype.typeOf(d.entries).TO.OF + entry = d.entries[i] entry.value = value if valid: return @@ -443,8 +445,9 @@ # the dict contains no deleted entries. This routine has the advantage # of never calling d.keyhash() and d.keyeq(), so it cannot call back # to user code. ll_dict_insertclean() doesn't resize the dict, either. - entry = ll_dict_lookup_clean(d, hash) - ENTRY = lltype.typeOf(entry).TO + i = ll_dict_lookup_clean(d, hash) + ENTRY = lltype.typeOf(d.entries).TO.OF + entry = d.entries[i] entry.value = value entry.key = key if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash @@ -454,19 +457,21 @@ d.num_pristine_entries -= 1 def ll_dict_delitem(d, key): - entry = ll_dict_lookup(d, key, d.keyhash(key)) - if not entry.valid(): + i = ll_dict_lookup(d, key, d.keyhash(key)) + if not d.entries.valid(i): raise KeyError - entry.mark_deleted() + d.entries.mark_deleted(i) d.num_items -= 1 # clear the key and the value if they are GC pointers - ENTRY = lltype.typeOf(entry).TO - if ENTRY.must_clear_key: + ENTRIES = lltype.typeOf(d.entries).TO + ENTRY = ENTRIES.OF + entry = d.entries[i] + if ENTRIES.must_clear_key: key = entry.key # careful about destructor side effects: # keep key alive until entry.value has also # been zeroed (if it must be) entry.key = lltype.nullptr(ENTRY.key.TO) - if ENTRY.must_clear_value: + if ENTRIES.must_clear_value: entry.value = lltype.nullptr(ENTRY.value.TO) num_entries = len(d.entries) if num_entries > DICT_INITSIZE and d.num_items < num_entries / 4: @@ -485,9 +490,10 @@ d.num_pristine_entries = new_size i = 0 while i < old_size: - entry = old_entries[i] - if entry.valid(): - ll_dict_insertclean(d, entry.key, entry.value, entry.hash()) + if old_entries.valid(i): + hash = old_entries.hash(i) + entry = old_entries[i] + ll_dict_insertclean(d, entry.key, entry.value, hash) i += 1 # ------- a port of CPython's dictobject.c's lookdict implementation ------- @@ -497,56 +503,61 @@ DICT = lltype.typeOf(d).TO entries = d.entries mask = len(entries) - 1 - i = r_uint(hash & mask) + i = hash & mask # do the first try before any looping - entry = entries[i] - if entry.valid(): - checkingkey = entry.key + if entries.valid(i): + checkingkey = entries[i].key if checkingkey == key: - return entry # found the entry - if d.keyeq is not None and entry.hash() == hash: + return i # found the entry + if d.keyeq is not None and entries.hash(i) == hash: # correct hash, maybe the key is e.g. a different pointer to # an equal object found = d.keyeq(checkingkey, key) if DICT.paranoia: if (entries != d.entries or - not entry.valid() or entry.key != checkingkey): + not entries.valid(i) or entries[i].key != checkingkey): # the compare did major nasty stuff to the dict: start over return ll_dict_lookup(d, key, hash) if found: - return entry # found the entry - freeslot = lltype.nullptr(lltype.typeOf(entry).TO) - elif entry.everused(): - freeslot = entry + return i # found the entry + freeslot = -1 + elif entries.everused(i): + freeslot = i else: - return entry # pristine entry -- lookup failed + return i # pristine entry -- lookup failed # In the loop, a deleted entry (everused and not valid) is by far # (factor of 100s) the least likely outcome, so test for that last. perturb = r_uint(hash) while 1: - i = ((i << 2) + i + perturb + 1) & mask - entry = entries[i] - if not entry.everused(): - return freeslot or entry - elif entry.valid(): - checkingkey = entry.key + # compute the next index using unsigned arithmetic + i = r_uint(i) + i = (i << 2) + i + perturb + 1 + i = intmask(i) & mask + # keep 'i' as a signed number here, to consistently pass signed + # arguments to the small helper methods. + if not entries.everused(i): + if freeslot == -1: + freeslot = i + return freeslot + elif entries.valid(i): + checkingkey = entries[i].key if checkingkey == key: - return entry - if d.keyeq is not None and entry.hash() == hash: + return i + if d.keyeq is not None and entries.hash(i) == hash: # correct hash, maybe the key is e.g. a different pointer to # an equal object found = d.keyeq(checkingkey, key) if DICT.paranoia: if (entries != d.entries or - not entry.valid() or entry.key != checkingkey): + not entries.valid(i) or entries[i].key != checkingkey): # the compare did major nasty stuff to the dict: # start over return ll_dict_lookup(d, key, hash) if found: - return entry # found the entry - elif not freeslot: - freeslot = entry + return i # found the entry + elif freeslot == -1: + freeslot = i perturb >>= PERTURB_SHIFT def ll_dict_lookup_clean(d, hash): @@ -555,14 +566,14 @@ # It only find the next free slot for the given hash. entries = d.entries mask = len(entries) - 1 - i = r_uint(hash & mask) - entry = entries[i] + i = hash & mask perturb = r_uint(hash) - while entry.everused(): - i = ((i << 2) + i + perturb + 1) & mask - entry = entries[i] + while entries.everused(i): + i = r_uint(i) + i = (i << 2) + i + perturb + 1 + i = intmask(i) & mask perturb >>= PERTURB_SHIFT - return entry + return i # ____________________________________________________________ # @@ -638,8 +649,9 @@ entries_len = len(entries) while index < entries_len: entry = entries[index] + is_valid = entries.valid(index) index = index + 1 - if entry.valid(): + if is_valid: iter.index = index if RETURNTYPE is lltype.Void: return None @@ -660,16 +672,18 @@ # methods def ll_get(dict, key, default): - entry = ll_dict_lookup(dict, key, dict.keyhash(key)) - if entry.valid(): - return entry.value + i = ll_dict_lookup(dict, key, dict.keyhash(key)) + entries = dict.entries + if entries.valid(i): + return entries[i].value else: return default def ll_setdefault(dict, key, default): - entry = ll_dict_lookup(dict, key, dict.keyhash(key)) - if entry.valid(): - return entry.value + i = ll_dict_lookup(dict, key, dict.keyhash(key)) + entries = dict.entries + if entries.valid(i): + return entries[i].value else: ll_dict_setitem(dict, key, default) return default @@ -687,7 +701,7 @@ while i < dictsize: d_entry = d.entries[i] entry = dict.entries[i] - ENTRY = lltype.typeOf(entry).TO + ENTRY = lltype.typeOf(d.entries).TO.OF d_entry.key = entry.key if hasattr(ENTRY, 'f_valid'): d_entry.f_valid = entry.f_valid if hasattr(ENTRY, 'f_everused'): d_entry.f_everused = entry.f_everused @@ -709,8 +723,8 @@ d2len = len(entries) i = 0 while i < d2len: - entry = entries[i] - if entry.valid(): + if entries.valid(i): + entry = entries[i] ll_dict_setitem(dic1, entry.key, entry.value) i += 1 @@ -733,10 +747,10 @@ i = 0 p = 0 while i < dlen: - entry = entries[i] - if entry.valid(): + if entries.valid(i): ELEM = lltype.typeOf(items).TO.OF if ELEM is not lltype.Void: + entry = entries[i] if func is dum_items: r = lltype.malloc(ELEM.TO) r.item0 = recast(ELEM.TO.item0, entry.key) @@ -751,7 +765,7 @@ return res def ll_contains(d, key): - entry = ll_dict_lookup(d, key, d.keyhash(key)) - return entry.valid() + i = ll_dict_lookup(d, key, d.keyhash(key)) + return d.entries.valid(i) ll_contains.oopspec = 'dict.contains(d, key)' ll_contains.oopargcheck = lambda d, key: bool(d) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/test/test_rdict.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/test/test_rdict.py Fri Oct 5 12:54:22 2007 @@ -548,7 +548,7 @@ res = self.interpret(func2, [ord(x), ord(y)]) for i in range(len(res.entries)): - assert not (res.entries[i].everused() and not res.entries[i].valid()) + assert not (res.entries.everused(i) and not res.entries.valid(i)) def func3(c0, c1, c2, c3, c4, c5, c6, c7): d = {} @@ -568,7 +568,7 @@ for i in range(rdict.DICT_INITSIZE)]) count_frees = 0 for i in range(len(res.entries)): - if not res.entries[i].everused(): + if not res.entries.everused(i): count_frees += 1 assert count_frees >= 3 From arigo at codespeak.net Fri Oct 5 15:11:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 15:11:15 +0200 (CEST) Subject: [pypy-svn] r47186 - pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem Message-ID: <20071005131115.1131A8158@code0.codespeak.net> Author: arigo Date: Fri Oct 5 15:11:14 2007 New Revision: 47186 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py Log: Fix for addr + FieldOffset(T, 'name_of_inlined_substruct'). Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py Fri Oct 5 15:11:14 2007 @@ -114,7 +114,8 @@ struct = lltype.cast_pointer(lltype.Ptr(self.TYPE), struct) FIELD = getattr(self.TYPE, self.fldname) if isinstance(FIELD, lltype.ContainerType): - return getattr(struct, self.fldname) + substruct = struct._obj._getattr(self.fldname) + return substruct._as_ptr() else: return lltype.direct_fieldptr(struct, self.fldname) From arigo at codespeak.net Fri Oct 5 15:17:10 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 15:17:10 +0200 (CEST) Subject: [pypy-svn] r47187 - in pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem: . test Message-ID: <20071005131710.258C5817F@code0.codespeak.net> Author: arigo Date: Fri Oct 5 15:17:08 2007 New Revision: 47187 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llmemory.py Log: A fix in the test, and another fix in the source. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py Fri Oct 5 15:17:08 2007 @@ -195,7 +195,9 @@ def ref(self, arrayptr): assert lltype.typeOf(arrayptr).TO == self.TYPE if isinstance(self.TYPE.OF, lltype.ContainerType): - return arrayptr[0] + # XXX this doesn't support empty arrays + o = arrayptr._obj.getitem(0) + return o._as_ptr() else: return lltype.direct_arrayitems(arrayptr) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llmemory.py Fri Oct 5 15:17:08 2007 @@ -98,7 +98,8 @@ for a in [lltype.malloc(lltype.GcArray(S), 5), lltype.malloc(lltype.FixedSizeArray(S, 5), immortal=True)]: a[3].x = 42 - adr_s = cast_ptr_to_adr(a[0]) + adr_s = cast_ptr_to_adr(a) + adr_s += itemoffsetof(lltype.typeOf(a).TO, 0) adr_s += sizeof(S) * 3 s = cast_adr_to_ptr(adr_s, lltype.Ptr(S)) assert s.x == 42 From arigo at codespeak.net Fri Oct 5 15:21:41 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 15:21:41 +0200 (CEST) Subject: [pypy-svn] r47188 - pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem Message-ID: <20071005132141.403C88181@code0.codespeak.net> Author: arigo Date: Fri Oct 5 15:21:40 2007 New Revision: 47188 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py Log: Remove this function - too tempting to use :-) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py Fri Oct 5 15:21:40 2007 @@ -695,12 +695,6 @@ return val.lltype() raise TypeError("typeOf(%r object)" % (tp.__name__,)) -def rawTypeOf(val): - if isinstance(val, _interior_ptr): - return Ptr(val._T) - else: - return typeOf(val) - _to_primitive = { Char: chr, UniChar: unichr, From arigo at codespeak.net Fri Oct 5 15:27:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 15:27:52 +0200 (CEST) Subject: [pypy-svn] r47189 - pypy/branch/kill-keepalives-again/pypy/rpython/memory/test Message-ID: <20071005132752.99C5D80CE@code0.codespeak.net> Author: arigo Date: Fri Oct 5 15:27:50 2007 New Revision: 47189 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_convertlltype.py Log: Unsure about this test. Seems unrelated to GcArray-vs-Array. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_convertlltype.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_convertlltype.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_convertlltype.py Fri Oct 5 15:27:50 2007 @@ -128,6 +128,6 @@ def test_convertsubstructure_of_array(): cvter = LLTypeConverter(lladdress.raw_malloc(100)) S1 = lltype.Struct("s1", ("v1", lltype.Signed)) - A = lltype.GcArray(S1) - lla = lltype.malloc(A, 3) + A = lltype.Array(S1) + lla = lltype.malloc(A, 3, immortal=True) s1 = cvter.convert(lla[0]) From arigo at codespeak.net Fri Oct 5 15:31:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 15:31:50 +0200 (CEST) Subject: [pypy-svn] r47190 - pypy/branch/kill-keepalives-again/pypy/rpython/rctypes Message-ID: <20071005133150.247FE8147@code0.codespeak.net> Author: arigo Date: Fri Oct 5 15:31:49 2007 New Revision: 47190 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py Log: Fix reccopy(), which is used by llmemory.raw_memcopy() (it should really be moved to llmemory). Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py Fri Oct 5 15:31:49 2007 @@ -272,8 +272,8 @@ ITEMTYPE = T.OF for i in range(source._obj.getlength()): if isinstance(ITEMTYPE, lltype.ContainerType): - subsrc = source[i] - subdst = dest[i] + subsrc = source._obj._getitem(i)._as_ptr() + subdst = dest._obj._getitem(i)._as_ptr() reccopy(subsrc, subdst) else: # this is a hack XXX de-hack this @@ -283,8 +283,8 @@ for name in T._names: FIELDTYPE = getattr(T, name) if isinstance(FIELDTYPE, lltype.ContainerType): - subsrc = getattr(source, name) - subdst = getattr(dest, name) + subsrc = source._obj._getattr(name)._as_ptr() + subdst = dest._obj._getattr(name)._as_ptr() reccopy(subsrc, subdst) else: # this is a hack XXX de-hack this From cfbolz at codespeak.net Fri Oct 5 17:24:34 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 5 Oct 2007 17:24:34 +0200 (CEST) Subject: [pypy-svn] r47193 - pypy/branch/kill-keepalives-again/pypy/translator/c Message-ID: <20071005152434.8D72F815D@code0.codespeak.net> Author: cfbolz Date: Fri Oct 5 17:24:33 2007 New Revision: 47193 Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/funcgen.py pypy/branch/kill-keepalives-again/pypy/translator/c/node.py Log: attempt at supporting the new operations by genc Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/funcgen.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/funcgen.py Fri Oct 5 17:24:33 2007 @@ -425,9 +425,9 @@ return result def generic_set(self, op, targetexpr): - newvalue = self.expr(op.args[2], special_case_void=False) + newvalue = self.expr(op.args[-1], special_case_void=False) result = ['%s = %s;' % (targetexpr, newvalue)] - T = self.lltypemap(op.args[2]) + T = self.lltypemap(op.args[-1]) result = '\n'.join(result) if T is Void: result = '/* %s */' % result @@ -490,6 +490,50 @@ return '%s = &%s;' % (self.expr(op.result), arraydef.itemindex_access_expr(ptr, index)) + def interior_expr(self, args, rettype=False): + TYPE = args[0].concretetype.TO + expr = self.expr(args[0]) + for i, arg in enumerate(args[1:]): + defnode = self.db.gettypedefnode(TYPE) + if arg.concretetype is Void: + fieldname = arg.value + if i == 0: + expr = defnode.ptr_access_expr(expr, fieldname) + else: + expr = defnode.access_expr(expr, fieldname) + if isinstance(TYPE, FixedSizeArray): + TYPE = TYPE.OF + else: + TYPE = getattr(TYPE, fieldname) + else: + indexexpr = self.expr(arg) + if i == 0: + expr = defnode.itemindex_access_expr(expr, indexexpr) + else: + expr = defnode.access_expr_varindex(expr, indexexpr) + TYPE = TYPE.OF + if rettype: + return expr, TYPE + else: + return expr + + def OP_GETINTERIORFIELD(self, op): + return self.generic_get(op, self.interior_expr(op.args)) + + def OP_BARE_SETINTERIORFIELD(self, op): + return self.generic_set(op, self.interior_expr(op.args[:-1])) + OP_SETINTERIORFIELD = OP_BARE_SETINTERIORFIELD + + def OP_GETINTERIORARRAYSIZE(self, op): + expr, ARRAY = self.interior_expr(op.args, True) + if isinstance(ARRAY, FixedSizeArray): + return '%s = %d;'%(self.expr(op.result), ARRAY.length) + else: + assert isinstance(ARRAY, Array) + return '%s = %s.length;'%(self.expr(op.result), expr) + + + def OP_PTR_NONZERO(self, op): return '%s = (%s != NULL);' % (self.expr(op.result), self.expr(op.args[0])) Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/node.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/node.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/node.py Fri Oct 5 17:24:33 2007 @@ -241,7 +241,8 @@ return '%s %s *@' % (self.typetag, self.name) def access_expr(self, baseexpr, index): - return '%s.items[%d]' % (baseexpr, index) + return '%s.items[%s]' % (baseexpr, index) + access_expr_varindex = access_expr def ptr_access_expr(self, baseexpr, index): assert 0 <= index <= sys.maxint, "invalid constant index %r" % (index,) @@ -334,6 +335,7 @@ def access_expr(self, baseexpr, index): return '%s[%d]' % (baseexpr, index) + access_expr_varindex = access_expr def ptr_access_expr(self, baseexpr, index): assert 0 <= index <= sys.maxint, "invalid constant index %r" % (index,) @@ -389,6 +391,9 @@ ptr_access_expr = access_expr + def access_expr_varindex(self, baseexpr, index): + return '%s[%s]' % (baseexpr, index) + def itemindex_access_expr(self, baseexpr, indexexpr): return 'RPyFxItem(%s, %s, %d)' % (baseexpr, indexexpr, self.FIXEDARRAY.length) From cfbolz at codespeak.net Fri Oct 5 18:10:21 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 5 Oct 2007 18:10:21 +0200 (CEST) Subject: [pypy-svn] r47194 - pypy/branch/kill-keepalives-again/pypy/translator/c/test Message-ID: <20071005161021.355BB80D0@code0.codespeak.net> Author: cfbolz Date: Fri Oct 5 18:10:19 2007 New Revision: 47194 Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lltyped.py Log: add some of the tests michael wrote on the old kill-keepalives branch. some of them are broken, investigate later. Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lltyped.py Fri Oct 5 18:10:19 2007 @@ -54,9 +54,9 @@ def test_recursivearray(self): A = ForwardReference() A.become(FixedSizeArray(Struct("S", ('a', Ptr(A))), 5)) - TREE = GcStruct("TREE", ("root", A), ("other", A)) + TREE = Struct("TREE", ("root", A), ("other", A)) + tree = malloc(TREE, immortal=True) def llf(): - tree = malloc(TREE) tree.root[0].a = tree.root tree.root[1].a = tree.other assert tree.root[0].a[0].a[0].a[0].a[0].a[1].a == tree.other @@ -96,8 +96,8 @@ def test_more_prebuilt_arrays(self): A = FixedSizeArray(Struct('s1', ('x', Signed)), 5) - S = GcStruct('s', ('a1', Ptr(A)), ('a2', A)) - s = malloc(S, zero=True) + S = Struct('s', ('a1', Ptr(A)), ('a2', A)) + s = malloc(S, zero=True, immortal=True) s.a1 = malloc(A, immortal=True) s.a1[2].x = 50 s.a2[2].x = 60 @@ -164,6 +164,7 @@ assert res == 0 + 10 + 30 + 1000 def test_structarray_add(self): + py.test.skip("XXX fix this!") from pypy.rpython.lltypesystem import llmemory S = Struct("S", ("x", Signed)) PS = Ptr(S) @@ -619,3 +620,55 @@ res = fn(-5) assert res != 0.4 # precision lost assert abs(res - 0.4) < 1E-6 + + + def test_array_of_array(self): + C = FixedSizeArray(Signed, 7) + B = Array(C) + A = FixedSizeArray(C, 6) + b = malloc(B, 5, immortal=True) + b[3][4] = 999 + a = malloc(A, immortal=True) + a[2][5] = 888000 + def llf(): + return b[3][4] + a[2][5] + fn = self.getcompiled(llf) + assert fn() == 888999 + + def test_prebuilt_nolength_array(self): + A = Array(Signed, hints={'nolength': True}) + a = malloc(A, 5, immortal=True) + a[0] = 8 + a[1] = 5 + a[2] = 12 + a[3] = 12 + a[4] = 15 + def llf(): + s = '' + for i in range(5): + s += chr(64+a[i]) + assert s == "HELLO" + fn = self.getcompiled(llf) + fn() + + def test_prebuilt_nolength_char_array(self): + py.test.skip("XXX fix this!") + for lastchar in ('\x00', 'X'): + A = Array(Char, hints={'nolength': True}) + a = malloc(A, 5, immortal=True) + a[0] = '8' + a[1] = '5' + a[2] = '?' + a[3] = '!' + a[4] = lastchar + def llf(): + s = '' + for i in range(5): + print i + print s + s += a[i] + print s + assert s == "85?!" + lastchar + fn = self.getcompiled(llf) + fn() + From arigo at codespeak.net Fri Oct 5 18:41:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 18:41:35 +0200 (CEST) Subject: [pypy-svn] r47195 - pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform Message-ID: <20071005164135.6E8CD8154@code0.codespeak.net> Author: arigo Date: Fri Oct 5 18:41:34 2007 New Revision: 47195 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/transform.py Log: This operation probably belongs to this list. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/transform.py Fri Oct 5 18:41:34 2007 @@ -51,7 +51,8 @@ gct.livevars.append(v_result) if var_ispyobj(v_result): if opname in ('getfield', 'getarrayitem', 'same_as', - 'cast_pointer', 'getsubstruct'): + 'cast_pointer', 'getsubstruct', + 'getinteriorfield'): # XXX more operations? gct.push_alive(v_result) elif opname not in ('direct_call', 'indirect_call'): From arigo at codespeak.net Fri Oct 5 18:42:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 18:42:43 +0200 (CEST) Subject: [pypy-svn] r47196 - pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform Message-ID: <20071005164243.C45848158@code0.codespeak.net> Author: arigo Date: Fri Oct 5 18:42:43 2007 New Revision: 47196 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/framework.py Log: Check in this, intermediatedly, because it needs to be fixed at some point. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/framework.py Fri Oct 5 18:42:43 2007 @@ -696,6 +696,7 @@ llops.genop('bare_setfield', [v, c_name, c_null]) elif isinstance(FIELD, lltype.Struct): c_name = rmodel.inputconst(lltype.Void, name) + XXX # should use 'setinteriorfield' v1 = llops.genop('getsubstruct', [v, c_name], resulttype = lltype.Ptr(FIELD)) gen_zero_gc_pointers(FIELD, v1, llops) From arigo at codespeak.net Fri Oct 5 18:53:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 18:53:22 +0200 (CEST) Subject: [pypy-svn] r47197 - pypy/branch/kill-keepalives-again/pypy/rpython Message-ID: <20071005165322.A33408154@code0.codespeak.net> Author: arigo Date: Fri Oct 5 18:53:22 2007 New Revision: 47197 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rptr.py Log: Don't need to generate a same_as explicitly. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rptr.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/rptr.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/rptr.py Fri Oct 5 18:53:22 2007 @@ -45,8 +45,7 @@ return hop.genop('cast_pointer', [hop.inputarg(self, 0)], resulttype=hop.r_result.lowleveltype) elif isinstance(hop.r_result, InteriorPtrRepr): - return hop.genop('same_as', [hop.inputarg(self, 0)], - resulttype=self.lowleveltype) + return hop.inputarg(self, 0) else: newopname = 'getsubstruct' else: @@ -276,8 +275,7 @@ return hop.inputarg(hop.r_result, arg=0) FIELD_TYPE = getattr(self.resulttype.TO, attr) if isinstance(FIELD_TYPE, lltype.ContainerType): - return hop.genop('same_as', [hop.inputarg(self, 0)], - resulttype=self.lowleveltype) + return hop.inputarg(self, 0) else: v_self, v_attr = hop.inputargs(self, lltype.Void) vlist = self.getinteriorfieldargs(hop, v_self) + [v_attr] From arigo at codespeak.net Fri Oct 5 19:05:30 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 19:05:30 +0200 (CEST) Subject: [pypy-svn] r47198 - pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test Message-ID: <20071005170530.2F50F813F@code0.codespeak.net> Author: arigo Date: Fri Oct 5 19:05:29 2007 New Revision: 47198 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test/test_transform.py Log: Arguments to the entry point function of GC tests cannot be GC objects themselves, because nobody would put a proper header on them. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test/test_transform.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test/test_transform.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test/test_transform.py Fri Oct 5 19:05:29 2007 @@ -22,6 +22,11 @@ cbuild = CStandaloneBuilder(t, f, t.config, gcpolicy=self.gcpolicy) db = cbuild.generate_graphs_for_llinterp() graph = cbuild.getentrypointptr()._obj.graph + # arguments cannot be GC objects because nobody would put a + # proper header on them + for v in graph.getargs(): + if isinstance(v.concretetype, lltype.Ptr): + assert v.concretetype.TO._gckind != 'gc', "fix the test!" llinterp = LLInterpreter(t.rtyper) if conftest.option.view: t.view() From cfbolz at codespeak.net Fri Oct 5 19:10:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 5 Oct 2007 19:10:23 +0200 (CEST) Subject: [pypy-svn] r47199 - pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test Message-ID: <20071005171023.0F9198154@code0.codespeak.net> Author: cfbolz Date: Fri Oct 5 19:10:21 2007 New Revision: 47199 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test/test_transform.py Log: fix the test to not have gced arguments any more Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test/test_transform.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test/test_transform.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test/test_transform.py Fri Oct 5 19:10:21 2007 @@ -76,21 +76,22 @@ assert res == f(10) def test_str(self): - from pypy.annotation.model import SomeString - from pypy.rpython.lltypesystem.rstr import string_repr + from pypy.annotation.model import SomeBool - def f(x): + def f(flag): + if flag: + x = 'a' + else: + x = 'brrrrrrr' return len(x + 'a') - llinterp, graph = self.llinterpreter_for_transformed_graph(f, [SomeString()]) + llinterp, graph = self.llinterpreter_for_transformed_graph(f, [SomeBool()]) - cc = string_repr.convert_const - - res = llinterp.eval_graph(graph, [cc('a')]) - assert res == f('a') - res = llinterp.eval_graph(graph, [cc('brrrrrr')]) - assert res == f('brrrrrr') + res = llinterp.eval_graph(graph, [True]) + assert res == f(True) + res = llinterp.eval_graph(graph, [False]) + assert res == f(False) class _TestGCTransformer(BaseGCTransformer): From cfbolz at codespeak.net Fri Oct 5 20:44:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 5 Oct 2007 20:44:55 +0200 (CEST) Subject: [pypy-svn] r47200 - in pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform: . test Message-ID: <20071005184455.4B1B580F8@code0.codespeak.net> Author: cfbolz Date: Fri Oct 5 20:44:53 2007 New Revision: 47200 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test/test_refcounting.py Log: fix the failing refcounting test: perform a malloc removal on the deallocators, since they could contain malloc through accessing internal arrays. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/refcounting.py Fri Oct 5 20:44:53 2007 @@ -120,6 +120,19 @@ self.dynamic_deallocator_funcptrs = {} self.queryptr2dynamic_deallocator_funcptr = {} + def finish_helpers(self): + GCTransformer.finish_helpers(self) + from pypy.translator.backendopt.malloc import remove_mallocs + seen = {} + graphs = [] + for fptr in self.static_deallocator_funcptrs.itervalues(): + graph = fptr._obj.graph + if graph in seen: + continue + seen[graph] = True + graphs.append(graph) + remove_mallocs(self.translator, graphs) + def var_needs_set_transform(self, var): return var_needsgc(var) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test/test_refcounting.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test/test_refcounting.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/test/test_refcounting.py Fri Oct 5 20:44:53 2007 @@ -169,8 +169,9 @@ dgraph, t = make_deallocator(S) ops = getops(dgraph) assert len(ops['direct_call']) == 4 - assert len(ops['getfield']) == 4 - assert len(ops['getarraysubstruct']) == 1 + assert len(ops['getfield']) == 2 + assert len(ops['getinteriorfield']) == 2 + assert len(ops['getinteriorarraysize']) == 1 assert len(ops['gc_free']) == 1 def test_deallocator_with_destructor(): From cfbolz at codespeak.net Fri Oct 5 20:58:29 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 5 Oct 2007 20:58:29 +0200 (CEST) Subject: [pypy-svn] r47201 - pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform Message-ID: <20071005185829.C00328103@code0.codespeak.net> Author: cfbolz Date: Fri Oct 5 20:58:29 2007 New Revision: 47201 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/framework.py Log: resolve this XXX by correctly generating bare_setinteriorfields Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/framework.py Fri Oct 5 20:58:29 2007 @@ -686,20 +686,22 @@ return llmemory.offsetof(WEAKREF, "weakptr") return -1 -def gen_zero_gc_pointers(TYPE, v, llops): +def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None): + if previous_steps is None: + previous_steps = [] assert isinstance(TYPE, lltype.Struct) for name in TYPE._names: + c_name = rmodel.inputconst(lltype.Void, name) FIELD = getattr(TYPE, name) if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): - c_name = rmodel.inputconst(lltype.Void, name) c_null = rmodel.inputconst(FIELD, lltype.nullptr(FIELD.TO)) - llops.genop('bare_setfield', [v, c_name, c_null]) + if not previous_steps: + llops.genop('bare_setfield', [v, c_name, c_null]) + else: + llops.genop('bare_setinteriorfield', + [v] + previous_steps + [c_name, c_null]) elif isinstance(FIELD, lltype.Struct): - c_name = rmodel.inputconst(lltype.Void, name) - XXX # should use 'setinteriorfield' - v1 = llops.genop('getsubstruct', [v, c_name], - resulttype = lltype.Ptr(FIELD)) - gen_zero_gc_pointers(FIELD, v1, llops) + gen_zero_gc_pointers(FIELD, v, llops, previous_steps + [c_name]) def gc_pointers_inside(v, adr): t = lltype.typeOf(v) From arigo at codespeak.net Fri Oct 5 21:28:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 21:28:20 +0200 (CEST) Subject: [pypy-svn] r47203 - in pypy/branch/kill-keepalives-again/pypy/rpython: memory/test rctypes Message-ID: <20071005192820.C8C5B8119@code0.codespeak.net> Author: arigo Date: Fri Oct 5 21:28:19 2007 New Revision: 47203 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py Log: * the interior pointer tests, ported from test_rptr and grouped into one to avoid taking too much time * fix for a typo found by them Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py Fri Oct 5 21:28:19 2007 @@ -160,7 +160,6 @@ from pypy.rpython.memory.gctransform import framework from pypy.rpython.memory.gctransform import stacklessframework from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rpython.lltypesystem import lltype from pypy.rpython.memory.support import INT_SIZE from pypy.rpython.memory.gc import X_CLONE, X_POOL, X_POOL_PTR from pypy import conftest @@ -681,6 +680,64 @@ res = run([3, 0]) assert res == 1 + def test_interior_ptrs(self): + from pypy.rpython.lltypesystem.lltype import Struct, GcStruct, GcArray + from pypy.rpython.lltypesystem.lltype import Array, Signed, malloc + + S1 = Struct("S1", ('x', Signed)) + T1 = GcStruct("T1", ('s', S1)) + def f1(): + t = malloc(T1) + t.s.x = 1 + return t.s.x + + S2 = Struct("S2", ('x', Signed)) + T2 = GcArray(S2) + def f2(): + t = malloc(T2, 1) + t[0].x = 1 + return t[0].x + + S3 = Struct("S3", ('x', Signed)) + T3 = GcStruct("T3", ('items', Array(S3))) + def f3(): + t = malloc(T3, 1) + t.items[0].x = 1 + return t.items[0].x + + S4 = Struct("S4", ('x', Signed)) + T4 = Struct("T4", ('s', S4)) + U4 = GcArray(T4) + def f4(): + u = malloc(U4, 1) + u[0].s.x = 1 + return u[0].s.x + + S5 = Struct("S5", ('x', Signed)) + T5 = GcStruct("T5", ('items', Array(S5))) + def f5(): + t = malloc(T5, 1) + return len(t.items) + + T6 = GcStruct("T6", ('s', Array(Signed))) + def f6(): + t = malloc(T6, 1) + t.s[0] = 1 + return t.s[0] + + def func(): + return (f1() * 100000 + + f2() * 10000 + + f3() * 1000 + + f4() * 100 + + f5() * 10 + + f6()) + + assert func() == 111111 + run = self.runner(func) + res = run([]) + assert res == 111111 + class TestStacklessMarkSweepGC(TestMarkSweepGC): Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py Fri Oct 5 21:28:19 2007 @@ -272,8 +272,8 @@ ITEMTYPE = T.OF for i in range(source._obj.getlength()): if isinstance(ITEMTYPE, lltype.ContainerType): - subsrc = source._obj._getitem(i)._as_ptr() - subdst = dest._obj._getitem(i)._as_ptr() + subsrc = source._obj.getitem(i)._as_ptr() + subdst = dest._obj.getitem(i)._as_ptr() reccopy(subsrc, subdst) else: # this is a hack XXX de-hack this From arigo at codespeak.net Fri Oct 5 22:13:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 22:13:12 +0200 (CEST) Subject: [pypy-svn] r47205 - in pypy/branch/kill-keepalives-again/pypy/rpython: lltypesystem rctypes rctypes/test Message-ID: <20071005201312.3C5A08120@code0.codespeak.net> Author: arigo Date: Fri Oct 5 22:13:09 2007 New Revision: 47205 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rarray.py pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rchar_p.py pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/test/test_rfunc.py Log: Some progress on rctypes. Based on a function unsafe_getfield(ptr, name), which does that a plain getattr(ptr, name) used to do before the introduction of interior pointer objects. In other words, it's grossly unsafe unless we put enough keepalives by hand around it, and has no chance to be safe on a moving GC. Good enough for rctypes. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py Fri Oct 5 22:13:09 2007 @@ -297,6 +297,7 @@ def offsetof(TYPE, fldname): assert fldname in TYPE._flds return FieldOffset(TYPE, fldname) +offsetof._annspecialcase_ = 'specialize:memo' def itemoffsetof(TYPE, n=0): return ArrayItemsOffset(TYPE) + ItemOffset(TYPE.OF) * n Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rarray.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rarray.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rarray.py Fri Oct 5 22:13:09 2007 @@ -7,11 +7,13 @@ from pypy.annotation.pairtype import pairtype from pypy.rpython.rctypes.rmodel import CTypesRefRepr, CTypesValueRepr from pypy.rpython.rctypes.rmodel import genreccopy_arrayitem, reccopy, C_ZERO +from pypy.rpython.rctypes.rmodel import unsafe_getfield from pypy.rpython.rctypes.rprimitive import PrimitiveRepr from pypy.rpython.rctypes.rpointer import PointerRepr from pypy.rpython.rctypes.aarray import VarSizedArrayType from pypy.annotation.model import SomeCTypesObject from pypy.objspace.flow.model import Constant +from pypy.rlib.objectmodel import keepalive_until_here ArrayType = type(ARRAY(c_int, 10)) @@ -171,12 +173,13 @@ def ll_chararrayvalue(box): from pypy.rpython.rctypes import rchar_p - p = box.c_data + p = unsafe_getfield(box, 'c_data') length = rchar_p.ll_strnlen(lltype.direct_arrayitems(p), len(p)) newstr = lltype.malloc(string_repr.lowleveltype.TO, length) newstr.hash = 0 for i in range(length): newstr.chars[i] = p[i] + keepalive_until_here(box) return newstr def ll_chararrayslice(box, slice): Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rchar_p.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rchar_p.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rchar_p.py Fri Oct 5 22:13:09 2007 @@ -3,6 +3,7 @@ from pypy.rpython.rstr import AbstractStringRepr from pypy.rpython.lltypesystem.rstr import string_repr from pypy.rpython.rctypes.rmodel import CTypesValueRepr, C_ZERO +from pypy.rpython.rctypes.rmodel import unsafe_getfield from pypy.rpython.rctypes.rarray import ArrayRepr from pypy.rpython.rctypes.rstringbuf import StringBufRepr from pypy.annotation.pairtype import pairtype @@ -116,7 +117,7 @@ return i def ll_str2charp(s): - return lltype.direct_arrayitems(s.chars) + return lltype.direct_arrayitems(unsafe_getfield(s, 'chars')) def ll_charp2str(p): if not p: Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py Fri Oct 5 22:13:09 2007 @@ -93,21 +93,15 @@ else: # we must return a non-memory-owning box that keeps the # memory-owning box alive + pdata = unsafe_getfield(p, 'c_data') result = lltype.malloc(self.lowleveltype.TO, zero=True) - result.c_data = p.c_data # initialize c_data pointer + result.c_data = pdata # initialize c_data pointer result.c_data_owner_keepalive = p self.const_cache[key] = result, keepalive return result def get_c_data(self, llops, v_box): - if self.ownsmemory: - inputargs = [v_box, inputconst(lltype.Void, "c_data")] - return llops.genop('getsubstruct', inputargs, - lltype.Ptr(self.c_data_type) ) - else: - inputargs = [v_box, inputconst(lltype.Void, "c_data")] - return llops.genop('getfield', inputargs, - lltype.Ptr(self.c_data_type) ) + return gen_unsafe_getfield(llops, v_box, 'c_data') def get_c_data_owner(self, llops, v_box): if self.ownsmemory: @@ -162,19 +156,10 @@ return self.allocate_instance_ref(llops, v_c_data) def getkeepalive(self, llops, v_box): - try: - TYPE = self.lowleveltype.TO.keepalive - except AttributeError: - return None + if hasattr(self.lowleveltype.TO, 'keepalive'): + return gen_unsafe_getfield(llops, v_box, 'keepalive') else: - if isinstance(TYPE, lltype.ContainerType): - TYPE = lltype.Ptr(TYPE) - opname = 'getsubstruct' - else: - opname = 'getfield' - c_name = inputconst(lltype.Void, 'keepalive') - return llops.genop(opname, [v_box, c_name], - resulttype = TYPE) + return None class __extend__(pairtype(CTypesRepr, CTypesRepr)): @@ -263,6 +248,39 @@ C_ZERO = inputconst(lltype.Signed, 0) +def unsafe_getfield(ptr, name): + """Equivalent of getattr(ptr, name), but if the 'name' belongs to an + inlined substructure of a GC pointer, it returns a raw pointer to it. + This is dangerous! You *have* to keep the 'ptr' alive as long as you + use the result - put a call to keepalive_until_here().""" + STRUCT = lltype.typeOf(ptr).TO + FIELD = getattr(STRUCT, name) + if isinstance(FIELD, lltype.ContainerType): + addr = llmemory.cast_ptr_to_adr(ptr) + addr += llmemory.offsetof(STRUCT, name) + return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(FIELD)) + else: + return getattr(ptr, name) +unsafe_getfield._annspecialcase_ = 'specialize:ll_and_arg(1)' + +def gen_unsafe_getfield(llops, v_ptr, name): + """Generate lloperations equivalent to 'unsafe_getfield(ptr, name)'.""" + STRUCT = v_ptr.concretetype.TO + assert isinstance(STRUCT, lltype.Struct) + FIELD = getattr(STRUCT, name) + if isinstance(FIELD, lltype.ContainerType): + v_addr = llops.genop('cast_ptr_to_adr', [v_ptr], + resulttype = llmemory.Address) + c_ofs = inputconst(lltype.Signed, llmemory.offsetof(STRUCT, name)) + v_addr = llops.genop('adr_add', [v_addr, c_ofs], + resulttype = llmemory.Address) + return llops.genop('cast_adr_to_ptr', [v_addr], + resulttype = lltype.Ptr(FIELD)) + else: + c_name = inputconst(lltype.Void, name) + return llops.genop('getfield', [v_ptr, c_name], + resulttype = FIELD) + def reccopy(source, dest): # copy recursively a structure or array onto another. T = lltype.typeOf(source).TO Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/test/test_rfunc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/test/test_rfunc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/test/test_rfunc.py Fri Oct 5 22:13:09 2007 @@ -5,6 +5,7 @@ import py import sys import pypy.rpython.rctypes.implementation +from pypy.rpython.rctypes.rmodel import unsafe_getfield from pypy.annotation import model as annmodel from pypy.annotation.annrpython import RPythonAnnotator from pypy.translator.translator import TranslationContext, graphof @@ -109,7 +110,7 @@ def str2subarray(string): llstring = string_repr.convert_const(string) keepalive.append(llstring) - return lltype.direct_arrayitems(llstring.chars) + return lltype.direct_arrayitems(unsafe_getfield(llstring, 'chars')) assert ll_atoi(str2subarray("")) == 0 assert ll_atoi(str2subarray("42z7")) == 42 assert ll_atoi(str2subarray("blah")) == 0 From arigo at codespeak.net Fri Oct 5 22:21:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 22:21:23 +0200 (CEST) Subject: [pypy-svn] r47206 - pypy/branch/kill-keepalives-again/pypy/rpython/rctypes Message-ID: <20071005202123.EA46880FE@code0.codespeak.net> Author: arigo Date: Fri Oct 5 22:21:23 2007 New Revision: 47206 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rvoid_p.py Log: More tweaks in rctypes. I think that tests pass now. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py Fri Oct 5 22:21:23 2007 @@ -374,11 +374,10 @@ llops.genop('setarrayitem', [v_destarray, v_destindex, v_source]) def genreccopy_structfield(llops, v_source, v_deststruct, fieldname): - c_name = inputconst(lltype.Void, fieldname) FIELDTYPE = getattr(v_deststruct.concretetype.TO, fieldname) if isinstance(FIELDTYPE, lltype.ContainerType): - v_dest = llops.genop('getsubstruct', [v_deststruct, c_name], - resulttype = lltype.Ptr(FIELDTYPE)) + v_dest = gen_unsafe_getfield(llops, v_deststruct, fieldname) genreccopy(llops, v_source, v_dest) else: + c_name = inputconst(lltype.Void, fieldname) llops.genop('setfield', [v_deststruct, c_name, v_source]) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rvoid_p.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rvoid_p.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rvoid_p.py Fri Oct 5 22:21:23 2007 @@ -1,4 +1,5 @@ from pypy.rpython.rctypes.rmodel import CTypesValueRepr, C_ZERO +from pypy.rpython.rctypes.rmodel import unsafe_getfield from pypy.rpython.rctypes.rstringbuf import StringBufRepr from pypy.annotation.pairtype import pairtype from pypy.rpython.rstr import AbstractStringRepr @@ -54,7 +55,7 @@ def ll_string2addr(s): if s: - ptr = lltype.direct_arrayitems(s.chars) + ptr = lltype.direct_arrayitems(unsafe_getfield(s, 'chars')) return llmemory.cast_ptr_to_adr(ptr) else: return llmemory.NULL From arigo at codespeak.net Fri Oct 5 22:35:58 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 22:35:58 +0200 (CEST) Subject: [pypy-svn] r47207 - pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/test Message-ID: <20071005203558.092C78102@code0.codespeak.net> Author: arigo Date: Fri Oct 5 22:35:58 2007 New Revision: 47207 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/test/test_rarray.py Log: Check in a failing test (skipped). It also fails in the trunk actually, so I think I'll just forget about it. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/test/test_rarray.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/test/test_rarray.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/test/test_rarray.py Fri Oct 5 22:35:58 2007 @@ -344,6 +344,24 @@ res = interpret(func, [10]) assert res == 50 + def test_specialize_array_of_struct(self): + py.test.skip("known to fail, sorry :-(") + class T(Structure): + _fields_ = [('x', c_int)] + class S(Structure): + _fields_ = [('p', POINTER(T))] + A = S * 10 + def func(): + a = A() + for i in range(10): + t = T() + t.x = i*i + a[i].p = pointer(t) + for i in range(10): + assert a[i].p.contents.x == i*i + func() + interpret(func, []) + class Test_compilation: def setup_class(self): if not test_c_compile: From arigo at codespeak.net Fri Oct 5 22:38:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Oct 2007 22:38:26 +0200 (CEST) Subject: [pypy-svn] r47208 - pypy/dist/pypy/rpython/rctypes/test Message-ID: <20071005203826.DB39280EF@code0.codespeak.net> Author: arigo Date: Fri Oct 5 22:38:26 2007 New Revision: 47208 Modified: pypy/dist/pypy/rpython/rctypes/test/test_rarray.py Log: A known failure - at least on the llinterp our keepalive logic is not sufficient. Unsure if we will ever work more on this, though. Modified: pypy/dist/pypy/rpython/rctypes/test/test_rarray.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_rarray.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_rarray.py Fri Oct 5 22:38:26 2007 @@ -341,6 +341,24 @@ res = interpret(func, [10]) assert res == 50 + def test_specialize_array_of_struct(self): + py.test.skip("known to fail, sorry :-(") + class T(Structure): + _fields_ = [('x', c_int)] + class S(Structure): + _fields_ = [('p', POINTER(T))] + A = S * 10 + def func(): + a = A() + for i in range(10): + t = T() + t.x = i*i + a[i].p = pointer(t) + for i in range(10): + assert a[i].p.contents.x == i*i + func() + interpret(func, []) + class Test_compilation: def setup_class(self): from pypy.translator.c.test.test_genc import compile From cfbolz at codespeak.net Fri Oct 5 22:45:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 5 Oct 2007 22:45:30 +0200 (CEST) Subject: [pypy-svn] r47209 - pypy/branch/kill-keepalives-again/pypy/translator/c/test Message-ID: <20071005204530.8C08C8106@code0.codespeak.net> Author: cfbolz Date: Fri Oct 5 22:45:30 2007 New Revision: 47209 Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_database.py Log: change various tests to not use GcStructs any more but only Structs. The new rules don't allow these access patterns any more, and I think the tests still test what they are written for in their new form. Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_database.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_database.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_database.py Fri Oct 5 22:45:30 2007 @@ -72,10 +72,10 @@ def test_codegen(): db = LowLevelDatabase() U = Struct('inlined', ('z', Signed)) - T = GcStruct('subtest', ('y', Signed)) - S = GcStruct('test', ('x', Ptr(T)), ('u', U), ('p', Ptr(U))) - s = malloc(S) - s.x = malloc(T) + T = Struct('subtest', ('y', Signed)) + S = Struct('test', ('x', Ptr(T)), ('u', U), ('p', Ptr(U))) + s = malloc(S, immortal=True) + s.x = malloc(T, immortal=True) s.x.y = 42 s.u.z = -100 s.p = s.u @@ -99,16 +99,16 @@ def test_codegen_3(): db = LowLevelDatabase() - A = GcStruct('varsizedstuff', ('x', Signed), ('y', Array(('i', Signed)))) - S = GcStruct('test', ('aptr', Ptr(A)), - ('anitem', Ptr(A.y.OF)), - ('anarray', Ptr(A.y))) - a = malloc(A, 3) + A = Struct('varsizedstuff', ('x', Signed), ('y', Array(('i', Signed)))) + S = Struct('test', ('aptr', Ptr(A)), + ('anitem', Ptr(A.y.OF)), + ('anarray', Ptr(A.y))) + a = malloc(A, 3, immortal=True) a.x = 99 a.y[0].i = 100 a.y[1].i = 101 a.y[2].i = 102 - s = malloc(S) + s = malloc(S, immortal=True) s.aptr = a s.anitem = a.y[1] s.anarray = a.y From cfbolz at codespeak.net Fri Oct 5 22:49:10 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 5 Oct 2007 22:49:10 +0200 (CEST) Subject: [pypy-svn] r47210 - pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem Message-ID: <20071005204910.332A48106@code0.codespeak.net> Author: cfbolz Date: Fri Oct 5 22:49:09 2007 New Revision: 47210 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py Log: add a helpful comment for the crashes that come from the restrictions to access interior pointers and do certain things with them. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py Fri Oct 5 22:49:09 2007 @@ -693,6 +693,9 @@ return UniChar if issubclass(tp, Symbolic): return val.lltype() + # if you get a TypeError: typeOf('_interior_ptr' object) + # here, it is very likely that you are accessing an interior pointer + # in an illegal way! raise TypeError("typeOf(%r object)" % (tp.__name__,)) _to_primitive = { From arigo at codespeak.net Sat Oct 6 12:30:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 6 Oct 2007 12:30:24 +0200 (CEST) Subject: [pypy-svn] r47230 - in pypy/branch/kill-keepalives-again/pypy: config config/test doc/config module/_stackless module/_stackless/test rlib rpython/memory/test translator/c translator/c/test Message-ID: <20071006103024.290DB80AB@code0.codespeak.net> Author: arigo Date: Sat Oct 6 12:30:23 2007 New Revision: 47230 Added: pypy/branch/kill-keepalives-again/pypy/doc/config/translation.stacklessgc.txt (contents, props changed) Modified: pypy/branch/kill-keepalives-again/pypy/config/test/test_pypyoption.py pypy/branch/kill-keepalives-again/pypy/config/translationoption.py pypy/branch/kill-keepalives-again/pypy/doc/config/translation.gc.txt pypy/branch/kill-keepalives-again/pypy/module/_stackless/__init__.py pypy/branch/kill-keepalives-again/pypy/module/_stackless/test/test_interp_clonable.py pypy/branch/kill-keepalives-again/pypy/rlib/rgc.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/kill-keepalives-again/pypy/translator/c/gc.py pypy/branch/kill-keepalives-again/pypy/translator/c/genc.py pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_boehm.py pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_newgc.py pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_stackless.py Log: Replace the option --gc=stacklessgc with its own flag --stacklessgc. We use --gc=framework in all cases. The goal is to make it easier to add options to choose various framework GCs like Mark&Sweep vs SemiSpace, which can each use either the stacklessgc root finding or not. Modified: pypy/branch/kill-keepalives-again/pypy/config/test/test_pypyoption.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/config/test/test_pypyoption.py (original) +++ pypy/branch/kill-keepalives-again/pypy/config/test/test_pypyoption.py Sat Oct 6 12:30:23 2007 @@ -18,9 +18,10 @@ def test_stacklessgc_required(): conf = get_pypy_config() - conf.translation.gc = "stacklessgc" + conf.translation.stacklessgc = True assert conf.translation.stackless assert conf.translation.type_system == "lltype" + assert conf.translation.gc == "framework" def test_check_documentation(): from pypy.doc.config.confrest import all_optiondescrs Modified: pypy/branch/kill-keepalives-again/pypy/config/translationoption.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/config/translationoption.py (original) +++ pypy/branch/kill-keepalives-again/pypy/config/translationoption.py Sat Oct 6 12:30:23 2007 @@ -41,14 +41,16 @@ default=False, cmdline="--llvm-via-c", requires=[("translation.backend", "llvm")]), ChoiceOption("gc", "Garbage Collection Strategy", - ["boehm", "ref", "framework", "none", "stacklessgc", - "exact_boehm"], + ["boehm", "ref", "framework", "none", "exact_boehm"], "ref", requires={ "ref": [("translation.rweakref", False)], # XXX "none": [("translation.rweakref", False)], # XXX - "stacklessgc": [("translation.stackless", True), - ]}, + }, cmdline="--gc"), + BoolOption("stacklessgc", "Use stackless to find roots in a framework GC", + default=False, cmdline="--stacklessgc", + requires=[("translation.gc", "framework"), + ("translation.stackless", True)]), BoolOption("thread", "enable use of threading primitives", default=False, cmdline="--thread", requires=[("translation.gc", "boehm")]), Modified: pypy/branch/kill-keepalives-again/pypy/doc/config/translation.gc.txt ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/doc/config/translation.gc.txt (original) +++ pypy/branch/kill-keepalives-again/pypy/doc/config/translation.gc.txt Sat Oct 6 12:30:23 2007 @@ -6,10 +6,6 @@ - "framework": our custom mark-and-sweep collector. Takes moderately long and is the fastest option without external dependencies. - - "stacklessgc": same as "framework" but uses a different method to find the - garbage collection roots on the stack, by unwinding it, using stackless: - :config:`translation.stackless`. - - "boehm": use the Boehm conservative GC Added: pypy/branch/kill-keepalives-again/pypy/doc/config/translation.stacklessgc.txt ============================================================================== --- (empty file) +++ pypy/branch/kill-keepalives-again/pypy/doc/config/translation.stacklessgc.txt Sat Oct 6 12:30:23 2007 @@ -0,0 +1,8 @@ +Use stackless to find roots in a framework GC. + +Only for framework GCs. This uses a different method to find the +garbage collection roots on the stack, by unwinding it, using stackless: +:config:`translation.stackless`. + +When stacklessgc is not enabled, the default method - which turns out to +be faster for now - is to maintain a custom root stack. Modified: pypy/branch/kill-keepalives-again/pypy/module/_stackless/__init__.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/module/_stackless/__init__.py (original) +++ pypy/branch/kill-keepalives-again/pypy/module/_stackless/__init__.py Sat Oct 6 12:30:23 2007 @@ -27,7 +27,7 @@ from pypy.module._stackless.interp_greenlet import post_install as post_install_greenlet post_install_greenlet(self) - if self.space.config.translation.gc in ('framework', 'stacklessgc'): + if self.space.config.translation.gc == 'framework': from pypy.module._stackless.interp_clonable import post_install as post_install_clonable self.extra_interpdef('clonable', 'interp_clonable.AppClonableCoroutine') self.extra_interpdef('fork', 'interp_clonable.fork') Modified: pypy/branch/kill-keepalives-again/pypy/module/_stackless/test/test_interp_clonable.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/module/_stackless/test/test_interp_clonable.py (original) +++ pypy/branch/kill-keepalives-again/pypy/module/_stackless/test/test_interp_clonable.py Sat Oct 6 12:30:23 2007 @@ -11,7 +11,8 @@ class TestClonableCoroutine(test_transformed_gc.GCTest): - gcname = "stacklessgc" + gcname = "framework" + stacklessgc = True class gcpolicy(gc.StacklessFrameworkGcPolicy): class transformerclass(stacklessframework.StacklessFrameworkGCTransformer): GC_PARAMS = {'start_heap_size': 4096 } Modified: pypy/branch/kill-keepalives-again/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rlib/rgc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rlib/rgc.py Sat Oct 6 12:30:23 2007 @@ -42,7 +42,7 @@ config = rtyper.getconfig() # if the gc policy doesn't support allocation pools, lltype # pools as Void. - if config.translation.gc not in ['framework', 'stacklessgc']: + if config.translation.gc != 'framework': from pypy.annotation.model import s_None return rtyper.getrepr(s_None) else: @@ -66,7 +66,7 @@ opname = 'gc_x_swap_pool' config = hop.rtyper.getconfig() - if config.translation.gc not in ['framework', 'stacklessgc']: + if config.translation.gc != 'framework': # when the gc policy doesn't support pools, just return # the argument (which is lltyped as Void anyway) opname = 'same_as' @@ -95,7 +95,7 @@ from pypy.rpython.memory.gc import X_CLONE, X_CLONE_PTR config = hop.rtyper.getconfig() - if config.translation.gc not in ['framework', 'stacklessgc']: + if config.translation.gc != 'framework': # if the gc policy does not support allocation pools, # gc_clone always raises RuntimeError hop.exception_is_here() Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py Sat Oct 6 12:30:23 2007 @@ -165,11 +165,12 @@ from pypy import conftest -def rtype(func, inputtypes, specialize=True, gcname='ref'): +def rtype(func, inputtypes, specialize=True, gcname='ref', stacklessgc=False): from pypy.translator.translator import TranslationContext t = TranslationContext() # XXX XXX XXX mess t.config.translation.gc = gcname + t.config.translation.stacklessgc = stacklessgc t.buildannotator().build_types(func, inputtypes) if specialize: t.buildrtyper().specialize() @@ -179,6 +180,7 @@ class GCTest(object): gcpolicy = None + stacklessgc = False def runner(self, f, nbargs=0, statistics=False): if nbargs == 2: @@ -198,14 +200,15 @@ ARGS = lltype.FixedSizeArray(lltype.Signed, nbargs) s_args = annmodel.SomePtr(lltype.Ptr(ARGS)) - t = rtype(entrypoint, [s_args], gcname=self.gcname) + t = rtype(entrypoint, [s_args], gcname=self.gcname, + stacklessgc=self.stacklessgc) cbuild = CStandaloneBuilder(t, entrypoint, config=t.config, gcpolicy=self.gcpolicy) db = cbuild.generate_graphs_for_llinterp() entrypointptr = cbuild.getentrypointptr() entrygraph = entrypointptr._obj.graph if conftest.option.view: - t.view() + t.viewcg() llinterp = LLInterpreter(t.rtyper) @@ -741,7 +744,7 @@ class TestStacklessMarkSweepGC(TestMarkSweepGC): - gcname = "stacklessgc" + stacklessgc = True class gcpolicy(gc.StacklessFrameworkGcPolicy): class transformerclass(stacklessframework.StacklessFrameworkGCTransformer): GC_PARAMS = {'start_heap_size': 4096 } Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/gc.py Sat Oct 6 12:30:23 2007 @@ -389,7 +389,7 @@ 'ref': RefcountingGcPolicy, 'none': NoneGcPolicy, 'framework': FrameworkGcPolicy, - 'stacklessgc': StacklessFrameworkGcPolicy, + 'framework+stacklessgc': StacklessFrameworkGcPolicy, } Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/genc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/genc.py Sat Oct 6 12:30:23 2007 @@ -107,7 +107,10 @@ def get_gcpolicyclass(self): if self.gcpolicy is None: - return gc.name_to_gcpolicy[self.config.translation.gc] + name = self.config.translation.gc + if self.config.translation.stacklessgc: + name = "%s+stacklessgc" % (name,) + return gc.name_to_gcpolicy[name] return self.gcpolicy # use generate_source(defines=DEBUG_DEFINES) to force the #definition Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_boehm.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_boehm.py Sat Oct 6 12:30:23 2007 @@ -11,6 +11,7 @@ class AbstractGCTestClass: gcpolicy = "boehm" + stacklessgc = False # deal with cleanups def setup_method(self, meth): @@ -25,6 +26,7 @@ from pypy.config.pypyoption import get_pypy_config config = get_pypy_config(translating=True) config.translation.gc = self.gcpolicy + config.translation.stacklessgc = self.stacklessgc config.translation.simplifying = True t = TranslationContext(config=config) self.t = t Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_newgc.py Sat Oct 6 12:30:23 2007 @@ -775,14 +775,13 @@ assert res == 42 class TestUsingStacklessFramework(TestUsingFramework): - gcpolicy = "stacklessgc" def getcompiled(self, f): # XXX quick hack from pypy.translator.c.test.test_stackless import StacklessTest runner = StacklessTest() runner.gcpolicy = self.gcpolicy - runner.stacklessmode = True + runner.stacklessgc = True try: res = runner.wrap_stackless_function(f) except py.process.cmdexec.Error, e: Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_stackless.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_stackless.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_stackless.py Sat Oct 6 12:30:23 2007 @@ -11,8 +11,8 @@ class StacklessTest(object): backendopt = False - stacklessmode = True gcpolicy = "boehm" + stacklessgc = False def setup_class(cls): import py @@ -22,7 +22,7 @@ # rpython/extfunctable. Doing so breaks translator/stackless/. import py py.test.skip("stackless + refcounting doesn't work any more for now") - elif cls.gcpolicy is "boehm": + elif cls.gcpolicy == "boehm": from pypy.translator.tool.cbuild import check_boehm_presence if not check_boehm_presence(): py.test.skip("Boehm GC not present") @@ -36,6 +36,7 @@ config = get_pypy_config(translating=True) config.translation.gc = self.gcpolicy config.translation.stackless = True + config.translation.stacklessgc = self.stacklessgc t = TranslationContext(config=config) self.t = t t.buildannotator().build_types(entry_point, [s_list_of_strings]) @@ -47,7 +48,7 @@ insert_ll_stackcheck(t) cbuilder = CStandaloneBuilder(t, entry_point, config=config) - cbuilder.stackless = self.stacklessmode + cbuilder.stackless = True cbuilder.generate_source() cbuilder.compile() res = cbuilder.cmdexec('') From fijal at codespeak.net Sat Oct 6 13:30:23 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Oct 2007 13:30:23 +0200 (CEST) Subject: [pypy-svn] r47232 - pypy/dist/pypy/module/mmap Message-ID: <20071006113023.6A4308116@code0.codespeak.net> Author: fijal Date: Sat Oct 6 13:30:22 2007 New Revision: 47232 Modified: pypy/dist/pypy/module/mmap/interp_mmap.py Log: Move mmap from rctypes to rffi. There is still issue with pointer arithmetic and no windows support Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Sat Oct 6 13:30:22 2007 @@ -1,12 +1,9 @@ -from pypy.rpython.rctypes.tool import ctypes_platform -from pypy.rpython.rctypes.tool.libc import libc -import pypy.rpython.rctypes.implementation # this defines rctypes magic -from pypy.rpython.rctypes.aerrno import geterrno +from pypy.rpython.tool import rffi_platform +from pypy.rpython.lltypesystem import rffi, lltype, llmemory from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app -from ctypes import * import sys import os import platform @@ -17,11 +14,11 @@ _LINUX = "linux" in sys.platform _64BIT = "64bit" in platform.architecture()[0] - class CConfig: - _includes_ = ("sys/types.h",) - size_t = ctypes_platform.SimpleType("size_t", c_long) - off_t = ctypes_platform.SimpleType("off_t", c_long) + _includes_ = ("sys/types.h",'unistd.h') + _header_ = '#define _GNU_SOURCE\n' + size_t = rffi_platform.SimpleType("size_t", rffi.LONG) + off_t = rffi_platform.SimpleType("off_t", rffi.LONG) constants = {} if _POSIX: @@ -36,14 +33,14 @@ 'PROT_EXEC', 'MAP_DENYWRITE', 'MAP_EXECUTABLE'] for name in constant_names: - setattr(CConfig, name, ctypes_platform.ConstantInteger(name)) + setattr(CConfig, name, rffi_platform.ConstantInteger(name)) for name in opt_constant_names: - setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name)) + setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name)) - has_mremap = hasattr(libc, "mremap") - if has_mremap: - CConfig.MREMAP_MAYMOVE = ( - ctypes_platform.DefinedConstantInteger("MREMAP_MAYMOVE")) + CConfig.MREMAP_MAYMOVE = ( + rffi_platform.DefinedConstantInteger("MREMAP_MAYMOVE")) + CConfig.has_mremap = rffi_platform.Has('mremap()') # a dirty hack, this + # is probably a macro elif _MS_WINDOWS: CConfig._includes_ += ("windows.h",) @@ -51,10 +48,11 @@ 'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY', 'DUPLICATE_SAME_ACCESS'] for name in constant_names: - setattr(CConfig, name, ctypes_platform.ConstantInteger(name)) + setattr(CConfig, name, rffi_platform.ConstantInteger(name)) # export the constants inside and outside. see __init__.py -constants.update(ctypes_platform.configure(CConfig)) +cConfig = rffi_platform.configure(CConfig) +constants.update(cConfig) if _POSIX: # MAP_ANONYMOUS is not always present but it's always available at CPython level @@ -64,38 +62,33 @@ constants["MAP_ANON"] = constants["MAP_ANONYMOUS"] locals().update(constants) - _ACCESS_DEFAULT, ACCESS_READ, ACCESS_WRITE, ACCESS_COPY = range(4) -PTR = POINTER(c_char) # cannot use c_void_p as return value of functions :-( +def external(name, args, result): + return rffi.llexternal(name, args, result, includes=CConfig._includes_) + +PTR = rffi.VOIDP # XXX? -memmove_ = libc.memmove -memmove_.argtypes = [PTR, PTR, size_t] +has_mremap = cConfig['has_mremap'] + +c_memmove = external('memmove', [PTR, PTR, size_t], lltype.Void) if _POSIX: - libc.mmap.argtypes = [PTR, size_t, c_int, c_int, c_int, off_t] - libc.mmap.restype = PTR - libc.mmap.includes = ("sys/mman.h",) - libc.munmap.argtypes = [PTR, size_t] - libc.munmap.restype = c_int - libc.munmap.includes = ("sys/mman.h",) - libc.msync.argtypes = [PTR, size_t, c_int] - libc.msync.restype = c_int - libc.msync.includes = ("sys/mman.h",) - + c_mmap = external('mmap', [PTR, size_t, rffi.INT, rffi.INT, + rffi.INT, off_t], PTR) + c_munmap = external('munmap', [PTR, size_t], rffi.INT) + c_msync = external('msync', [PTR, size_t, rffi.INT], rffi.INT) if has_mremap: - libc.mremap.argtypes = [PTR, size_t, size_t, c_ulong] - libc.mremap.restype = PTR - libc.mremap.includes = ("sys/mman.h",) + c_mremap = external('mremap', [PTR, size_t, size_t, rffi.ULONG], PTR) - def _get_page_size(): - return libc.getpagesize() + _get_page_size = external('getpagesize', [], rffi.INT) def _get_error_msg(): - errno = geterrno() + errno = rffi.get_errno() return os.strerror(errno) elif _MS_WINDOWS: + XXX from ctypes import wintypes WORD = wintypes.WORD @@ -105,7 +98,7 @@ LPVOID = PTR LPCVOID = LPVOID DWORD_PTR = DWORD - c_int = wintypes.c_int + rffi.INT = wintypes.rffi.INT INVALID_c_int_VALUE = c_int(-1).value class SYSINFO_STRUCT(Structure): @@ -130,39 +123,39 @@ windll.kernel32.GetSystemInfo.argtypes = [POINTER(SYSTEM_INFO)] GetFileSize = windll.kernel32.GetFileSize - GetFileSize.argtypes = [c_int, POINTER(c_int)] - GetFileSize.restype = c_int + GetFileSize.argtypes = [rffi.INT, POINTER(rffi.INT)] + GetFileSize.restype = rffi.INT GetCurrentProcess = windll.kernel32.GetCurrentProcess - GetCurrentProcess.restype = c_int + GetCurrentProcess.restype = rffi.INT DuplicateHandle = windll.kernel32.DuplicateHandle - DuplicateHandle.argtypes = [c_int, c_int, c_int, POINTER(c_int), DWORD, + DuplicateHandle.argtypes = [rffi.INT, rffi.INT, rffi.INT, POINTER(rffi.INT), DWORD, BOOL, DWORD] DuplicateHandle.restype = BOOL CreateFileMapping = windll.kernel32.CreateFileMappingA - CreateFileMapping.argtypes = [c_int, PTR, c_int, c_int, c_int, + CreateFileMapping.argtypes = [rffi.INT, PTR, rffi.INT, rffi.INT, rffi.INT, c_char_p] - CreateFileMapping.restype = c_int + CreateFileMapping.restype = rffi.INT MapViewOfFile = windll.kernel32.MapViewOfFile - MapViewOfFile.argtypes = [c_int, DWORD, DWORD, DWORD, DWORD] + MapViewOfFile.argtypes = [rffi.INT, DWORD, DWORD, DWORD, DWORD] MapViewOfFile.restype = PTR CloseHandle = windll.kernel32.CloseHandle - CloseHandle.argtypes = [c_int] + CloseHandle.argtypes = [rffi.INT] CloseHandle.restype = BOOL UnmapViewOfFile = windll.kernel32.UnmapViewOfFile UnmapViewOfFile.argtypes = [LPCVOID] UnmapViewOfFile.restype = BOOL FlushViewOfFile = windll.kernel32.FlushViewOfFile - FlushViewOfFile.argtypes = [LPCVOID, c_int] + FlushViewOfFile.argtypes = [LPCVOID, rffi.INT] FlushViewOfFile.restype = BOOL SetFilePointer = windll.kernel32.SetFilePointer - SetFilePointer.argtypes = [c_int, c_int, POINTER(c_int), c_int] + SetFilePointer.argtypes = [rffi.INT, rffi.INT, POINTER(rffi.INT), rffi.INT] SetEndOfFile = windll.kernel32.SetEndOfFile - SetEndOfFile.argtypes = [c_int] + SetEndOfFile.argtypes = [rffi.INT] msvcr71 = cdll.LoadLibrary("msvcr71.dll") - msvcr71._get_osfhandle.argtypes = [c_int] - msvcr71._get_osfhandle.restype = c_int - # libc._lseek.argtypes = [c_int, c_int, c_int] - # libc._lseek.restype = c_int + msvcr71._get_osfhandle.argtypes = [rffi.INT] + msvcr71._get_osfhandle.restype = rffi.INT + # libc._lseek.argtypes = [rffi.INT, rffi.INT, rffi.INT] + # libc._lseek.restype = rffi.INT def _get_page_size(): @@ -172,8 +165,8 @@ def _get_file_size(space, handle): # XXX use native Windows types like WORD - high = c_int(0) - low = c_int(windll.kernel32.GetFileSize(c_int(handle.value), byref(high))) + high = rffi.INT(0) + low = rffi.INT(windll.kernel32.GetFileSize(rffi.INT(handle.value), byref(high))) # low might just happen to have the value INVALID_FILE_SIZE # so we need to check the last error also INVALID_FILE_SIZE = -1 @@ -189,9 +182,9 @@ return os.strerror(errno) PAGESIZE = _get_page_size() -NULL = PTR() -EMPTY_DATA = (c_char * 0)() -NODATA = cast(pointer(EMPTY_DATA), PTR) +NULL = lltype.nullptr(PTR.TO) +NODATA = lltype.nullptr(PTR.TO) +INVALID_INT_VALUE = -1 # ____________________________________________________________ @@ -205,8 +198,8 @@ self.access = access if _MS_WINDOWS: - self.map_handle = wintypes.c_int() - self.file_handle = wintypes.c_int() + self.map_handle = 0 + self.file_handle = 0 self.tagname = "" elif _POSIX: self.fd = -1 @@ -214,7 +207,7 @@ def check_valid(self): if _MS_WINDOWS: - to_close = self.map_handle.value == INVALID_c_int_VALUE + to_close = self.map_handle.value == INVALID_INT_VALUE elif _POSIX: to_close = self.closed @@ -236,8 +229,7 @@ def setdata(self, data, size): """Set the internal data and map size from a PTR.""" assert size >= 0 - arraytype = c_char * size - self.data = cast(data, POINTER(arraytype)).contents + self.data = data self.size = size def close(self): @@ -245,19 +237,19 @@ if self.size > 0: self.unmapview() self.setdata(NODATA, 0) - if self.map_handle.value != INVALID_c_int_VALUE: + if self.map_handle.value != INVALID_rffi.INT_VALUE: CloseHandle(self.map_handle) - self.map_handle.value = INVALID_c_int_VALUE - if self.file_handle.value != INVALID_c_int_VALUE: + self.map_handle.value = INVALID_rffi.INT_VALUE + if self.file_handle.value != INVALID_rffi.INT_VALUE: CloseHandle(self.file_handle) - self.file_handle.value = INVALID_c_int_VALUE + self.file_handle.value = INVALID_rffi.INT_VALUE elif _POSIX: self.closed = True if self.fd != -1: os.close(self.fd) self.fd = -1 if self.size > 0: - libc.munmap(self.getptr(0), self.size) + c_munmap(self.getptr(0), self.size) self.setdata(NODATA, 0) close.unwrap_spec = ['self'] @@ -287,7 +279,7 @@ else: # no '\n' found eol = self.size - res = data[self.pos:eol] + res = "".join([data[i] for i in range(self.pos, eol)]) self.pos += len(res) return self.space.wrap(res) readline.unwrap_spec = ['self'] @@ -304,7 +296,8 @@ if eol > self.size: eol = self.size - res = self.data[self.pos:eol] + res = [self.data[i] for i in range(self.pos, eol)] + res = "".join(res) self.pos += len(res) return self.space.wrap(res) read.unwrap_spec = ['self', int] @@ -363,11 +356,11 @@ size = self.size if _MS_WINDOWS: - if self.file_handle.value != INVALID_c_int_VALUE: + if self.file_handle.value != INVALID_rffi.INT_VALUE: low, high = _get_file_size(self.space, self.file_handle) if not high and low <= sys.maxint: return self.space.wrap(low) - size = c_int((high << 32) + low).value + size = rffi.INT((high << 32) + low).value elif _POSIX: st = os.fstat(self.fd) size = st[stat.ST_SIZE] @@ -409,11 +402,12 @@ def getptr(self, offset): if offset > 0: # XXX 64-bit support for pointer arithmetic! - dataptr = cast(pointer(self.data), c_void_p) - dataptr = c_void_p(dataptr.value + offset) - return cast(dataptr, PTR) + # is this still valid? + dataptr = lltype.cast_int_to_ptr(PTR, lltype.cast_ptr_to_int( + self.data) + offset) + return dataptr else: - return cast(pointer(self.data), PTR) + return self.data def flush(self, offset=0, size=0): self.check_valid() @@ -439,7 +433,7 @@ ## # the size should be increased too. otherwise the final ## # part is not "msynced" ## new_size = size + value & (PAGESIZE - 1) - res = libc.msync(start, size, MS_SYNC) + res = c_msync(start, size, MS_SYNC) if res == -1: raise OperationError(self.space.w_EnvironmentError, self.space.wrap(_get_error_msg())) @@ -460,7 +454,7 @@ datasrc = self.getptr(src) datadest = self.getptr(dest) - memmove_(datadest, datasrc, count) + c_memmove(datadest, datasrc, count) move.unwrap_spec = ['self', int, int, int] def resize(self, newsize): @@ -482,8 +476,8 @@ self.space.wrap(os.strerror(e.errno))) # now resize the mmap - newdata = libc.mremap(self.getptr(0), self.size, newsize, - MREMAP_MAYMOVE or 0) + newdata = c_mremap(self.getptr(0), self.size, newsize, + MREMAP_MAYMOVE or 0) self.setdata(newdata, newsize) elif _MS_WINDOWS: # disconnect the mapping @@ -495,10 +489,10 @@ newsize_high = DWORD(newsize >> 32) newsize_low = DWORD(newsize & 0xFFFFFFFF) else: - newsize_high = c_int(0) - newsize_low = c_int(newsize) + newsize_high = rffi.INT(0) + newsize_low = rffi.INT(newsize) - FILE_BEGIN = c_int(0) + FILE_BEGIN = rffi.INT(0) SetFilePointer(self.file_handle, newsize_low, byref(newsize_high), FILE_BEGIN) # resize the file @@ -506,7 +500,7 @@ # create another mapping object and remap the file view res = CreateFileMapping(self.file_handle, NULL, PAGE_READWRITE, newsize_high, newsize_low, self.tagname) - self.map_handle = c_int(res) + self.map_handle = rffi.INT(res) dwErrCode = DWORD(0) if self.map_handle: @@ -539,7 +533,7 @@ return space.wrap(self.data[start]) elif step == 1: if 0 <= start <= stop: - res = self.data[start:stop] + res = "".join([self.data[i] for i in range(start, stop)]) else: res = '' return space.wrap(res) @@ -601,7 +595,7 @@ if size < 0: raise OperationError(space.w_TypeError, space.wrap("memory mapped size must be positive")) - if size_t(size).value != size: + if rffi.cast(size_t, size) != size: raise OperationError(space.w_OverflowError, space.wrap("memory mapped size is too large (limited by C int)")) @@ -671,8 +665,8 @@ raise OperationError(space.w_EnvironmentError, space.wrap(os.strerror(e.errno))) - res = libc.mmap(NULL, map_size, prot, flags, fd, 0) - if cast(res, c_void_p).value == -1: + res = c_mmap(NULL, map_size, prot, flags, fd, 0) + if lltype.cast_ptr_to_int(res) == -1: raise OperationError(space.w_EnvironmentError, space.wrap(_get_error_msg())) @@ -716,8 +710,8 @@ m = W_MMap(space, access) # XXX the following two attributes should be plain RPython ints - m.file_handle = c_int(INVALID_c_int_VALUE) - m.map_handle = c_int(INVALID_c_int_VALUE) + m.file_handle = rffi.INT(INVALID_rffi.INT_VALUE) + m.map_handle = rffi.INT(INVALID_rffi.INT_VALUE) if fh: # it is necessary to duplicate the handle, so the @@ -734,9 +728,9 @@ space.wrap(_get_error_msg())) if not map_size: - low, high = _get_file_size(space, c_int(fh)) + low, high = _get_file_size(space, rffi.INT(fh)) if _64BIT: - map_size = c_int((low << 32) + 1).value + map_size = rffi.INT((low << 32) + 1).value else: if high: # file is too large to map completely @@ -752,10 +746,10 @@ size_hi = DWORD(map_size >> 32) size_lo = DWORD(map_size & 0xFFFFFFFF) else: - size_hi = c_int(0) - size_lo = c_int(map_size) + size_hi = rffi.INT(0) + size_lo = rffi.INT(map_size) - m.map_handle = c_int(CreateFileMapping(m.file_handle, NULL, flProtect, + m.map_handle = rffi.INT(CreateFileMapping(m.file_handle, NULL, flProtect, size_hi, size_lo, m.tagname)) if m.map_handle: From fijal at codespeak.net Sat Oct 6 13:32:59 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Oct 2007 13:32:59 +0200 (CEST) Subject: [pypy-svn] r47233 - pypy/dist/pypy/doc Message-ID: <20071006113259.963D98116@code0.codespeak.net> Author: fijal Date: Sat Oct 6 13:32:59 2007 New Revision: 47233 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: Delete mmap. Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Sat Oct 6 13:32:59 2007 @@ -37,7 +37,6 @@ - _demo - _ssl - - mmap - move the following from using old-style C-api From fijal at codespeak.net Sat Oct 6 13:38:45 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Oct 2007 13:38:45 +0200 (CEST) Subject: [pypy-svn] r47236 - pypy/dist/pypy/config Message-ID: <20071006113845.B1AE48116@code0.codespeak.net> Author: fijal Date: Sat Oct 6 13:38:45 2007 New Revision: 47236 Modified: pypy/dist/pypy/config/pypyoption.py Log: Add bz2 to allworkingmodules. Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Sat Oct 6 13:38:45 2007 @@ -24,7 +24,7 @@ working_modules.update(dict.fromkeys( ["_socket", "unicodedata", "mmap", "fcntl", "rctime", "select", "crypt", "signal", "dyngram", "readline", "termios", "zlib", - "struct", "md5", "sha", + "struct", "md5", "sha", "bz2" ] )) From fijal at codespeak.net Sat Oct 6 13:41:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Oct 2007 13:41:04 +0200 (CEST) Subject: [pypy-svn] r47237 - pypy/dist/pypy/doc/config Message-ID: <20071006114104.D69648116@code0.codespeak.net> Author: fijal Date: Sat Oct 6 13:41:04 2007 New Revision: 47237 Modified: pypy/dist/pypy/doc/config/objspace.usemodules.bz2.txt Log: Update comment Modified: pypy/dist/pypy/doc/config/objspace.usemodules.bz2.txt ============================================================================== --- pypy/dist/pypy/doc/config/objspace.usemodules.bz2.txt (original) +++ pypy/dist/pypy/doc/config/objspace.usemodules.bz2.txt Sat Oct 6 13:41:04 2007 @@ -1,3 +1,2 @@ Use the 'bz2' module. -This module is working "sometimes". It fails to build on 64 bit machines and -can have issues on 32 bit machines. +This module is expected to be working and is included by default. From fijal at codespeak.net Sat Oct 6 13:41:55 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Oct 2007 13:41:55 +0200 (CEST) Subject: [pypy-svn] r47238 - pypy/dist/pypy/doc/config Message-ID: <20071006114155.E9BAC8116@code0.codespeak.net> Author: fijal Date: Sat Oct 6 13:41:55 2007 New Revision: 47238 Modified: pypy/dist/pypy/doc/config/objspace.usemodules.rbench.txt Log: Fix rest Modified: pypy/dist/pypy/doc/config/objspace.usemodules.rbench.txt ============================================================================== --- pypy/dist/pypy/doc/config/objspace.usemodules.rbench.txt (original) +++ pypy/dist/pypy/doc/config/objspace.usemodules.rbench.txt Sat Oct 6 13:41:55 2007 @@ -1,4 +1,4 @@ Use the built-in 'rbench' module. This module contains geninterpreted versions of pystone and richards, so it is useful to measure the interpretation overhead of the various -pypy-*. +pypy-\*. From arigo at codespeak.net Sat Oct 6 13:53:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 6 Oct 2007 13:53:42 +0200 (CEST) Subject: [pypy-svn] r47239 - pypy/dist/pypy/module/mmap Message-ID: <20071006115342.8B29E80E4@code0.codespeak.net> Author: arigo Date: Sat Oct 6 13:53:42 2007 New Revision: 47239 Modified: pypy/dist/pypy/module/mmap/interp_mmap.py Log: For pointer arithmetic, we don't need the same broken logic as we did with rctypes: there is rffi.ptradd(). Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Sat Oct 6 13:53:42 2007 @@ -68,7 +68,7 @@ def external(name, args, result): return rffi.llexternal(name, args, result, includes=CConfig._includes_) -PTR = rffi.VOIDP # XXX? +PTR = rffi.CCHARP has_mremap = cConfig['has_mremap'] @@ -400,14 +400,7 @@ write_byte.unwrap_spec = ['self', str] def getptr(self, offset): - if offset > 0: - # XXX 64-bit support for pointer arithmetic! - # is this still valid? - dataptr = lltype.cast_int_to_ptr(PTR, lltype.cast_ptr_to_int( - self.data) + offset) - return dataptr - else: - return self.data + return rffi.ptradd(self.data, offset) def flush(self, offset=0, size=0): self.check_valid() @@ -666,7 +659,7 @@ space.wrap(os.strerror(e.errno))) res = c_mmap(NULL, map_size, prot, flags, fd, 0) - if lltype.cast_ptr_to_int(res) == -1: + if res == rffi.cast(PTR, -1): raise OperationError(space.w_EnvironmentError, space.wrap(_get_error_msg())) From arigo at codespeak.net Sat Oct 6 17:29:36 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 6 Oct 2007 17:29:36 +0200 (CEST) Subject: [pypy-svn] r47243 - in pypy/branch/kill-keepalives-again/pypy: rpython/memory rpython/memory/test translator/c Message-ID: <20071006152936.C2C608140@code0.codespeak.net> Author: arigo Date: Sat Oct 6 17:29:35 2007 New Revision: 47243 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/kill-keepalives-again/pypy/translator/c/primitive.py Log: Intermediate check-in trying to revive the SemiSpaceGC. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py Sat Oct 6 17:29:35 2007 @@ -88,6 +88,18 @@ def setup(self): pass + def statistics(self, index): + return -1 + + def x_swap_pool(self, newpool): + return newpool + + def x_clone(self, clonedata): + raise RuntimeError("no support for x_clone in the GC") + + def x_become(self, target_addr, source_addr): + raise RuntimeError("no support for x_become in the GC") + class DummyGC(GCBase): _alloc_flavor_ = "raw" @@ -995,7 +1007,8 @@ HDR = lltype.Struct('header', ('forw', lltype.Signed), ('typeid', lltype.Signed)) - def __init__(self, AddressLinkedList, space_size=1024*int_size, + def __init__(self, AddressLinkedList, + space_size=llmemory.arena(lltype.Signed, 1024), get_roots=None): self.bytes_malloced = 0 self.space_size = space_size @@ -1013,6 +1026,8 @@ self.fromspace = raw_malloc(self.space_size) debug_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace + raw_memclear(self.tospace, self.space_size) + raw_memclear(self.fromspace, self.space_size) def free_memory(self): "NOT_RPYTHON" @@ -1034,7 +1049,12 @@ # should eventually be killed return llmemory.cast_ptr_to_adr(ref) - def malloc_fixedsize(self, typeid, size, can_collect): + def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, + contains_weakptr=False): + if has_finalizer: + raise NotImplementedError("finalizers in SemiSpaceGC") + if contains_weakptr: + raise NotImplementedError("weakptr in SemiSpaceGC") size_gc_header = self.gcheaderbuilder.size_gc_header totalsize = size_gc_header + size if can_collect and self.free + totalsize > self.top_of_space: @@ -1049,15 +1069,17 @@ return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, - can_collect): + can_collect, has_finalizer=False): + if has_finalizer: + raise NotImplementedError("finalizers in SemiSpaceGC") + size_gc_header = self.gcheaderbuilder.size_gc_header + nonvarsize = size_gc_header + size try: varsize = ovfcheck(itemsize * length) + totalsize = ovfcheck(nonvarsize + varsize) except OverflowError: raise memoryError - # XXX also check for overflow on the various '+' below! - size += varsize - size_gc_header = self.gcheaderbuilder.size_gc_header - totalsize = size_gc_header + size + # XXX we can't use ovfcheck() for self.free + totalsize... if can_collect and self.free + totalsize > self.top_of_space: self.collect() #XXX need to increase the space size if the object is too big @@ -1070,6 +1092,10 @@ self.free += totalsize return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) + # for now, the spaces are filled with zeroes in advance + malloc_fixedsize_clear = malloc_fixedsize + malloc_varsize_clear = malloc_varsize + def collect(self): ## print "collecting" tospace = self.fromspace @@ -1090,6 +1116,7 @@ curr = scan + self.size_gc_header() self.trace_and_copy(curr) scan += self.get_size(curr) + self.size_gc_header() + raw_memclear(fromspace, self.space_size) def copy(self, obj): if not self.fromspace <= obj < self.fromspace + self.space_size: @@ -1166,9 +1193,14 @@ return self.gcheaderbuilder.size_gc_header def init_gc_object(self, addr, typeid): - addr.signed[0] = 0 - addr.signed[1] = typeid - init_gc_object_immortal = init_gc_object + hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) + #hdr.forw = 0 -- unneeded, the space is initially filled with zero + hdr.typeid = typeid + + def init_gc_object_immortal(self, addr, typeid): + hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) + hdr.forw = 0 + hdr.typeid = typeid class DeferredRefcountingGC(GCBase): _alloc_flavor_ = "raw" Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py Sat Oct 6 17:29:35 2007 @@ -773,12 +773,11 @@ class TestSemiSpaceGC(TestMarkSweepGC): - gcname = "semispace" def setup_class(cls): py.test.skip("in-progress") class gcpolicy(gc.StacklessFrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): from pypy.rpython.memory.gc import SemiSpaceGC as GCClass - GC_PARAMS = {'space_size': 4096 } + GC_PARAMS = {'space_size': llmemory.arena(lltype.Signed, 512)} root_stack_depth = 200 Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/primitive.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/primitive.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/primitive.py Sat Oct 6 17:29:35 2007 @@ -7,7 +7,7 @@ from pypy.rpython.lltypesystem.llmemory import Address, \ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ CompositeOffset, ArrayLengthOffset, \ - GCHeaderOffset + GCHeaderOffset, ArenaRange from pypy.translator.c.support import cdecl, barebonearray # ____________________________________________________________ @@ -48,6 +48,8 @@ return '0' elif type(value) == GCHeaderOffset: return '0' + elif isinstance(value, ArenaRange): + return '(%d * %d)' % (value.n, name_signed(value.unitsize, db)) elif isinstance(value, CDefinedIntSymbolic): return str(value.expr) elif isinstance(value, ComputedIntSymbolic): From cfbolz at codespeak.net Sat Oct 6 20:35:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 6 Oct 2007 20:35:53 +0200 (CEST) Subject: [pypy-svn] r47246 - pypy/branch/kill-keepalives-again/pypy/translator/c/test Message-ID: <20071006183553.044A5811F@code0.codespeak.net> Author: cfbolz Date: Sat Oct 6 20:35:52 2007 New Revision: 47246 Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lltyped.py Log: this one fails on the trunk as well, won't fix before merging, I guess. Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lltyped.py Sat Oct 6 20:35:52 2007 @@ -652,7 +652,7 @@ fn() def test_prebuilt_nolength_char_array(self): - py.test.skip("XXX fix this!") + py.test.skip("fails on the trunk too") for lastchar in ('\x00', 'X'): A = Array(Char, hints={'nolength': True}) a = malloc(A, 5, immortal=True) From cfbolz at codespeak.net Sat Oct 6 23:52:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 6 Oct 2007 23:52:55 +0200 (CEST) Subject: [pypy-svn] r47247 - pypy/branch/kill-keepalives-again/pypy/annotation Message-ID: <20071006215255.C770380F6@code0.codespeak.net> Author: cfbolz Date: Sat Oct 6 23:52:54 2007 New Revision: 47247 Modified: pypy/branch/kill-keepalives-again/pypy/annotation/builtin.py Log: humpf, if you do something like cast_ptr_to_adr(array_of_structs[0]) you get the address of a newly allocated structure representing the interior pointer to the first struct in the array. Modified: pypy/branch/kill-keepalives-again/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/annotation/builtin.py (original) +++ pypy/branch/kill-keepalives-again/pypy/annotation/builtin.py Sat Oct 6 23:52:54 2007 @@ -325,6 +325,8 @@ return immutablevalue(None) def llmemory_cast_ptr_to_adr(s): + from pypy.annotation.model import SomeInteriorPtr + assert not isinstance(s, SomeInteriorPtr) return SomeAddress() def llmemory_cast_adr_to_ptr(s, s_type): From cfbolz at codespeak.net Sat Oct 6 23:58:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 6 Oct 2007 23:58:27 +0200 (CEST) Subject: [pypy-svn] r47249 - pypy/branch/kill-keepalives-again/pypy/translator/c/test Message-ID: <20071006215827.8B1DE80DF@code0.codespeak.net> Author: cfbolz Date: Sat Oct 6 23:58:27 2007 New Revision: 47249 Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lltyped.py Log: fix this test Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lltyped.py Sat Oct 6 23:58:27 2007 @@ -164,17 +164,21 @@ assert res == 0 + 10 + 30 + 1000 def test_structarray_add(self): - py.test.skip("XXX fix this!") from pypy.rpython.lltypesystem import llmemory S = Struct("S", ("x", Signed)) PS = Ptr(S) size = llmemory.sizeof(S) A = GcArray(S) + itemoffset = llmemory.itemoffsetof(A, 0) def llf(n): a = malloc(A, 5) + a[0].x = 1 + a[1].x = 2 + a[2].x = 3 a[3].x = 42 - adr_s = llmemory.cast_ptr_to_adr(a[0]) - adr_s += size * n + a[4].x = 4 + adr_s = llmemory.cast_ptr_to_adr(a) + adr_s += itemoffset + size * n s = llmemory.cast_adr_to_ptr(adr_s, PS) return s.x fn = self.getcompiled(llf, [int]) From arigo at codespeak.net Sun Oct 7 12:17:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 7 Oct 2007 12:17:17 +0200 (CEST) Subject: [pypy-svn] r47255 - in pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem: . test Message-ID: <20071007101717.4E377812C@code0.codespeak.net> Author: arigo Date: Sun Oct 7 12:17:15 2007 New Revision: 47255 Added: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py (contents, props changed) pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py (contents, props changed) Log: Experimental. Arenas that allow objects to be put anywhere in them, in any order. When run directly (or via the llinterp), they do best-effort checks against overflows and overlaps by using the raw_malloc_usage() estimate. The advantage is that can be directly use address arithmetic to handle the arenas, as the existing GCs do. A drawback is that the implementation looks a little bit like the memory simulator, but at least it tries to be reasonably integrated with the rest of lltype and llmemory. For now I'll make arena_malloc, arena_free and arena_reset new lloperations, instead of using raw_malloc with a symbolic arena size. I can imagine situations where it's more reasonable in the backend to use a different implementation to allocate small objects and whole arenas (e.g. mmap). Similarly, arena_reset can optionally fill the arena with zero, which for very large amounts of memory can be implemented in more efficient ways than just memset(). Added: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- (empty file) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py Sun Oct 7 12:17:15 2007 @@ -0,0 +1,143 @@ +import array +from pypy.rpython.lltypesystem import lltype, llmemory + +# An "arena" is a large area of memory which can hold a number of +# objects, not necessarily all of the same type or size. It's used by +# some of our framework GCs. Addresses that point inside arenas support +# direct arithmetic: adding and subtracting integers, and taking the +# difference of two addresses. When not translated to C, the arena +# keeps track of which bytes are used by what object to detect GC bugs; +# it internally uses raw_malloc_usage() to estimate the number of bytes +# it needs to reserve. + +class ArenaError(Exception): + pass + +class Arena(object): + + def __init__(self, nbytes, zero): + self.nbytes = nbytes + self.usagemap = array.array('c') + self.objects = {} + self.freed = False + self.reset(zero) + + def reset(self, zero): + self.check() + for obj in self.objects.itervalues(): + obj._free() + self.objects.clear() + if zero: + initialbyte = "0" + else: + initialbyte = "#" + self.usagemap[:] = array.array('c', initialbyte * self.nbytes) + + def check(self): + if self.freed: + raise ArenaError("arena was already freed") + + def _getid(self): + address, length = self.usagemap.buffer_info() + return address + + def getaddr(self, offset): + if not (0 <= offset <= self.nbytes): + raise ArenaError("Address offset is outside the arena") + return fakearenaaddress(self, offset) + + def allocate_object(self, offset, TYPE): + self.check() + size = llmemory.raw_malloc_usage(llmemory.sizeof(TYPE)) + if offset + size > self.nbytes: + raise ArenaError("object overflows beyond the end of the arena") + zero = True + for c in self.usagemap[offset:offset+size]: + if c == '0': + pass + elif c == '#': + zero = False + else: + raise ArenaError("new object overlaps a previous object") + p = lltype.malloc(TYPE, flavor='raw', zero=zero) + self.usagemap[offset:offset+size] = array.array('c', 'X' * size) + self.objects[offset] = p._obj + +class fakearenaaddress(llmemory.fakeaddress): + + def __init__(self, arena, offset): + self.arena = arena + self.offset = offset + + def _getptr(self): + try: + obj = self.arena.objects[self.offset] + except KeyError: + self.arena.check() + raise ArenaError("don't know yet what type of object " + "is at offset %d" % (self.offset,)) + return obj._as_ptr() + ptr = property(_getptr) + + def __repr__(self): + return '' % (self.arena, self.offset) + + def __add__(self, other): + if isinstance(other, llmemory.AddressOffset): + other = llmemory.raw_malloc_usage(other) + if isinstance(other, (int, long)): + return self.arena.getaddr(self.offset + other) + return NotImplemented + + def __sub__(self, other): + if isinstance(other, llmemory.AddressOffset): + other = llmemory.raw_malloc_usage(other) + if isinstance(other, (int, long)): + return self.arena.getaddr(self.offset - other) + if isinstance(other, fakearenaaddress): + if self.other is not other.arena: + raise ArenaError("The two addresses are from different arenas") + return other.offset - self.offset + return NotImplemented + + def __nonzero__(self): + return True + + def __eq__(self, other): + if isinstance(other, fakearenaaddress): + return self.arena is other.arena and self.offset == other.offset + else: + return llmemory.fakeaddress.__eq__(self, other) + + def _cast_to_ptr(self, EXPECTED_TYPE): + # the first cast determines what object type is at this address + if self.offset not in self.arena.objects: + self.arena.allocate_object(self.offset, EXPECTED_TYPE.TO) + return llmemory.fakeaddress._cast_to_ptr(self, EXPECTED_TYPE) + + def _cast_to_int(self): + return self.arena._getid() + self.offset + +# ____________________________________________________________ +# +# Public interface: arena_malloc(), arena_free() and arena_reset() +# which directly correspond to lloperations. Although the operations +# are similar to raw_malloc(), raw_free() and raw_memclear(), the +# backend can choose a different implementation for arenas, one that +# is more suited to very large chunks of memory. + +def arena_malloc(nbytes, zero): + """Allocate and return a new arena, optionally zero-initialized.""" + return Arena(nbytes, zero).getaddr(0) + +def arena_free(arena_addr): + """Release an arena.""" + arena_reset(arena_addr, False) + arena_addr.arena.freed = True + +def arena_reset(arena_addr, zero): + """Free all objects in the arena, which can then be reused. + The arena is filled with zeroes if 'zero' is True.""" + assert isinstance(arena_addr, fakearenaaddress) + assert arena_addr.offset == 0 + arena_addr.arena.reset(zero) Added: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- (empty file) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py Sun Oct 7 12:17:15 2007 @@ -0,0 +1,47 @@ +import py +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr +from pypy.rpython.lltypesystem.llarena import arena_malloc, arena_reset +from pypy.rpython.lltypesystem.llarena import ArenaError + +def test_arena(): + S = lltype.Struct('S', ('x',lltype.Signed)) + SPTR = lltype.Ptr(S) + ssize = llmemory.raw_malloc_usage(llmemory.sizeof(S)) + a = arena_malloc(2*ssize+1, False) + + s1_ptr1 = cast_adr_to_ptr(a, SPTR) + s1_ptr1.x = 1 + s1_ptr2 = cast_adr_to_ptr(a, SPTR) + assert s1_ptr2.x == 1 + assert s1_ptr1 == s1_ptr2 + + s2_ptr1 = cast_adr_to_ptr(a + ssize + 1, SPTR) + py.test.raises(lltype.UninitializedMemoryAccess, 's2_ptr1.x') + s2_ptr1.x = 2 + s2_ptr2 = cast_adr_to_ptr(a + ssize + 1, SPTR) + assert s2_ptr2.x == 2 + assert s2_ptr1 == s2_ptr2 + assert s1_ptr1 != s2_ptr1 + assert not (s2_ptr2 == s1_ptr2) + assert s1_ptr1 == cast_adr_to_ptr(a, SPTR) + + S2 = lltype.Struct('S2', ('y',lltype.Char)) + S2PTR = lltype.Ptr(S2) + py.test.raises(TypeError, cast_adr_to_ptr, a, S2PTR) + py.test.raises(ArenaError, cast_adr_to_ptr, a+1, SPTR) + py.test.raises(ArenaError, cast_adr_to_ptr, a+ssize, SPTR) + py.test.raises(ArenaError, cast_adr_to_ptr, a+2*ssize, SPTR) + py.test.raises(ArenaError, cast_adr_to_ptr, a+2*ssize+1, SPTR) + + arena_reset(a, True) + s1_ptr1 = cast_adr_to_ptr(a, SPTR) + assert s1_ptr1.x == 0 + s1_ptr1.x = 5 + + s2_ptr1 = cast_adr_to_ptr(a + ssize, S2PTR) + assert s2_ptr1.y == '\x00' + s2_ptr1.y = 'X' + + assert cast_adr_to_ptr(a + 0, SPTR).x == 5 + assert cast_adr_to_ptr((a + ssize + 1) - 1, S2PTR).y == 'X' From arigo at codespeak.net Sun Oct 7 13:09:18 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 7 Oct 2007 13:09:18 +0200 (CEST) Subject: [pypy-svn] r47256 - in pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem: . test Message-ID: <20071007110918.CD366813F@code0.codespeak.net> Author: arigo Date: Sun Oct 7 13:09:17 2007 New Revision: 47256 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py Log: Minor fixes. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py Sun Oct 7 13:09:17 2007 @@ -106,6 +106,8 @@ def __eq__(self, other): if isinstance(other, fakearenaaddress): return self.arena is other.arena and self.offset == other.offset + elif isinstance(other, llmemory.fakeaddress) and not other: + return False # 'self' can't be equal to NULL else: return llmemory.fakeaddress.__eq__(self, other) @@ -132,12 +134,15 @@ def arena_free(arena_addr): """Release an arena.""" - arena_reset(arena_addr, False) + assert isinstance(arena_addr, fakearenaaddress) + assert arena_addr.offset == 0 + arena_addr.arena.reset(False) arena_addr.arena.freed = True -def arena_reset(arena_addr, zero): +def arena_reset(arena_addr, myarenasize, zero): """Free all objects in the arena, which can then be reused. The arena is filled with zeroes if 'zero' is True.""" assert isinstance(arena_addr, fakearenaaddress) assert arena_addr.offset == 0 + assert myarenasize == arena_addr.arena.nbytes arena_addr.arena.reset(zero) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py Sun Oct 7 13:09:17 2007 @@ -8,7 +8,10 @@ S = lltype.Struct('S', ('x',lltype.Signed)) SPTR = lltype.Ptr(S) ssize = llmemory.raw_malloc_usage(llmemory.sizeof(S)) - a = arena_malloc(2*ssize+1, False) + myarenasize = 2*ssize+1 + a = arena_malloc(myarenasize, False) + assert a != llmemory.NULL + assert a + 3 != llmemory.NULL s1_ptr1 = cast_adr_to_ptr(a, SPTR) s1_ptr1.x = 1 @@ -34,7 +37,7 @@ py.test.raises(ArenaError, cast_adr_to_ptr, a+2*ssize, SPTR) py.test.raises(ArenaError, cast_adr_to_ptr, a+2*ssize+1, SPTR) - arena_reset(a, True) + arena_reset(a, myarenasize, True) s1_ptr1 = cast_adr_to_ptr(a, SPTR) assert s1_ptr1.x == 0 s1_ptr1.x = 5 From arigo at codespeak.net Sun Oct 7 13:10:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 7 Oct 2007 13:10:49 +0200 (CEST) Subject: [pypy-svn] r47257 - in pypy/branch/kill-keepalives-again/pypy: config config/test doc/config Message-ID: <20071007111049.17FA8813F@code0.codespeak.net> Author: arigo Date: Sun Oct 7 13:10:48 2007 New Revision: 47257 Modified: pypy/branch/kill-keepalives-again/pypy/config/test/test_pypyoption.py pypy/branch/kill-keepalives-again/pypy/config/translationoption.py pypy/branch/kill-keepalives-again/pypy/doc/config/translation.gc.txt Log: Intermediate check-in: add a --frameworkgc option to choose between GCs. Not implemented yet. Modified: pypy/branch/kill-keepalives-again/pypy/config/test/test_pypyoption.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/config/test/test_pypyoption.py (original) +++ pypy/branch/kill-keepalives-again/pypy/config/test/test_pypyoption.py Sun Oct 7 13:10:48 2007 @@ -23,6 +23,13 @@ assert conf.translation.type_system == "lltype" assert conf.translation.gc == "framework" +def test_frameworkgc(): + for name in ["marksweep", "semispace"]: + conf = get_pypy_config() + assert conf.translation.gc != "framework" + conf.translation.frameworkgc = name + assert conf.translation.gc == "framework" + def test_check_documentation(): from pypy.doc.config.confrest import all_optiondescrs configdocdir = thisdir.dirpath().dirpath().join("doc", "config") Modified: pypy/branch/kill-keepalives-again/pypy/config/translationoption.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/config/translationoption.py (original) +++ pypy/branch/kill-keepalives-again/pypy/config/translationoption.py Sun Oct 7 13:10:48 2007 @@ -51,6 +51,12 @@ default=False, cmdline="--stacklessgc", requires=[("translation.gc", "framework"), ("translation.stackless", True)]), + ChoiceOption("frameworkgc", "Select one of our custom GCs", + ["marksweep", "semispace"], + "marksweep", requires={ + "marksweep": [("translation.gc", "framework")], + "semispace": [("translation.gc", "framework")], + }), BoolOption("thread", "enable use of threading primitives", default=False, cmdline="--thread", requires=[("translation.gc", "boehm")]), Modified: pypy/branch/kill-keepalives-again/pypy/doc/config/translation.gc.txt ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/doc/config/translation.gc.txt (original) +++ pypy/branch/kill-keepalives-again/pypy/doc/config/translation.gc.txt Sun Oct 7 13:10:48 2007 @@ -3,8 +3,9 @@ - "ref": reference counting. Takes very long to translate and the result is slow. - - "framework": our custom mark-and-sweep collector. Takes moderately long and - is the fastest option without external dependencies. + - "framework": one of our custom garbage collectors. Takes moderately long + and is the fastest option without external dependencies. See + :config:`translation.frameworkgc`. - "boehm": use the Boehm conservative GC From arigo at codespeak.net Sun Oct 7 13:41:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 7 Oct 2007 13:41:12 +0200 (CEST) Subject: [pypy-svn] r47258 - pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem Message-ID: <20071007114112.6F411813D@code0.codespeak.net> Author: arigo Date: Sun Oct 7 13:41:12 2007 New Revision: 47258 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/opimpl.py Log: op_getinteriorfield() was too strict about checking for _immutable everywhere. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/opimpl.py Sun Oct 7 13:41:12 2007 @@ -133,15 +133,17 @@ def op_getinteriorfield(obj, *offsets): checkptr(obj) ob = obj - T = lltype.typeOf(obj).TO for o in offsets: - if not T._hints.get('immutable'): - raise TypeError("cannot fold getinteriorfield on mutable struct") + innermostcontainer = ob if isinstance(o, str): ob = getattr(ob, o) - T = getattr(T, o) else: - raise TypeError("cannot fold getfield on mutable struct") + ob = ob[o] + # we can constant-fold this if the innermost structure from which we + # read the final field is immutable. + T = lltype.typeOf(innermostcontainer).TO + if not T._hints.get('immutable'): + raise TypeError("cannot fold getinteriorfield on mutable struct") assert not isinstance(ob, lltype._interior_ptr) return ob From arigo at codespeak.net Sun Oct 7 13:42:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 7 Oct 2007 13:42:44 +0200 (CEST) Subject: [pypy-svn] r47259 - in pypy/branch/kill-keepalives-again/pypy/rpython: . lltypesystem Message-ID: <20071007114244.5B7DF814D@code0.codespeak.net> Author: arigo Date: Sun Oct 7 13:42:44 2007 New Revision: 47259 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llheap.py Log: Intermediate check-in: remove the 'gc' attribute of llinterp. Now the heap is responsible for handling all gc-flavored operations in a special way if it wants to. (rpython/memory needs fixing now) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py Sun Oct 7 13:42:44 2007 @@ -49,16 +49,13 @@ self.heap = heap #module that provides malloc, etc for lltypes self.exc_data_ptr = exc_data_ptr self.active_frame = None - # XXX hack: set gc to None because - # prepare_graphs_and_create_gc might already use the llinterpreter! - self.gc = None self.tracer = None self.malloc_check = malloc_check self.frame_class = LLFrame self.mallocs = {} if hasattr(heap, "prepare_graphs_and_create_gc"): flowgraphs = typer.annotator.translator.graphs - self.gc = heap.prepare_graphs_and_create_gc(self, flowgraphs) + heap.prepare_graphs_and_create_gc(self, flowgraphs) if tracing: self.tracer = Tracer() @@ -273,8 +270,7 @@ if nextblock is None: self.llinterpreter.active_frame = self.f_back for obj in self.alloca_objects: - #XXX slighly unclean - obj._setobj(None) + self.heap.free(obj, flavor='raw') return args finally: if tracer: @@ -548,15 +544,14 @@ def op_setfield(self, obj, fieldname, fieldvalue): # obj should be pointer FIELDTYPE = getattr(lltype.typeOf(obj).TO, fieldname) - if FIELDTYPE != lltype.Void: - gc = self.llinterpreter.gc - if gc is None or not gc.needs_write_barrier(FIELDTYPE): - setattr(obj, fieldname, fieldvalue) - else: - args = gc.get_arg_write_barrier(obj, fieldname, fieldvalue) - write_barrier = gc.get_funcptr_write_barrier() - result = self.op_direct_call(write_barrier, *args) - op_bare_setfield = op_setfield + if FIELDTYPE is not lltype.Void: + self.heap.setfield(obj, fieldname, fieldvalue) + + def op_bare_setfield(self, obj, fieldname, fieldvalue): + # obj should be pointer + FIELDTYPE = getattr(lltype.typeOf(obj).TO, fieldname) + if FIELDTYPE is not lltype.Void: + setattr(obj, fieldname, fieldvalue) def op_getinteriorfield(self, obj, *offsets): checkptr(obj) @@ -569,7 +564,7 @@ assert not isinstance(ob, lltype._interior_ptr) return ob - def op_setinteriorfield(self, obj, *fieldnamesval): + def setinterior(self, heap, obj, *fieldnamesval): prefields, finalfield, fieldvalue = ( fieldnamesval[:-2], fieldnamesval[-2], fieldnamesval[-1]) for o in prefields: @@ -577,11 +572,19 @@ obj = getattr(obj, o) else: obj = obj[o] + T = obj._T if isinstance(finalfield, str): - setattr(obj, finalfield, fieldvalue) + if getattr(T, fieldname) is not lltype.Void: + heap.setfield(obj, finalfield, fieldvalue) else: - obj[finalfield] = fieldvalue - op_bare_setinteriorfield = op_setinteriorfield + if T.OF is not lltype.Void: + heap.setarrayitem(obj, finalfield, fieldvalue) + + def op_setinteriorfield(self, obj, *fieldnamesval): + self.setinterior(self.heap, obj, *fieldnamesval) + + def op_bare_setinteriorfield(self, obj, *fieldnamesval): + self.setinterior(llheap, obj, *fieldnamesval) def op_getarrayitem(self, array, index): return array[index] @@ -589,15 +592,14 @@ def op_setarrayitem(self, array, index, item): # array should be a pointer ITEMTYPE = lltype.typeOf(array).TO.OF - if ITEMTYPE != lltype.Void: - gc = self.llinterpreter.gc - if gc is None or not gc.needs_write_barrier(ITEMTYPE): - array[index] = item - else: - args = gc.get_arg_write_barrier(array, index, item) - write_barrier = gc.get_funcptr_write_barrier() - self.op_direct_call(write_barrier, *args) - op_bare_setarrayitem = op_setarrayitem + if ITEMTYPE is not lltype.Void: + self.heap.setarrayitem(array, index, item) + + def op_bare_setarrayitem(self, array, index, item): + # array should be a pointer + ITEMTYPE = lltype.typeOf(array).TO.OF + if ITEMTYPE is not lltype.Void: + array[index] = item def perform_call(self, f, ARGS, args): @@ -654,20 +656,10 @@ def op_malloc(self, obj, flags): flavor = flags['flavor'] zero = flags.get('zero', False) - if self.llinterpreter.gc is not None and flavor == 'gc': - assert not zero - args = self.llinterpreter.gc.get_arg_malloc(obj) - malloc = self.llinterpreter.gc.get_funcptr_malloc() - result = self.op_direct_call(malloc, *args) - return self.llinterpreter.gc.adjust_result_malloc(result, obj) - elif flavor == "stack": - if isinstance(obj, lltype.Struct) and obj._arrayfld is None: - result = self.heap.malloc(obj) - self.alloca_objects.append(result) - return result - else: - raise ValueError("cannot allocate variable-sized things on the stack") - + if flavor == "stack": + result = self.heap.malloc(obj, zero=zero, flavor='raw') + self.alloca_objects.append(result) + return result ptr = self.heap.malloc(obj, zero=zero, flavor=flavor) if flavor == 'raw' and self.llinterpreter.malloc_check: self.llinterpreter.remember_malloc(ptr, self) @@ -683,12 +675,6 @@ def op_malloc_varsize(self, obj, flags, size): flavor = flags['flavor'] zero = flags.get('zero', False) - if self.llinterpreter.gc is not None and flavor == 'gc': - assert not zero - args = self.llinterpreter.gc.get_arg_malloc(obj, size) - malloc = self.llinterpreter.gc.get_funcptr_malloc() - result = self.op_direct_call(malloc, *args) - return self.llinterpreter.gc.adjust_result_malloc(result, obj, size) assert flavor in ('gc', 'raw') try: ptr = self.heap.malloc(obj, size, zero=zero, flavor=flavor) @@ -742,8 +728,7 @@ op_cast_weakrefptr_to_ptr.need_result_type = True def op_gc__collect(self): - import gc - gc.collect() + self.heap.collect() def op_gc_free(self, addr): # what can you do? @@ -1030,7 +1015,7 @@ checkinst(inst) assert isinstance(name, str) FIELDTYPE = lltype.typeOf(inst)._field_type(name) - if FIELDTYPE != lltype.Void: + if FIELDTYPE is not lltype.Void: setattr(inst, name, value) def op_oogetfield(self, inst, name): Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llheap.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llheap.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llheap.py Sun Oct 7 13:42:44 2007 @@ -4,3 +4,7 @@ from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free from pypy.rpython.lltypesystem.llmemory import raw_memclear, raw_memcopy from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage + +setfield = setattr +from operator import setitem as setarrayitem +from gc import collect From arigo at codespeak.net Sun Oct 7 13:43:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 7 Oct 2007 13:43:37 +0200 (CEST) Subject: [pypy-svn] r47260 - in pypy/branch/kill-keepalives-again/pypy/rpython/memory: . test Message-ID: <20071007114337.863F7813D@code0.codespeak.net> Author: arigo Date: Sun Oct 7 13:43:37 2007 New Revision: 47260 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_gc.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_llinterpsim.py Log: Intermediate check-in: try to port SemiSpaceGC to use llarena. Untested. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py Sun Oct 7 13:43:37 2007 @@ -1,9 +1,10 @@ -from pypy.rpython.memory.lladdress import raw_malloc, raw_free, raw_memcopy, raw_memclear -from pypy.rpython.memory.lladdress import NULL, _address, raw_malloc_usage +from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free +from pypy.rpython.lltypesystem.llmemory import raw_memcopy, raw_memclear +from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.memory.support import get_address_linked_list from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.memory import lltypesimulation -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object, debug_assert from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck @@ -1007,8 +1008,7 @@ HDR = lltype.Struct('header', ('forw', lltype.Signed), ('typeid', lltype.Signed)) - def __init__(self, AddressLinkedList, - space_size=llmemory.arena(lltype.Signed, 1024), + def __init__(self, AddressLinkedList, space_size=4096, get_roots=None): self.bytes_malloced = 0 self.space_size = space_size @@ -1020,20 +1020,18 @@ self.gcheaderbuilder = GCHeaderBuilder(self.HDR) def setup(self): - self.tospace = raw_malloc(self.space_size) + self.tospace = llarena.arena_malloc(self.space_size, True) debug_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size - self.fromspace = raw_malloc(self.space_size) + self.fromspace = llarena.arena_malloc(self.space_size, True) debug_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace - raw_memclear(self.tospace, self.space_size) - raw_memclear(self.fromspace, self.space_size) def free_memory(self): "NOT_RPYTHON" - raw_free(self.tospace) + llarena.arena_free(self.tospace) self.tospace = NULL - raw_free(self.fromspace) + llarena.arena_free(self.fromspace) self.fromspace = NULL def malloc(self, typeid, length=0): @@ -1116,7 +1114,7 @@ curr = scan + self.size_gc_header() self.trace_and_copy(curr) scan += self.get_size(curr) + self.size_gc_header() - raw_memclear(fromspace, self.space_size) + llarena.arena_reset(fromspace, self.space_size, True) def copy(self, obj): if not self.fromspace <= obj < self.fromspace + self.space_size: Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_gc.py Sun Oct 7 13:43:37 2007 @@ -94,31 +94,13 @@ assert res == concat(100) assert simulator.current_size - curr < 16000 * INT_SIZE / 4 - -class GCTestOnLLInterp(GCTest): - - def setup_class(cls): - gclltype.prepare_graphs_and_create_gc = gclltype.create_gc_run_on_llinterp - gclltype.use_gc = cls.GCClass - from pypy.rpython.memory import gc as gcimpl - gcimpl.DEBUG_PRINT = False - class TestMarkSweepGC(GCTest): GCClass = MarkSweepGC -if 0: ## - - - disabling this: the simulator is getting deprecated - class TestMarkSweepGCRunningOnLLinterp(GCTestOnLLInterp): - GCClass = MarkSweepGC - class TestSemiSpaceGC(GCTest): GCClass = SemiSpaceGC -class TestSemiSpaceGCRunningOnLLinterp(GCTestOnLLInterp): - GCClass = SemiSpaceGC - class TestDeferredRefcountingGC(GCTest): GCClass = DeferredRefcountingGC - -class TestDeferredRefcountingGCRunningOnLLinterp(GCTestOnLLInterp): - GCClass = DeferredRefcountingGC - + def setup_class(cls): + py.test.skip("DeferredRefcounting is unmaintained") Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_llinterpsim.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_llinterpsim.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_llinterpsim.py Sun Oct 7 13:43:37 2007 @@ -1,5 +1,6 @@ import py +py.test.skip("kill me!") from pypy.rpython.lltypesystem.lltype import typeOf, Ptr, PyObject from pypy.rpython.rtyper import RPythonTyper from pypy.rpython.llinterp import LLInterpreter, LLException,log From cfbolz at codespeak.net Sun Oct 7 13:56:21 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 7 Oct 2007 13:56:21 +0200 (CEST) Subject: [pypy-svn] r47261 - pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform Message-ID: <20071007115621.46FF1814D@code0.codespeak.net> Author: cfbolz Date: Sun Oct 7 13:56:21 2007 New Revision: 47261 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/transform.py Log: make sure that setinteriorfield is transformed correctly Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/transform.py Sun Oct 7 13:56:21 2007 @@ -321,9 +321,7 @@ else: hop.rename('bare_' + hop.spaceop.opname) gct_setarrayitem = gct_setfield - - #def gct_safe_call(self, hop): - # hop.rename("direct_call") + gct_setinteriorfield = gct_setfield def gct_zero_gc_pointers_inside(self, hop): pass From cfbolz at codespeak.net Sun Oct 7 13:57:59 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 7 Oct 2007 13:57:59 +0200 (CEST) Subject: [pypy-svn] r47262 - pypy/branch/kill-keepalives-again/pypy/translator/c Message-ID: <20071007115759.E2165814E@code0.codespeak.net> Author: cfbolz Date: Sun Oct 7 13:57:59 2007 New Revision: 47262 Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/funcgen.py Log: don't support the non-bare versions of get*. The GC transformer should have removed them already. Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/funcgen.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/funcgen.py Sun Oct 7 13:57:59 2007 @@ -441,7 +441,7 @@ op.args[1].value) return self.generic_get(op, expr) - def OP_SETFIELD(self, op): + def OP_BARE_SETFIELD(self, op): assert isinstance(op.args[1], Constant) STRUCT = self.lltypemap(op.args[0]).TO structdef = self.db.gettypedefnode(STRUCT) @@ -449,8 +449,6 @@ op.args[1].value) return self.generic_set(op, expr) - OP_BARE_SETFIELD = OP_SETFIELD - def OP_GETSUBSTRUCT(self, op): RESULT = self.lltypemap(op.result).TO if isinstance(RESULT, FixedSizeArray): @@ -522,7 +520,6 @@ def OP_BARE_SETINTERIORFIELD(self, op): return self.generic_set(op, self.interior_expr(op.args[:-1])) - OP_SETINTERIORFIELD = OP_BARE_SETINTERIORFIELD def OP_GETINTERIORARRAYSIZE(self, op): expr, ARRAY = self.interior_expr(op.args, True) From lac at codespeak.net Sun Oct 7 15:17:33 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Sun, 7 Oct 2007 15:17:33 +0200 (CEST) Subject: [pypy-svn] r47263 - pypy/dist/demo/distribution Message-ID: <20071007131733.65AB18151@code0.codespeak.net> Author: lac Date: Sun Oct 7 15:17:31 2007 New Revision: 47263 Modified: pypy/dist/demo/distribution/fileserver.py Log: fix typo Modified: pypy/dist/demo/distribution/fileserver.py ============================================================================== --- pypy/dist/demo/distribution/fileserver.py (original) +++ pypy/dist/demo/distribution/fileserver.py Sun Oct 7 15:17:31 2007 @@ -1,5 +1,5 @@ """ This is a sample demo showcasing file server, done by the pypy -distriobution library. +distribution library. Not counting __doc__ and initialization this is 2 line, fully operational file server, From lac at codespeak.net Sun Oct 7 15:44:40 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Sun, 7 Oct 2007 15:44:40 +0200 (CEST) Subject: [pypy-svn] r47264 - pypy/dist/pypy/doc Message-ID: <20071007134440.346908158@code0.codespeak.net> Author: lac Date: Sun Oct 7 15:44:38 2007 New Revision: 47264 Modified: pypy/dist/pypy/doc/stackless.txt Log: fix typos Modified: pypy/dist/pypy/doc/stackless.txt ============================================================================== --- pypy/dist/pypy/doc/stackless.txt (original) +++ pypy/dist/pypy/doc/stackless.txt Sun Oct 7 15:44:38 2007 @@ -130,7 +130,7 @@ sequence of values, while another consumes them. For our purposes we assume that the producer can generate several values at once, and the consumer can process up to 3 values in a batch - it can also process -batches with less than 3 values without waiting for the producer (which +batches with fewer than 3 values without waiting for the producer (which would be messy to express with a classical Python generator). :: def producer(lst): @@ -187,7 +187,7 @@ Tasklets are a bit like threads: they encapsulate a function in such a way that they can be suspended/restarted any time. Unlike threads, they won't run concurrently, but must be cooperative. When using stackless -features, it is vitaly important that no action is performed that blocks +features, it is vitally important that no action is performed that blocks everything else. In particular, blocking input/output should be centralized to a single tasklet. From ac at codespeak.net Sun Oct 7 16:58:53 2007 From: ac at codespeak.net (ac at codespeak.net) Date: Sun, 7 Oct 2007 16:58:53 +0200 (CEST) Subject: [pypy-svn] r47265 - pypy/dist/pypy/interpreter/test Message-ID: <20071007145853.0499E8155@code0.codespeak.net> Author: ac Date: Sun Oct 7 16:58:52 2007 New Revision: 47265 Modified: pypy/dist/pypy/interpreter/test/test_syntax.py Log: Allways run the tests of features of python 2.5 Modified: pypy/dist/pypy/interpreter/test/test_syntax.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_syntax.py (original) +++ pypy/dist/pypy/interpreter/test/test_syntax.py Sun Oct 7 16:58:52 2007 @@ -1,4 +1,5 @@ import py +from pypy.conftest import gettestobjspace def splitcases(s): lines = [line.rstrip() for line in s.split('\n')] @@ -258,14 +259,8 @@ class Py25AppTest: def setup_class(self): - space = self.space - w_not_25 = space.appexec([], """(): - import sys - return sys.version_info < (2,5) - """) - not_25 = space.is_true(w_not_25) - if not_25: - py.test.skip('Needs python 2.5 grammar') + self.space = gettestobjspace(pyversion='2.5a') + return class AppTestCondExpr(Py25AppTest): def test_condexpr(self): From ac at codespeak.net Sun Oct 7 16:59:39 2007 From: ac at codespeak.net (ac at codespeak.net) Date: Sun, 7 Oct 2007 16:59:39 +0200 (CEST) Subject: [pypy-svn] r47266 - pypy/dist/pypy/interpreter/test Message-ID: <20071007145939.9375E8155@code0.codespeak.net> Author: ac Date: Sun Oct 7 16:59:39 2007 New Revision: 47266 Modified: pypy/dist/pypy/interpreter/test/test_syntax.py Log: Allways run the tests of features of python 2.5 Modified: pypy/dist/pypy/interpreter/test/test_syntax.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_syntax.py (original) +++ pypy/dist/pypy/interpreter/test/test_syntax.py Sun Oct 7 16:59:39 2007 @@ -270,6 +270,9 @@ assert x == expected class AppTestWith(Py25AppTest): + def test_with_as_identifier(self): + exec "with = 9" + def test_with_simple(self): s = """from __future__ import with_statement @@ -475,9 +478,6 @@ assert acontextfact.calls == '__enter__ __body__ __exit__ __return__'.split() assert acontextfact.exit_params == (None, None, None) - def test_with_as_identifier(self): - exec "with = 9" - def test_with_as_keyword(self): try: exec "from __future__ import with_statement\nwith = 9" From ac at codespeak.net Sun Oct 7 17:01:57 2007 From: ac at codespeak.net (ac at codespeak.net) Date: Sun, 7 Oct 2007 17:01:57 +0200 (CEST) Subject: [pypy-svn] r47267 - pypy/dist/pypy/interpreter/test Message-ID: <20071007150157.A37A88155@code0.codespeak.net> Author: ac Date: Sun Oct 7 17:01:57 2007 New Revision: 47267 Modified: pypy/dist/pypy/interpreter/test/test_syntax.py Log: Oops. Last commit was accidental. Modified: pypy/dist/pypy/interpreter/test/test_syntax.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_syntax.py (original) +++ pypy/dist/pypy/interpreter/test/test_syntax.py Sun Oct 7 17:01:57 2007 @@ -270,9 +270,6 @@ assert x == expected class AppTestWith(Py25AppTest): - def test_with_as_identifier(self): - exec "with = 9" - def test_with_simple(self): s = """from __future__ import with_statement @@ -478,6 +475,9 @@ assert acontextfact.calls == '__enter__ __body__ __exit__ __return__'.split() assert acontextfact.exit_params == (None, None, None) + def test_with_as_identifier(self): + exec "with = 9" + def test_with_as_keyword(self): try: exec "from __future__ import with_statement\nwith = 9" From arigo at codespeak.net Sun Oct 7 18:24:58 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 7 Oct 2007 18:24:58 +0200 (CEST) Subject: [pypy-svn] r47268 - pypy/branch/kill-keepalives-again/pypy/doc/config Message-ID: <20071007162458.F28A98163@code0.codespeak.net> Author: arigo Date: Sun Oct 7 18:24:57 2007 New Revision: 47268 Added: pypy/branch/kill-keepalives-again/pypy/doc/config/translation.frameworkgc.txt (contents, props changed) Log: Forgot to check this in. Added: pypy/branch/kill-keepalives-again/pypy/doc/config/translation.frameworkgc.txt ============================================================================== --- (empty file) +++ pypy/branch/kill-keepalives-again/pypy/doc/config/translation.frameworkgc.txt Sun Oct 7 18:24:57 2007 @@ -0,0 +1,5 @@ +Select one of our custom Garbage Collectors for the translated program: + + - "marksweep": naive mark & sweep. Default. + + - "semispace": a copying semi-space GC. From arigo at codespeak.net Sun Oct 7 18:56:38 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 7 Oct 2007 18:56:38 +0200 (CEST) Subject: [pypy-svn] r47269 - in pypy/branch/kill-keepalives-again/pypy: annotation rpython rpython/lltypesystem rpython/memory rpython/memory/gctransform rpython/memory/test translator Message-ID: <20071007165638.2AE52816B@code0.codespeak.net> Author: arigo Date: Sun Oct 7 18:56:37 2007 New Revision: 47269 Added: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctypelayout.py (contents, props changed) pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_gctypelayout.py - copied, changed from r47254, pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_lltypelayout.py Removed: pypy/branch/kill-keepalives-again/pypy/rpython/memory/convertlltype.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/gclltype.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/lladdress.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/lltypesimulation.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_address.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_convertlltype.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_llinterpsim.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_lltypelayout.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_lltypesimulation.py Modified: pypy/branch/kill-keepalives-again/pypy/annotation/bookkeeper.py pypy/branch/kill-keepalives-again/pypy/annotation/builtin.py pypy/branch/kill-keepalives-again/pypy/annotation/unaryop.py pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/framework.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/transform.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcwrapper.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/lltypelayout.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/support.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_gc.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_support.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/kill-keepalives-again/pypy/rpython/raddress.py pypy/branch/kill-keepalives-again/pypy/rpython/rbuiltin.py pypy/branch/kill-keepalives-again/pypy/translator/exceptiontransform.py Log: Kill the memory simulator and lladdress. Factor out the parts that compute the GC table into a new module gctypelayout.py, used both by gctransform/framework and by gcwrapper. The latter can be used to test GCs by running them directly, with the llinterpreter doing the calls to it. Many tests pass, but it's still work in progress. Modified: pypy/branch/kill-keepalives-again/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/kill-keepalives-again/pypy/annotation/bookkeeper.py Sun Oct 7 18:56:37 2007 @@ -24,7 +24,6 @@ from pypy.tool.algo.unionfind import UnionFind from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype -from pypy.rpython.memory import lladdress from pypy.rpython import extregistry class Stats: Modified: pypy/branch/kill-keepalives-again/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/annotation/builtin.py (original) +++ pypy/branch/kill-keepalives-again/pypy/annotation/builtin.py Sun Oct 7 18:56:37 2007 @@ -629,8 +629,6 @@ #_________________________________ # memory address -from pypy.rpython.memory import lladdress - def raw_malloc(s_size): assert isinstance(s_size, SomeInteger) #XXX add noneg...? return SomeAddress() @@ -655,12 +653,6 @@ assert not s_addr2.is_null assert isinstance(s_int, SomeInteger) #XXX add noneg...? -BUILTIN_ANALYZERS[lladdress.raw_malloc] = raw_malloc -BUILTIN_ANALYZERS[lladdress.raw_malloc_usage] = raw_malloc_usage -BUILTIN_ANALYZERS[lladdress.raw_free] = raw_free -BUILTIN_ANALYZERS[lladdress.raw_memclear] = raw_memclear -BUILTIN_ANALYZERS[lladdress.raw_memcopy] = raw_memcopy - BUILTIN_ANALYZERS[llmemory.raw_malloc] = raw_malloc BUILTIN_ANALYZERS[llmemory.raw_malloc_usage] = raw_malloc_usage BUILTIN_ANALYZERS[llmemory.raw_free] = raw_free Modified: pypy/branch/kill-keepalives-again/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/annotation/unaryop.py (original) +++ pypy/branch/kill-keepalives-again/pypy/annotation/unaryop.py Sun Oct 7 18:56:37 2007 @@ -763,15 +763,15 @@ #_________________________________________ # memory addresses -from pypy.rpython.memory import lladdress +from pypy.rpython.lltypesystem import llmemory class __extend__(SomeAddress): def getattr(s_addr, s_attr): assert s_attr.is_constant() assert isinstance(s_attr, SomeString) - assert s_attr.const in lladdress.supported_access_types + assert s_attr.const in llmemory.supported_access_types return SomeTypedAddressAccess( - lladdress.supported_access_types[s_attr.const]) + llmemory.supported_access_types[s_attr.const]) getattr.can_only_throw = [] def is_true(s_addr): Modified: pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py Sun Oct 7 18:56:37 2007 @@ -42,20 +42,18 @@ class LLInterpreter(object): """ low level interpreter working with concrete values. """ - def __init__(self, typer, heap=llheap, tracing=True, exc_data_ptr=None, + def __init__(self, typer, tracing=True, exc_data_ptr=None, malloc_check=True): self.bindings = {} self.typer = typer - self.heap = heap #module that provides malloc, etc for lltypes + # 'heap' is module or object that provides malloc, etc for lltype ops + self.heap = llheap self.exc_data_ptr = exc_data_ptr self.active_frame = None self.tracer = None self.malloc_check = malloc_check self.frame_class = LLFrame self.mallocs = {} - if hasattr(heap, "prepare_graphs_and_create_gc"): - flowgraphs = typer.annotator.translator.graphs - heap.prepare_graphs_and_create_gc(self, flowgraphs) if tracing: self.tracer = Tracer() @@ -574,7 +572,7 @@ obj = obj[o] T = obj._T if isinstance(finalfield, str): - if getattr(T, fieldname) is not lltype.Void: + if getattr(T, finalfield) is not lltype.Void: heap.setfield(obj, finalfield, fieldvalue) else: if T.OF is not lltype.Void: Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py Sun Oct 7 18:56:37 2007 @@ -470,6 +470,11 @@ raise TypeError(TARGETTYPE) ptr[0] = value +supported_access_types = {"signed": lltype.Signed, + "unsigned": lltype.Unsigned, + "char": lltype.Char, + "address": Address, + } fakeaddress.signed = property(_signed_fakeaccessor) fakeaddress.char = property(_char_fakeaccessor) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py Sun Oct 7 18:56:37 2007 @@ -3,7 +3,6 @@ from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.memory.support import get_address_linked_list from pypy.rpython.memory.gcheader import GCHeaderBuilder -from pypy.rpython.memory import lltypesimulation from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object, debug_assert from pypy.rpython.lltypesystem.lloperation import llop @@ -11,9 +10,6 @@ import sys, os -int_size = lltypesimulation.sizeof(lltype.Signed) -gc_header_two_ints = 2*int_size - X_POOL = lltype.GcOpaqueType('gc.pool') X_POOL_PTR = lltype.Ptr(X_POOL) X_CLONE = lltype.GcStruct('CloneData', ('gcobjectptr', llmemory.GCREF), @@ -184,7 +180,7 @@ size = self.fixed_size(typeid) needs_finalizer = bool(self.getfinalizer(typeid)) contains_weakptr = self.weakpointer_offset(typeid) != -1 - assert needs_finalizer != contains_weakptr + assert not (needs_finalizer and contains_weakptr) if self.is_varsize(typeid): assert not contains_weakptr itemsize = self.varsize_item_sizes(typeid) @@ -1326,5 +1322,5 @@ addr.signed[1] = typeid def size_gc_header(self, typeid=0): - return gc_header_two_ints + XXX Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/framework.py Sun Oct 7 18:56:37 2007 @@ -3,7 +3,7 @@ get_rtti, ll_call_destructor, type_contains_pyobjs from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython import rmodel -from pypy.rpython.memory import gc, lladdress +from pypy.rpython.memory import gc, gctypelayout from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rlib.rarithmetic import ovfcheck from pypy.rlib.objectmodel import debug_assert @@ -11,6 +11,8 @@ from pypy.annotation import model as annmodel from pypy.rpython import annlowlevel from pypy.rpython.rbuiltin import gen_cast +from pypy.rpython.memory.gctypelayout import ll_weakref_deref, WEAKREF +from pypy.rpython.memory.gctypelayout import convert_weakref_to, WEAKREFPTR import sys @@ -27,7 +29,6 @@ class FrameworkGCTransformer(GCTransformer): use_stackless = False extra_static_slots = 0 - finished_tables = False root_stack_depth = 163840 from pypy.rpython.memory.gc import MarkSweepGC as GCClass @@ -38,7 +39,6 @@ super(FrameworkGCTransformer, self).__init__(translator, inline=True) AddressLinkedList = get_address_linked_list() GCClass = self.GCClass - self.finalizer_funcptrs = {} self.FINALIZERTYPE = lltype.Ptr(ADDRESS_VOID_FUNC) class GCData(object): # types of the GC information tables @@ -92,6 +92,9 @@ debug_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].weakptrofs + self.layoutbuilder = TransformerLayoutBuilder(self) + self.get_type_id = self.layoutbuilder.get_type_id + gcdata = GCData() # set up dummy a table, to be overwritten with the real one in finish() gcdata.type_info_table = lltype.malloc(GCData.TYPE_INFO_TABLE, 0, @@ -105,14 +108,6 @@ gcdata.static_root_start = a_random_address # patched in finish() gcdata.static_root_end = a_random_address # patched in finish() self.gcdata = gcdata - dummy = {"weakptrofs": -1, - "ofstolength": -1} - self.type_info_list = [dummy] # don't use typeid 0, helps debugging - self.id_of_type = {} # {LLTYPE: type_id} - self.seen_roots = {} - self.static_gc_roots = [] - self.addresses_of_static_ptrs_in_nongc = [] - self.offsettable_cache = {} self.malloc_fnptr_cache = {} sizeofaddr = llmemory.sizeof(llmemory.Address) @@ -272,9 +267,9 @@ class StackRootIterator: _alloc_flavor_ = 'raw' def setup_root_stack(): - stackbase = lladdress.raw_malloc(rootstacksize) + stackbase = llmemory.raw_malloc(rootstacksize) debug_assert(bool(stackbase), "could not allocate root stack") - lladdress.raw_memclear(stackbase, rootstacksize) + llmemory.raw_memclear(stackbase, rootstacksize) gcdata.root_stack_top = stackbase gcdata.root_stack_base = stackbase i = 0 @@ -327,101 +322,15 @@ return StackRootIterator - def get_type_id(self, TYPE): - try: - return self.id_of_type[TYPE] - except KeyError: - assert not self.finished_tables - assert isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)) - # Record the new type_id description as a small dict for now. - # It will be turned into a Struct("type_info") in finish() - type_id = len(self.type_info_list) - info = {} - self.type_info_list.append(info) - self.id_of_type[TYPE] = type_id - offsets = offsets_to_gc_pointers(TYPE) - info["ofstoptrs"] = self.offsets2table(offsets, TYPE) - info["finalyzer"] = self.finalizer_funcptr_for_type(TYPE) - info["weakptrofs"] = weakpointer_offset(TYPE) - if not TYPE._is_varsize(): - info["isvarsize"] = False - info["fixedsize"] = llmemory.sizeof(TYPE) - info["ofstolength"] = -1 - else: - info["isvarsize"] = True - info["fixedsize"] = llmemory.sizeof(TYPE, 0) - if isinstance(TYPE, lltype.Struct): - ARRAY = TYPE._flds[TYPE._arrayfld] - ofs1 = llmemory.offsetof(TYPE, TYPE._arrayfld) - info["ofstolength"] = ofs1 + llmemory.ArrayLengthOffset(ARRAY) - if ARRAY.OF != lltype.Void: - info["ofstovar"] = ofs1 + llmemory.itemoffsetof(ARRAY, 0) - else: - info["fixedsize"] = ofs1 + llmemory.sizeof(lltype.Signed) - if ARRAY._hints.get('isrpystring'): - info["fixedsize"] = llmemory.sizeof(TYPE, 1) - else: - ARRAY = TYPE - info["ofstolength"] = llmemory.ArrayLengthOffset(ARRAY) - if ARRAY.OF != lltype.Void: - info["ofstovar"] = llmemory.itemoffsetof(TYPE, 0) - else: - info["fixedsize"] = llmemory.ArrayLengthOffset(ARRAY) + llmemory.sizeof(lltype.Signed) - assert isinstance(ARRAY, lltype.Array) - if ARRAY.OF != lltype.Void: - offsets = offsets_to_gc_pointers(ARRAY.OF) - info["varofstoptrs"] = self.offsets2table(offsets, ARRAY.OF) - info["varitemsize"] = llmemory.sizeof(ARRAY.OF) - else: - info["varofstoptrs"] = self.offsets2table((), lltype.Void) - info["varitemsize"] = llmemory.sizeof(ARRAY.OF) - return type_id - - def finalizer_funcptr_for_type(self, TYPE): - if TYPE in self.finalizer_funcptrs: - return self.finalizer_funcptrs[TYPE] - - rtti = get_rtti(TYPE) - if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'): - destrptr = rtti._obj.destructor_funcptr - DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0] - else: - destrptr = None - DESTR_ARG = None - - assert not type_contains_pyobjs(TYPE), "not implemented" - if destrptr: - def ll_finalizer(addr): - v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) - ll_call_destructor(destrptr, v) - fptr = self.annotate_helper(ll_finalizer, [llmemory.Address], lltype.Void) - else: - fptr = lltype.nullptr(ADDRESS_VOID_FUNC) - - self.finalizer_funcptrs[TYPE] = fptr - return fptr - def consider_constant(self, TYPE, value): - if value is not lltype.top_container(value): - return - if id(value) in self.seen_roots: - return - self.seen_roots[id(value)] = True + self.layoutbuilder.consider_constant(TYPE, value, self.gcdata.gc) - if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): - typeid = self.get_type_id(TYPE) - hdrbuilder = self.gcdata.gc.gcheaderbuilder - hdr = hdrbuilder.new_header(value) - adr = llmemory.cast_ptr_to_adr(hdr) - self.gcdata.gc.init_gc_object(adr, typeid) - - if find_gc_ptrs_in_type(TYPE): - adr = llmemory.cast_ptr_to_adr(value._as_ptr()) - if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): - self.static_gc_roots.append(adr) - else: - for a in gc_pointers_inside(value, adr): - self.addresses_of_static_ptrs_in_nongc.append(a) + #def get_type_id(self, TYPE): + # this method is attached to the instance and redirects to + # layoutbuilder.get_type_id(). + + def finalizer_funcptr_for_type(self, TYPE): + return self.layoutbuilder.finalizer_funcptr_for_type(TYPE) def gc_fields(self): return self._gc_fields @@ -431,25 +340,8 @@ HDR = self._gc_HDR return [getattr(hdr, fldname) for fldname in HDR._names] - def offsets2table(self, offsets, TYPE): - try: - return self.offsettable_cache[TYPE] - except KeyError: - cachedarray = lltype.malloc(self.gcdata.OFFSETS_TO_GC_PTR, - len(offsets), immortal=True) - for i, value in enumerate(offsets): - cachedarray[i] = value - self.offsettable_cache[TYPE] = cachedarray - return cachedarray - def finish_tables(self): - self.finished_tables = True - table = lltype.malloc(self.gcdata.TYPE_INFO_TABLE, - len(self.type_info_list), immortal=True) - for tableentry, newcontent in zip(table, self.type_info_list): - for key, value in newcontent.items(): - setattr(tableentry, key, value) - self.offsettable_cache = None + table = self.layoutbuilder.flatten_table() # replace the type_info_table pointer in gcdata -- at this point, # the database is in principle complete, so it has already seen @@ -468,20 +360,23 @@ ll_instance.inst_type_info_table = table #self.gcdata.type_info_table = table + static_gc_roots = self.layoutbuilder.static_gc_roots ll_static_roots = lltype.malloc(lltype.Array(llmemory.Address), - len(self.static_gc_roots) + + len(static_gc_roots) + self.extra_static_slots, immortal=True) - for i in range(len(self.static_gc_roots)): - adr = self.static_gc_roots[i] + for i in range(len(static_gc_roots)): + adr = static_gc_roots[i] ll_static_roots[i] = adr ll_instance.inst_static_roots = ll_static_roots + addresses_of_static_ptrs_in_nongc = \ + self.layoutbuilder.addresses_of_static_ptrs_in_nongc ll_static_roots_inside = lltype.malloc(lltype.Array(llmemory.Address), - len(self.addresses_of_static_ptrs_in_nongc), + len(addresses_of_static_ptrs_in_nongc), immortal=True) - for i in range(len(self.addresses_of_static_ptrs_in_nongc)): - ll_static_roots_inside[i] = self.addresses_of_static_ptrs_in_nongc[i] + for i in range(len(addresses_of_static_ptrs_in_nongc)): + ll_static_roots_inside[i] = addresses_of_static_ptrs_in_nongc[i] ll_instance.inst_static_root_start = llmemory.cast_ptr_to_adr(ll_static_roots_inside) + llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address)) ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(ll_static_roots_inside) @@ -511,10 +406,10 @@ type_id = self.get_type_id(TYPE) c_type_id = rmodel.inputconst(lltype.Signed, type_id) - info = self.type_info_list[type_id] + info = self.layoutbuilder.type_info_list[type_id] c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"]) - c_has_finalizer = rmodel.inputconst( - lltype.Bool, bool(self.finalizer_funcptr_for_type(TYPE))) + has_finalizer = bool(self.finalizer_funcptr_for_type(TYPE)) + c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer) if not op.opname.endswith('_varsize'): #malloc_ptr = self.malloc_fixedsize_ptr @@ -594,7 +489,7 @@ type_id = self.get_type_id(WEAKREF) c_type_id = rmodel.inputconst(lltype.Signed, type_id) - info = self.type_info_list[type_id] + info = self.layoutbuilder.type_info_list[type_id] c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"]) malloc_ptr = self.malloc_fixedsize_ptr c_has_finalizer = rmodel.inputconst(lltype.Bool, False) @@ -653,38 +548,57 @@ ## hop.genop("direct_call", [self.pop_root_ptr]) ## #hop.genop("gc_reload_possibly_moved", [var]) -# XXX copied and modified from lltypelayout.py -def offsets_to_gc_pointers(TYPE): - offsets = [] - if isinstance(TYPE, lltype.Struct): - for name in TYPE._names: - FIELD = getattr(TYPE, name) - if isinstance(FIELD, lltype.Array): - continue # skip inlined array - baseofs = llmemory.offsetof(TYPE, name) - suboffsets = offsets_to_gc_pointers(FIELD) - for s in suboffsets: - try: - knownzero = s == 0 - except TypeError: - knownzero = False - if knownzero: - offsets.append(baseofs) - else: - offsets.append(baseofs + s) - # sanity check - #ex = lltype.Ptr(TYPE)._example() - #adr = llmemory.cast_ptr_to_adr(ex) - #for off in offsets: - # (adr + off) - elif isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': - offsets.append(0) - return offsets - -def weakpointer_offset(TYPE): - if TYPE == WEAKREF: - return llmemory.offsetof(WEAKREF, "weakptr") - return -1 + +class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder): + + def __init__(self, transformer): + super(TransformerLayoutBuilder, self).__init__() + self.transformer = transformer + self.offsettable_cache = {} + + def make_finalizer_funcptr_for_type(self, TYPE): + rtti = get_rtti(TYPE) + if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'): + destrptr = rtti._obj.destructor_funcptr + DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0] + else: + destrptr = None + DESTR_ARG = None + + assert not type_contains_pyobjs(TYPE), "not implemented" + if destrptr: + def ll_finalizer(addr): + v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) + ll_call_destructor(destrptr, v) + fptr = self.transformer.annotate_helper(ll_finalizer, + [llmemory.Address], + lltype.Void) + else: + fptr = lltype.nullptr(ADDRESS_VOID_FUNC) + return fptr + + def offsets2table(self, offsets, TYPE): + try: + return self.offsettable_cache[TYPE] + except KeyError: + gcdata = self.transformer.gcdata + cachedarray = lltype.malloc(gcdata.OFFSETS_TO_GC_PTR, + len(offsets), immortal=True) + for i, value in enumerate(offsets): + cachedarray[i] = value + self.offsettable_cache[TYPE] = cachedarray + return cachedarray + + def flatten_table(self): + self.can_add_new_types = False + table = lltype.malloc(self.transformer.gcdata.TYPE_INFO_TABLE, + len(self.type_info_list), immortal=True) + for tableentry, newcontent in zip(table, self.type_info_list): + for key, value in newcontent.items(): + setattr(tableentry, key, value) + self.offsettable_cache = None + return table + def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None): if previous_steps is None: @@ -702,45 +616,3 @@ [v] + previous_steps + [c_name, c_null]) elif isinstance(FIELD, lltype.Struct): gen_zero_gc_pointers(FIELD, v, llops, previous_steps + [c_name]) - -def gc_pointers_inside(v, adr): - t = lltype.typeOf(v) - if isinstance(t, lltype.Struct): - for n, t2 in t._flds.iteritems(): - if isinstance(t2, lltype.Ptr) and t2.TO._gckind == 'gc': - yield adr + llmemory.offsetof(t, n) - elif isinstance(t2, (lltype.Array, lltype.Struct)): - for a in gc_pointers_inside(getattr(v, n), adr + llmemory.offsetof(t, n)): - yield a - elif isinstance(t, lltype.Array): - if isinstance(t.OF, lltype.Ptr) and t2._needsgc(): - for i in range(len(v.items)): - yield adr + llmemory.itemoffsetof(t, i) - elif isinstance(t.OF, lltype.Struct): - for i in range(len(v.items)): - for a in gc_pointers_inside(v.items[i], adr + llmemory.itemoffsetof(t, i)): - yield a - - -########## weakrefs ########## -# framework: weakref objects are small structures containing only an address - -WEAKREF = lltype.GcStruct("weakref", ("weakptr", llmemory.Address)) -WEAKREFPTR = lltype.Ptr(WEAKREF) -sizeof_weakref= llmemory.sizeof(WEAKREF) -empty_weakref = lltype.malloc(WEAKREF, immortal=True) -empty_weakref.weakptr = llmemory.NULL - -def ll_weakref_deref(wref): - wref = llmemory.cast_weakrefptr_to_ptr(WEAKREFPTR, wref) - return wref.weakptr - -def convert_weakref_to(targetptr): - # Prebuilt weakrefs don't really need to be weak at all, - # but we need to emulate the structure expected by ll_weakref_deref(). - if not targetptr: - return empty_weakref - else: - link = lltype.malloc(WEAKREF, immortal=True) - link.weakptr = llmemory.cast_ptr_to_adr(targetptr) - return link Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/refcounting.py Sun Oct 7 18:56:37 2007 @@ -6,7 +6,6 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.translator.backendopt.support import var_needsgc from pypy.rpython import rmodel -from pypy.rpython.memory import lladdress from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.rbuiltin import gen_cast @@ -73,11 +72,11 @@ llop.cpy_free(lltype.Void, adr) mh = mallocHelpers() - mh.allocate = lladdress.raw_malloc + mh.allocate = llmemory.raw_malloc def ll_malloc_fixedsize(size): size = gc_header_offset + size result = mh._ll_malloc_fixedsize(size) - lladdress.raw_memclear(result, size) + llmemory.raw_memclear(result, size) result += gc_header_offset return result def ll_malloc_varsize_no_length(length, size, itemsize): @@ -88,7 +87,7 @@ except OverflowError: raise MemoryError() result = mh._ll_malloc_fixedsize(tot_size) - lladdress.raw_memclear(result, tot_size) + llmemory.raw_memclear(result, tot_size) result += gc_header_offset return result mh.ll_malloc_varsize_no_length = ll_malloc_varsize_no_length Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctransform/transform.py Sun Oct 7 18:56:37 2007 @@ -18,7 +18,6 @@ from pypy.rpython.rbuiltin import gen_cast from pypy.rlib.rarithmetic import ovfcheck import sets, os, sys -from pypy.rpython.memory import lladdress from pypy.rpython.lltypesystem.lloperation import llop def var_ispyobj(var): @@ -404,7 +403,7 @@ result = mh.allocate(tot_size) if not result: raise MemoryError() - lladdress.raw_memclear(result, tot_size) + llmemory.raw_memclear(result, tot_size) return result mh.ll_malloc_varsize_no_length_zero = _ll_malloc_varsize_no_length_zero @@ -416,7 +415,7 @@ super(GCTransformer, self).__init__(translator, inline=inline) mh = mallocHelpers() - mh.allocate = lladdress.raw_malloc + mh.allocate = llmemory.raw_malloc ll_raw_malloc_fixedsize = mh._ll_malloc_fixedsize ll_raw_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length ll_raw_malloc_varsize = mh.ll_malloc_varsize Added: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctypelayout.py ============================================================================== --- (empty file) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gctypelayout.py Sun Oct 7 18:56:37 2007 @@ -0,0 +1,224 @@ +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.memory.gctransform.support import find_gc_ptrs_in_type + + +class TypeLayoutBuilder(object): + can_add_new_types = True + + def __init__(self): + dummy = {"weakptrofs": -1, + "ofstolength": -1} + self.type_info_list = [dummy] # don't use typeid 0, helps debugging + self.id_of_type = {} # {LLTYPE: type_id} + self.seen_roots = {} + self.static_gc_roots = [] + self.addresses_of_static_ptrs_in_nongc = [] + self.finalizer_funcptrs = {} + + def get_type_id(self, TYPE): + try: + return self.id_of_type[TYPE] + except KeyError: + assert self.can_add_new_types + assert isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)) + # Record the new type_id description as a small dict for now. + # It will be turned into a Struct("type_info") in finish() + type_id = len(self.type_info_list) + info = {} + self.type_info_list.append(info) + self.id_of_type[TYPE] = type_id + offsets = offsets_to_gc_pointers(TYPE) + info["ofstoptrs"] = self.offsets2table(offsets, TYPE) + info["finalyzer"] = self.make_finalizer_funcptr_for_type(TYPE) + info["weakptrofs"] = weakpointer_offset(TYPE) + if not TYPE._is_varsize(): + info["isvarsize"] = False + info["fixedsize"] = llmemory.sizeof(TYPE) + info["ofstolength"] = -1 + else: + info["isvarsize"] = True + info["fixedsize"] = llmemory.sizeof(TYPE, 0) + if isinstance(TYPE, lltype.Struct): + ARRAY = TYPE._flds[TYPE._arrayfld] + ofs1 = llmemory.offsetof(TYPE, TYPE._arrayfld) + info["ofstolength"] = ofs1 + llmemory.ArrayLengthOffset(ARRAY) + if ARRAY.OF != lltype.Void: + info["ofstovar"] = ofs1 + llmemory.itemoffsetof(ARRAY, 0) + else: + info["fixedsize"] = ofs1 + llmemory.sizeof(lltype.Signed) + if ARRAY._hints.get('isrpystring'): + info["fixedsize"] = llmemory.sizeof(TYPE, 1) + else: + ARRAY = TYPE + info["ofstolength"] = llmemory.ArrayLengthOffset(ARRAY) + if ARRAY.OF != lltype.Void: + info["ofstovar"] = llmemory.itemoffsetof(TYPE, 0) + else: + info["fixedsize"] = llmemory.ArrayLengthOffset(ARRAY) + llmemory.sizeof(lltype.Signed) + assert isinstance(ARRAY, lltype.Array) + if ARRAY.OF != lltype.Void: + offsets = offsets_to_gc_pointers(ARRAY.OF) + info["varofstoptrs"] = self.offsets2table(offsets, ARRAY.OF) + info["varitemsize"] = llmemory.sizeof(ARRAY.OF) + else: + info["varofstoptrs"] = self.offsets2table((), lltype.Void) + info["varitemsize"] = llmemory.sizeof(ARRAY.OF) + return type_id + + def offsets2table(self, offsets, TYPE): + return offsets + + def finalizer_funcptr_for_type(self, TYPE): + if TYPE in self.finalizer_funcptrs: + return self.finalizer_funcptrs[TYPE] + fptr = self.make_finalizer_funcptr_for_type(TYPE) + self.finalizer_funcptrs[TYPE] = fptr + return fptr + + def make_finalizer_funcptr_for_type(self, TYPE): + return None # must be overridden for proper finalizer support + + def q_is_varsize(self, typeid): + assert typeid > 0 + return self.type_info_list[typeid]["isvarsize"] + + def q_finalyzer(self, typeid): + assert typeid > 0 + return self.type_info_list[typeid]["finalyzer"] + + def q_offsets_to_gc_pointers(self, typeid): + assert typeid > 0 + return self.type_info_list[typeid]["ofstoptrs"] + + def q_fixed_size(self, typeid): + assert typeid > 0 + return self.type_info_list[typeid]["fixedsize"] + + def q_varsize_item_sizes(self, typeid): + assert typeid > 0 + return self.type_info_list[typeid]["varitemsize"] + + def q_varsize_offset_to_variable_part(self, typeid): + assert typeid > 0 + return self.type_info_list[typeid]["ofstovar"] + + def q_varsize_offset_to_length(self, typeid): + assert typeid > 0 + return self.type_info_list[typeid]["ofstolength"] + + def q_varsize_offsets_to_gcpointers_in_var_part(self, typeid): + assert typeid > 0 + return self.type_info_list[typeid]["varofstoptrs"] + + def q_weakpointer_offset(self, typeid): + assert typeid > 0 + return self.type_info_list[typeid]["weakptrofs"] + + def get_query_functions(self): + return (self.q_is_varsize, + self.q_finalyzer, + self.q_offsets_to_gc_pointers, + self.q_fixed_size, + self.q_varsize_item_sizes, + self.q_varsize_offset_to_variable_part, + self.q_varsize_offset_to_length, + self.q_varsize_offsets_to_gcpointers_in_var_part, + self.q_weakpointer_offset) + + def consider_constant(self, TYPE, value, gc): + if value is not lltype.top_container(value): + return + if id(value) in self.seen_roots: + return + self.seen_roots[id(value)] = True + + if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): + typeid = self.get_type_id(TYPE) + hdr = gc.gcheaderbuilder.new_header(value) + adr = llmemory.cast_ptr_to_adr(hdr) + gc.init_gc_object_immortal(adr, typeid) + + if find_gc_ptrs_in_type(TYPE): + adr = llmemory.cast_ptr_to_adr(value._as_ptr()) + if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): + self.static_gc_roots.append(adr) + else: + for a in gc_pointers_inside(value, adr): + self.addresses_of_static_ptrs_in_nongc.append(a) + +# ____________________________________________________________ +# +# Helpers to discover GC pointers inside structures + +def offsets_to_gc_pointers(TYPE): + offsets = [] + if isinstance(TYPE, lltype.Struct): + for name in TYPE._names: + FIELD = getattr(TYPE, name) + if isinstance(FIELD, lltype.Array): + continue # skip inlined array + baseofs = llmemory.offsetof(TYPE, name) + suboffsets = offsets_to_gc_pointers(FIELD) + for s in suboffsets: + try: + knownzero = s == 0 + except TypeError: + knownzero = False + if knownzero: + offsets.append(baseofs) + else: + offsets.append(baseofs + s) + # sanity check + #ex = lltype.Ptr(TYPE)._example() + #adr = llmemory.cast_ptr_to_adr(ex) + #for off in offsets: + # (adr + off) + elif isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': + offsets.append(0) + return offsets + +def weakpointer_offset(TYPE): + if TYPE == WEAKREF: + return llmemory.offsetof(WEAKREF, "weakptr") + return -1 + +def gc_pointers_inside(v, adr): + t = lltype.typeOf(v) + if isinstance(t, lltype.Struct): + for n, t2 in t._flds.iteritems(): + if isinstance(t2, lltype.Ptr) and t2.TO._gckind == 'gc': + yield adr + llmemory.offsetof(t, n) + elif isinstance(t2, (lltype.Array, lltype.Struct)): + for a in gc_pointers_inside(getattr(v, n), adr + llmemory.offsetof(t, n)): + yield a + elif isinstance(t, lltype.Array): + if isinstance(t.OF, lltype.Ptr) and t2._needsgc(): + for i in range(len(v.items)): + yield adr + llmemory.itemoffsetof(t, i) + elif isinstance(t.OF, lltype.Struct): + for i in range(len(v.items)): + for a in gc_pointers_inside(v.items[i], adr + llmemory.itemoffsetof(t, i)): + yield a + +########## weakrefs ########## +# framework: weakref objects are small structures containing only an address + +WEAKREF = lltype.GcStruct("weakref", ("weakptr", llmemory.Address)) +WEAKREFPTR = lltype.Ptr(WEAKREF) +sizeof_weakref= llmemory.sizeof(WEAKREF) +empty_weakref = lltype.malloc(WEAKREF, immortal=True) +empty_weakref.weakptr = llmemory.NULL + +def ll_weakref_deref(wref): + wref = llmemory.cast_weakrefptr_to_ptr(WEAKREFPTR, wref) + return wref.weakptr + +def convert_weakref_to(targetptr): + # Prebuilt weakrefs don't really need to be weak at all, + # but we need to emulate the structure expected by ll_weakref_deref(). + if not targetptr: + return empty_weakref + else: + link = lltype.malloc(WEAKREF, immortal=True) + link.weakptr = llmemory.cast_ptr_to_adr(targetptr) + return link Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcwrapper.py Sun Oct 7 18:56:37 2007 @@ -1,346 +1,111 @@ -from pypy.annotation.annrpython import RPythonAnnotator -from pypy.rpython.rtyper import RPythonTyper from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython.memory.support import get_address_linked_list, INT_SIZE -from pypy.rpython.memory.lladdress import raw_malloc, raw_free, NULL -from pypy.rpython.memory import lltypelayout -from pypy.rpython.memory import lltypesimulation -from pypy.rpython.memory import gc -from pypy.rpython.memory.convertlltype import FlowGraphConstantConverter - -class QueryTypes(object): - def __init__(self): - self.types = [] - self.type_to_typeid = {} - - def get_typeid(self, TYPE, nonewtype=False): - if TYPE not in self.type_to_typeid: - if nonewtype: - raise Exception, "unknown type: %s" % TYPE - index = len(self.types) - self.type_to_typeid[TYPE] = index - self.types.append(TYPE) - return index - typeid = self.type_to_typeid[TYPE] - return typeid - - def create_query_functions(self): - from pypy.rpython.lltypesystem import rstr - _is_varsize = [] - _finalizers = [] - _offsets_to_gc_pointers = [] - _fixed_size = [] - _varsize_item_sizes = [] - _varsize_offset_to_variable_part = [] - _varsize_offset_to_length = [] - _varsize_offsets_to_gcpointers_in_var_part = [] - tttid = zip(*zip(*self.type_to_typeid.items())[::-1]) - tttid.sort() - tttid = zip(*zip(*tttid)[::-1]) - for TYPE, typeid in tttid: - varsize = self.is_varsize(typeid) - _is_varsize.append(varsize) - _finalizers.append(None) - _offsets_to_gc_pointers.append(self.offsets_to_gc_pointers(typeid)) - _fixed_size.append(self.fixed_size(typeid)) - if varsize: - _varsize_item_sizes.append(self.varsize_item_sizes(typeid)) - _varsize_offset_to_variable_part.append( - self.varsize_offset_to_variable_part(typeid)) - _varsize_offset_to_length.append( - self.varsize_offset_to_length(typeid)) - _varsize_offsets_to_gcpointers_in_var_part.append( - lltypelayout.varsize_offsets_to_gcpointers_in_var_part(TYPE)) - else: - _varsize_item_sizes.append(0) - _varsize_offset_to_variable_part.append(0) - _varsize_offset_to_length.append(0) - _varsize_offsets_to_gcpointers_in_var_part.append([]) - # trick to make the annotator see that the list can contain functions: - _finalizers.append(lambda addr: None) - def is_varsize(typeid): - return _is_varsize[typeid] - def getfinalizer(typeid): - return _finalizers[typeid] - def offsets_to_gc_pointers(typeid): - return _offsets_to_gc_pointers[typeid] - def fixed_size(typeid): - return _fixed_size[typeid] - def varsize_item_sizes(typeid): - return _varsize_item_sizes[typeid] - def varsize_offset_to_variable_part(typeid): - return _varsize_offset_to_variable_part[typeid] - def varsize_offset_to_length(typeid): - return _varsize_offset_to_length[typeid] - def varsize_offsets_to_gcpointers_in_var_part(typeid): - return _varsize_offsets_to_gcpointers_in_var_part[typeid] - def weakpointer_offset(typeid): - return -1 - return (is_varsize, getfinalizer, offsets_to_gc_pointers, fixed_size, - varsize_item_sizes, varsize_offset_to_variable_part, - varsize_offset_to_length, - varsize_offsets_to_gcpointers_in_var_part, - weakpointer_offset) - - def is_varsize(self, typeid): - assert typeid >= 0 - TYPE = self.types[typeid] - return (isinstance(TYPE, lltype.Array) or - (isinstance(TYPE, lltype.Struct) and - TYPE._arrayfld is not None)) - - def getfinalizer(self, typeid): - return None - - def offsets_to_gc_pointers(self, typeid): - assert typeid >= 0 - return lltypelayout.offsets_to_gc_pointers(self.types[typeid]) - - def fixed_size(self, typeid): - assert typeid >= 0 - return lltypelayout.get_fixed_size(self.types[typeid]) - - def varsize_item_sizes(self, typeid): - assert typeid >= 0 - if self.is_varsize(typeid): - return lltypelayout.get_variable_size(self.types[typeid]) - else: - return 0 - - def varsize_offset_to_variable_part(self, typeid): - assert typeid >= 0 - if self.is_varsize(typeid): - return lltypelayout.get_fixed_size(self.types[typeid]) - else: - return 0 - - def varsize_offset_to_length(self, typeid): - assert typeid >= 0 - if self.is_varsize(typeid): - return lltypelayout.varsize_offset_to_length(self.types[typeid]) - else: - return 0 - - def varsize_offsets_to_gcpointers_in_var_part(self, typeid): - assert typeid >= 0 - if self.is_varsize(typeid): - return lltypelayout.varsize_offsets_to_gcpointers_in_var_part( - self.types[typeid]) - else: - return 0 - - def weakpointer_offset(self, typeid): - return -1 +from pypy.rpython.memory.support import get_address_linked_list +from pypy.rpython.memory import gctypelayout +from pypy.objspace.flow.model import Constant - def get_setup_query_functions(self): - return (self.is_varsize, self.getfinalizer, - self.offsets_to_gc_pointers, self.fixed_size, - self.varsize_item_sizes, self.varsize_offset_to_variable_part, - self.varsize_offset_to_length, - self.varsize_offsets_to_gcpointers_in_var_part, - self.weakpointer_offset) - - -def getfunctionptr(annotator, graphfunc): - """Make a functionptr from the given Python function.""" - graph = annotator.bookkeeper.getdesc(graphfunc).getuniquegraph() - llinputs = [v.concretetype for v in graph.getargs()] - lloutput = graph.getreturnvar().concretetype - FT = lltype.FuncType(llinputs, lloutput) - _callable = graphfunc - return lltypesimulation.functionptr(FT, graphfunc.func_name, - graph=graph, _callable=_callable) +class GCManagedHeap(object): -class GcWrapper(object): def __init__(self, llinterp, flowgraphs, gc_class): - self.query_types = QueryTypes() - self.AddressLinkedList = get_address_linked_list(3, hackishpop=True) - # XXX there might me GCs that have headers that depend on the type - # therefore we have to change the query functions to annotatable ones - # later + self.AddressLinkedList = get_address_linked_list(10, hackishpop=True) self.gc = gc_class(self.AddressLinkedList) - self.gc.set_query_functions(*self.query_types.get_setup_query_functions()) - fgcc = FlowGraphConstantConverter(flowgraphs, self.gc, self.query_types) - fgcc.convert() - self.gc.set_query_functions(*self.query_types.create_query_functions()) + self.gc.get_roots = self.get_roots_from_llinterp self.llinterp = llinterp - self.gc.get_roots = self.get_roots - self.constantroots = fgcc.cvter.constantroots - self.pseudo_root_pointers = NULL - self.roots = [] + self.constantroots = [] + self.prepare_graphs(flowgraphs) self.gc.setup() - - def get_arg_malloc(self, TYPE, size=0): - typeid = self.query_types.get_typeid(TYPE, nonewtype=True) - return [typeid, size] - - def get_funcptr_malloc(self): - return self.llinterp.heap.functionptr(gc.gc_interface["malloc"], "malloc", - _callable=self.gc.malloc) - - def adjust_result_malloc(self, address, TYPE, size=0): - result = lltypesimulation.init_object_on_address(address, TYPE, size) - self.update_changed_addresses() - return result - - - def needs_write_barrier(self, TYPE): - return (hasattr(self.gc, "write_barrier") and - isinstance(TYPE, lltype.Ptr) and - isinstance(TYPE.TO, (lltype.GcStruct, lltype.GcArray))) - - def get_arg_write_barrier(self, obj, index_or_field, item): - #XXX: quick hack to get the correct addresses, fix later - layout = lltypelayout.get_layout(lltype.typeOf(obj).TO) - if isinstance(lltype.typeOf(obj).TO, lltype.Array): - assert isinstance(index_or_field, int) - offset = layout[0] + layout[1] * index_or_field - addr_to = obj._address + layout[0] + index_or_field * layout[1] - return item._address, addr_to, obj._address - else: - offset = layout[index_or_field] - addr_to = obj._address + offset - return item._address, addr_to, obj._address - - - def get_funcptr_write_barrier(self): - return self.llinterp.heap.functionptr(gc.gc_interface["write_barrier"], - "write_barrier", - _callable=self.gc.write_barrier) - - - def update_changed_addresses(self): - for i, root in enumerate(self.roots): - root.__dict__['_address'] = self.pseudo_root_pointers.address[i] + def prepare_graphs(self, flowgraphs): + layoutbuilder = DirectRunLayoutBuilder() + self.get_type_id = layoutbuilder.get_type_id + self.gc.set_query_functions(*layoutbuilder.get_query_functions()) + + constants = collect_constants(flowgraphs) + for obj in constants: + TYPE = lltype.typeOf(obj) + layoutbuilder.consider_constant(TYPE, obj, self.gc) + + sizeofaddr = llmemory.sizeof(llmemory.Address) + for addr in layoutbuilder.static_gc_roots: + addrofaddr = llmemory.raw_malloc(sizeofaddr) + addrofaddr.address[0] = addr + self.constantroots.append(addrofaddr) + self.constantroots += layoutbuilder.addresses_of_static_ptrs_in_nongc def get_roots_from_llinterp(self): - if self.pseudo_root_pointers != NULL: - raw_free(self.pseudo_root_pointers) - roots = [r for r in self.llinterp.find_roots() - if isinstance(r._TYPE.TO, - (lltype.GcStruct, lltype.GcArray))] - self.roots = roots + self.constantroots - self.roots = [r for r in self.roots - if isinstance(r._TYPE.TO, - (lltype.Struct, lltype.Array))] - if len(self.roots) == 0: - self.pseudo_root_pointers = NULL - else: - self.pseudo_root_pointers = raw_malloc(len(self.roots) * INT_SIZE) - return self.roots - - def get_roots(self): - self.get_roots_from_llinterp() + sizeofaddr = llmemory.sizeof(llmemory.Address) ll = self.AddressLinkedList() - for i, root in enumerate(self.roots): - self.pseudo_root_pointers.address[i] = root._address - ll.append(self.pseudo_root_pointers + INT_SIZE * i) + for addrofaddr in self.constantroots: + ll.append(addrofaddr) + for root in self.llinterp.find_roots(): + if lltype.typeOf(root).TO._gckind == 'gc': + addrofaddr = llmemory.raw_malloc(sizeofaddr) + addrofaddr.address[0] = llmemory.cast_ptr_to_adr(root) + ll.append(addrofaddr) return ll -class AnnotatingGcWrapper(GcWrapper): - def __init__(self, llinterp, flowgraphs, gc_class): - super(AnnotatingGcWrapper, self).__init__(llinterp, flowgraphs, gc_class) - # tell the real-built gc to free its memory as it is only used for - # initialisation - self.gc.free_memory() - self.annotate_rtype_gc() - - def annotate_rtype_gc(self): - # annotate and specialize functions - gc_class = self.gc.__class__ - AddressLinkedList = self.AddressLinkedList - def instantiate_linked_list(): - return AddressLinkedList() - f1, f2, f3, f4, f5, f6, f7, f8, f9 = self.query_types.create_query_functions() - the_gc = gc_class(AddressLinkedList) - def instantiate_gc(): - the_gc.set_query_functions(f1, f2, f3, f4, f5, f6, f7, f8, f9) - the_gc.setup() - return the_gc - func, dummy_get_roots1, dummy_get_roots2 = gc.get_dummy_annotate( - the_gc, self.AddressLinkedList) - self.gc.get_roots = dummy_get_roots1 - a = RPythonAnnotator() - a.build_types(instantiate_gc, []) - a.build_types(func, []) - a.build_types(instantiate_linked_list, []) - typer = RPythonTyper(a) - typer.specialize() - self.annotator = a - - # convert constants - fgcc = FlowGraphConstantConverter(a.translator.graphs) - fgcc.convert() - self.malloc_graph = a.bookkeeper.getdesc(self.gc.malloc.im_func).getuniquegraph() - self.write_barrier_graph = a.bookkeeper.getdesc(self.gc.write_barrier.im_func).getuniquegraph() - - # create a gc via invoking instantiate_gc - self.gcptr = self.llinterp.eval_graph( - a.bookkeeper.getdesc(instantiate_gc).getuniquegraph()) - GETROOTS_FUNCTYPE = lltype.typeOf( - getfunctionptr(a, dummy_get_roots1)).TO - setattr(self.gcptr, "inst_get_roots", - lltypesimulation.functionptr(GETROOTS_FUNCTYPE, "get_roots", - _callable=self.get_roots)) - #get funcptrs neccessary to build the result of get_roots - self.instantiate_linked_list = getfunctionptr( - a, instantiate_linked_list) - self.append_linked_list = getfunctionptr( - a, AddressLinkedList.append.im_func) - self.pop_linked_list = getfunctionptr( - a, AddressLinkedList.pop.im_func) - self.gc.get_roots = None - self.translator = a.translator -## a.translator.view() - - def get_arg_malloc(self, TYPE, size=0): - typeid = self.query_types.get_typeid(TYPE, nonewtype=True) - return [self.gcptr, typeid, size] - - def get_funcptr_malloc(self): - FUNC = gc.gc_interface["malloc"] - FUNC = lltype.FuncType([lltype.typeOf(self.gcptr)] + list(FUNC.ARGS), FUNC.RESULT) - return self.llinterp.heap.functionptr(FUNC, "malloc", - _callable=self.gc.malloc, - graph=self.malloc_graph) - - def adjust_result_malloc(self, address, TYPE, size=0): - result = lltypesimulation.init_object_on_address(address, TYPE, size) - self.update_changed_addresses() - return result - - def get_arg_write_barrier(self, obj, index_or_field, item): - #XXX: quick hack to get the correct addresses, fix later - layout = lltypelayout.get_layout(lltype.typeOf(obj).TO) - if isinstance(lltype.typeOf(obj).TO, lltype.Array): - assert isinstance(index_or_field, int) - offset = layout[0] + layout[1] * index_or_field - addr_to = obj._address + layout[0] + index_or_field * layout[1] - return self.gcptr, item._address, addr_to, obj._address - else: - offset = layout[index_or_field] - addr_to = obj._address + offset - return self.gcptr, item._address, addr_to, obj._address - - def get_funcptr_write_barrier(self): - FUNC = gc.gc_interface["write_barrier"] - FUNC = lltype.FuncType([lltype.typeOf(self.gcptr)] + list(FUNC.ARGS), FUNC.RESULT) - return self.llinterp.heap.functionptr(FUNC, - "write_barrier", - _callable=self.gc.write_barrier, - graph=self.write_barrier_graph) - - - def get_roots(self): - # call the llinterpreter to construct the result in a suitable way - self.get_roots_from_llinterp() - ll = self.llinterp.active_frame.op_direct_call( - self.instantiate_linked_list) - for i, root in enumerate(self.roots): - self.pseudo_root_pointers.address[i] = root._address - self.llinterp.active_frame.op_direct_call( - self.append_linked_list, ll, - self.pseudo_root_pointers + INT_SIZE * i) - return ll + # ____________________________________________________________ + # + # Interface for the llinterp + # + def malloc(self, TYPE, n=None, flavor='gc', zero=False): + if flavor == 'gc': + typeid = self.get_type_id(TYPE) + addr = self.gc.malloc(typeid, n) + return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) + else: + return lltype.malloc(TYPE, n, flavor=flavor, zero=zero) + + def setfield(self, obj, fieldname, fieldvalue): + # XXX use write_barrier - but we need the address of the GcStruct + setattr(obj, fieldname, fieldvalue) + + def setarrayitem(self, array, index, newitem): + # XXX use write_barrier - but we need the address of the GcStruct + array[index] = newitem + + # ____________________________________________________________ + + +class DirectRunLayoutBuilder(gctypelayout.TypeLayoutBuilder): + + def make_finalizer_funcptr_for_type(self, TYPE): + return None # XXX + + +def collect_constants(graphs): + constants = {} + def collect_args(args): + for arg in args: + if (isinstance(arg, Constant) and + arg.concretetype is not lltype.Void): + reccollect(constants, arg.value) + for graph in graphs: + for block in graph.iterblocks(): + collect_args(block.inputargs) + for op in block.operations: + collect_args(op.args) + for link in graph.iterlinks(): + collect_args(link.args) + if hasattr(link, "llexitcase"): + reccollect(constants, link.llexitcase) + return constants + +def reccollect(constants, llvalue): + T = lltype.typeOf(llvalue) + if isinstance(T, lltype.Ptr) and llvalue and llvalue._obj not in constants: + constants[llvalue._obj] = True + TYPE = T.TO + if isinstance(TYPE, lltype.Struct): + for name in TYPE._names: + reccollect(constants, getattr(llvalue, name)) + elif isinstance(TYPE, lltype.Array): + for llitem in llvalue: + reccollect(constants, llitem) + parent, parentindex = lltype.parentlink(llvalue._obj) + if parent is not None: + reccollect(constants, parent._as_ptr()) + +def prepare_graphs_and_create_gc(llinterp, GCClass): + flowgraphs = llinterp.typer.annotator.translator.graphs[:] + llinterp.heap = GCManagedHeap(llinterp, flowgraphs, GCClass) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/lltypelayout.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/lltypelayout.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/lltypelayout.py Sun Oct 7 18:56:37 2007 @@ -108,39 +108,3 @@ return 0 else: raise Exception("unknown offset type %r"%offset) - -# _____________________________________________________________________________ -# the following functions are used to find contained pointers - - -def offsets_to_gc_pointers(TYPE): - if isinstance(TYPE, lltype.Struct): - offsets = [] - for name in TYPE._names: - FIELD = getattr(TYPE, name) - if isinstance(FIELD, lltype.Ptr) and FIELD.TO._gckind == 'gc': - offsets.append(llmemory.offsetof(TYPE, name)) - elif isinstance(FIELD, lltype.Struct): - suboffsets = offsets_to_gc_pointers(FIELD) - offsets += [s + llmemory.offsetof(TYPE, name) for s in suboffsets] - return offsets - return [] - -def varsize_offset_to_length(TYPE): - if isinstance(TYPE, lltype.Array): - return 0 - elif isinstance(TYPE, lltype.Struct): - layout = get_layout(TYPE) - return layout[TYPE._arrayfld] - -def varsize_offsets_to_gcpointers_in_var_part(TYPE): - if isinstance(TYPE, lltype.Array): - if isinstance(TYPE.OF, lltype.Ptr): - return [0] - elif isinstance(TYPE.OF, lltype.Struct): - return offsets_to_gc_pointers(TYPE.OF) - return [] - elif isinstance(TYPE, lltype.Struct): - return varsize_offsets_to_gcpointers_in_var_part(getattr(TYPE, - TYPE._arrayfld)) - Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/support.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/support.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/support.py Sun Oct 7 18:56:37 2007 @@ -1,9 +1,6 @@ from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython.memory.lltypelayout import sizeof from pypy.rlib.objectmodel import free_non_gc_object -INT_SIZE = sizeof(lltype.Signed) - DEFAULT_CHUNK_SIZE = 1019 def get_address_linked_list(chunk_size=DEFAULT_CHUNK_SIZE, hackishpop=False): @@ -16,8 +13,6 @@ llmemory.Address, chunk_size)))) null_chunk = lltype.nullptr(CHUNK) - SIZEOF_CHUNK = llmemory.sizeof(CHUNK) - class FreeList(object): _alloc_flavor_ = "raw" @@ -26,10 +21,7 @@ def get(self): if not self.free_list: - from pypy.rpython.memory.lladdress import raw_memclear - r = lltype.malloc(CHUNK, flavor="raw") - raw_memclear(llmemory.cast_ptr_to_adr(r), SIZEOF_CHUNK) - return r + return lltype.malloc(CHUNK, flavor="raw") result = self.free_list self.free_list = result.previous Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_gc.py Sun Oct 7 18:56:37 2007 @@ -1,48 +1,36 @@ import py import sys -from pypy.annotation import model as annmodel -from pypy.annotation.annrpython import RPythonAnnotator -from pypy.rpython.rtyper import RPythonTyper -from pypy.rpython.memory.gc import GCError, MarkSweepGC, SemiSpaceGC -from pypy.rpython.memory.gc import DeferredRefcountingGC, DummyGC -from pypy.rpython.memory.support import INT_SIZE -from pypy.rpython.memory import support -from pypy.rpython.memory.lladdress import raw_malloc, raw_free, NULL -from pypy.rpython.memory.simulator import MemorySimulatorError -from pypy.rpython.memory import gclltype -from pypy.rpython.memory.test.test_llinterpsim import interpret -from pypy.rpython.memory.lladdress import simulator -from pypy.rlib.objectmodel import free_non_gc_object - -def setup_module(mod): - def stdout_ignore_ll_functions(msg): - strmsg = str(msg) - if "evaluating" in strmsg and "ll_" in strmsg: - return - print >>sys.stdout, strmsg - mod.logstate = py.log._getstate() - py.log.setconsumer("llinterp", py.log.STDOUT) - py.log.setconsumer("llinterp frame", stdout_ignore_ll_functions) - py.log.setconsumer("llinterp operation", None) +#from pypy.rpython.memory.support import INT_SIZE +from pypy.rpython.memory import gcwrapper +from pypy.rpython.test.test_llinterp import get_interpreter + + +def stdout_ignore_ll_functions(msg): + strmsg = str(msg) + if "evaluating" in strmsg and "ll_" in strmsg: + return + print >>sys.stdout, strmsg class GCTest(object): def setup_class(cls): - gclltype.prepare_graphs_and_create_gc = gclltype.create_gc - gclltype.use_gc = cls.GCClass - from pypy.rpython.memory import gc as gcimpl - gcimpl.DEBUG_PRINT = False - + cls._saved_logstate = py.log._getstate() + py.log.setconsumer("llinterp", py.log.STDOUT) + py.log.setconsumer("llinterp frame", stdout_ignore_ll_functions) + py.log.setconsumer("llinterp operation", None) + def teardown_class(cls): - gclltype.prepare_graphs_and_create_gc = gclltype.create_no_gc - gclltype.use_gc = MarkSweepGC - from pypy.rpython.memory import gc as gcimpl - gcimpl.DEBUG_PRINT = True + py.log._setstate(cls._saved_logstate) + + def interpret(self, func, values, **kwds): + interp, graph = get_interpreter(func, values, **kwds) + gcwrapper.prepare_graphs_and_create_gc(interp, self.GCClass) + return interp.eval_graph(graph, values) def test_llinterp_lists(self): - curr = simulator.current_size + #curr = simulator.current_size def malloc_a_lot(): i = 0 while i < 10: @@ -52,12 +40,12 @@ while j < 20: j += 1 a.append(j) - res = interpret(malloc_a_lot, []) - assert simulator.current_size - curr < 16000 * INT_SIZE / 4 - print "size before: %s, size after %s" % (curr, simulator.current_size) + res = self.interpret(malloc_a_lot, []) + #assert simulator.current_size - curr < 16000 * INT_SIZE / 4 + #print "size before: %s, size after %s" % (curr, simulator.current_size) def test_llinterp_tuples(self): - curr = simulator.current_size + #curr = simulator.current_size def malloc_a_lot(): i = 0 while i < 10: @@ -68,39 +56,41 @@ while j < 20: j += 1 b.append((1, j, i)) - res = interpret(malloc_a_lot, []) - assert simulator.current_size - curr < 16000 * INT_SIZE / 4 - print "size before: %s, size after %s" % (curr, simulator.current_size) + res = self.interpret(malloc_a_lot, []) + #assert simulator.current_size - curr < 16000 * INT_SIZE / 4 + #print "size before: %s, size after %s" % (curr, simulator.current_size) def test_global_list(self): lst = [] def append_to_list(i, j): lst.append([i] * 50) return lst[j][0] - res = interpret(append_to_list, [0, 0]) + res = self.interpret(append_to_list, [0, 0]) assert res == 0 for i in range(1, 15): - res = interpret(append_to_list, [i, i - 1]) + res = self.interpret(append_to_list, [i, i - 1]) assert res == i - 1 # crashes if constants are not considered roots def test_string_concatenation(self): - curr = simulator.current_size + #curr = simulator.current_size def concat(j): lst = [] for i in range(j): lst.append(str(i)) return len("".join(lst)) - res = interpret(concat, [100]) + res = self.interpret(concat, [100]) assert res == concat(100) - assert simulator.current_size - curr < 16000 * INT_SIZE / 4 + #assert simulator.current_size - curr < 16000 * INT_SIZE / 4 class TestMarkSweepGC(GCTest): - GCClass = MarkSweepGC + from pypy.rpython.memory.gc import MarkSweepGC as GCClass class TestSemiSpaceGC(GCTest): - GCClass = SemiSpaceGC + from pypy.rpython.memory.gc import SemiSpaceGC as GCClass + def setup_class(cls): + py.test.skip("in-progress") class TestDeferredRefcountingGC(GCTest): - GCClass = DeferredRefcountingGC + from pypy.rpython.memory.gc import DeferredRefcountingGC as GCClass def setup_class(cls): py.test.skip("DeferredRefcounting is unmaintained") Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_support.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_support.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_support.py Sun Oct 7 18:56:37 2007 @@ -3,55 +3,60 @@ from pypy.rpython.test.test_llinterp import interpret from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free, NULL class TestAddressLinkedList(object): def test_simple_access(self): - from pypy.rpython.memory.lladdress import raw_malloc, raw_free, NULL AddressLinkedList = get_address_linked_list() - addr = raw_malloc(100) + addr0 = raw_malloc(llmemory.sizeof(lltype.Signed)) + addr1 = raw_malloc(llmemory.sizeof(lltype.Signed)) + addr2 = raw_malloc(llmemory.sizeof(lltype.Signed)) ll = AddressLinkedList() - ll.append(addr) - ll.append(addr + 1) - ll.append(addr + 2) + ll.append(addr0) + ll.append(addr1) + ll.append(addr2) assert ll.non_empty() a = ll.pop() - assert a - addr == 2 + assert a == addr2 assert ll.non_empty() a = ll.pop() - assert a - addr == 1 + assert a == addr1 assert ll.non_empty() a = ll.pop() - assert a == addr + assert a == addr0 assert not ll.non_empty() - ll.append(addr) + ll.append(addr0) ll.delete() ll = AddressLinkedList() - ll.append(addr) - ll.append(addr + 1) - ll.append(addr + 2) + ll.append(addr0) + ll.append(addr1) + ll.append(addr2) ll.delete() - raw_free(addr) + raw_free(addr2) + raw_free(addr1) + raw_free(addr0) def test_big_access(self): - from pypy.rpython.memory.lladdress import raw_malloc, raw_free, NULL AddressLinkedList = get_address_linked_list() - addr = raw_malloc(1) + addrs = [raw_malloc(llmemory.sizeof(lltype.Signed)) + for i in range(3000)] ll = AddressLinkedList() for i in range(3000): print i - ll.append(addr + i) + ll.append(addrs[i]) for i in range(3000)[::-1]: a = ll.pop() - assert a - addr == i + assert a == addrs[i] for i in range(3000): print i - ll.append(addr + i) + ll.append(addrs[i]) for i in range(3000)[::-1]: a = ll.pop() - assert a - addr == i + assert a == addrs[i] ll.delete() - raw_free(addr) - + for addr in addrs: + raw_free(addr) + def test_linked_list_annotate(): AddressLinkedList = get_address_linked_list(60) INT_SIZE = llmemory.sizeof(lltype.Signed) @@ -89,10 +94,7 @@ raw_free(addr) return res - NULL = llmemory.NULL - raw_malloc, raw_free = llmemory.raw_malloc, llmemory.raw_free assert f() - from pypy.rpython.memory.lladdress import raw_malloc, raw_free AddressLinkedList = get_address_linked_list() res = interpret(f, [], malloc_check=False) assert res Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_transformed_gc.py Sun Oct 7 18:56:37 2007 @@ -1,169 +1,17 @@ import py import sys - -## from pypy.annotation import model as annmodel -## from pypy.annotation.annrpython import RPythonAnnotator -## from pypy.rpython.rtyper import RPythonTyper -## from pypy.rpython.memory.gc import GCError, MarkSweepGC, SemiSpaceGC -## from pypy.rpython.memory.gc import DeferredRefcountingGC, DummyGC -## from pypy.rpython.memory import support -## from pypy.rpython.memory.lladdress import raw_malloc, raw_free, NULL -## from pypy.rpython.memory.simulator import MemorySimulatorError -## from pypy.rpython.memory import gclltype -## from pypy.rpython.memory.test.test_llinterpsim import interpret -## from pypy.rpython.memory.lladdress import simulator -## from pypy.rlib.objectmodel import free_non_gc_object - -## def setup_module(mod): -## def stdout_ignore_ll_functions(msg): -## strmsg = str(msg) -## if "evaluating" in strmsg and "ll_" in strmsg: -## return -## print >>sys.stdout, strmsg -## mod.logstate = py.log._getstate() -## py.log.setconsumer("llinterp", py.log.STDOUT) -## py.log.setconsumer("llinterp frame", stdout_ignore_ll_functions) -## py.log.setconsumer("llinterp operation", None) -## gclltype.prepare_graphs_and_create_gc = gclltype.create_gc - -## def teardown_module(mod): -## gclltype.prepare_graphs_and_create_gc = gclltype.create_no_gc - -## class TestMarkSweepGC(object): -## def setup_class(cls): -## cls.prep_old = gclltype.prepare_graphs_and_create_gc -## cls.old = gclltype.use_gc -## gclltype.use_gc = MarkSweepGC - -## def teardown_class(cls): -## gclltype.prepare_graphs_and_create_gc = cls.prep_old.im_func -## gclltype.use_gc = cls.old - -## def test_llinterp_lists(self): -## curr = simulator.current_size -## def malloc_a_lot(): -## i = 0 -## while i < 10: -## i += 1 -## a = [1] * 10 -## j = 0 -## while j < 20: -## j += 1 -## a.append(j) -## res = interpret(malloc_a_lot, []) -## assert simulator.current_size - curr < 16000 * INT_SIZE / 4 -## print "size before: %s, size after %s" % (curr, simulator.current_size) - -## def test_llinterp_tuples(self): -## curr = simulator.current_size -## def malloc_a_lot(): -## i = 0 -## while i < 10: -## i += 1 -## a = (1, 2, i) -## b = [a] * 10 -## j = 0 -## while j < 20: -## j += 1 -## b.append((1, j, i)) -## res = interpret(malloc_a_lot, []) -## assert simulator.current_size - curr < 16000 * INT_SIZE / 4 -## print "size before: %s, size after %s" % (curr, simulator.current_size) - -## def test_global_list(self): -## lst = [] -## def append_to_list(i, j): -## lst.append([i] * 50) -## return lst[j][0] -## res = interpret(append_to_list, [0, 0]) -## assert res == 0 -## for i in range(1, 15): -## res = interpret(append_to_list, [i, i - 1]) -## assert res == i - 1 # crashes if constants are not considered roots - -## def test_string_concatenation(self): -## curr = simulator.current_size -## def concat(j): -## lst = [] -## for i in range(j): -## lst.append(str(i)) -## return len("".join(lst)) -## res = interpret(concat, [100]) -## assert res == concat(100) -## assert simulator.current_size - curr < 16000 * INT_SIZE / 4 - -## class TestMarkSweepGCRunningOnLLinterp(TestMarkSweepGC): -## def setup_class(cls): -## cls.prep_old = gclltype.prepare_graphs_and_create_gc -## gclltype.prepare_graphs_and_create_gc = gclltype.create_gc_run_on_llinterp -## def teardown_class(cls): -## gclltype.prepare_graphs_and_create_gc = cls.prep_old.im_func - -## class TestSemiSpaceGC(TestMarkSweepGC): -## def setup_class(cls): -## gclltype.use_gc = SemiSpaceGC -## cls.old = gclltype.use_gc -## def teardown_class(cls): -## gclltype.use_gc = cls.old - -## class TestSemiSpaceGCRunningOnLLinterp(TestMarkSweepGC): -## def setup_class(cls): -## cls.prep_old = gclltype.prepare_graphs_and_create_gc -## gclltype.prepare_graphs_and_create_gc = gclltype.create_gc_run_on_llinterp -## gclltype.use_gc = SemiSpaceGC -## cls.old = gclltype.use_gc - -## def teardown_class(cls): -## gclltype.prepare_graphs_and_create_gc = cls.prep_old.im_func -## gclltype.use_gc = cls.old - -## class TestDeferredRefcountingGC(TestMarkSweepGC): -## def setup_class(cls): -## gclltype.use_gc = DeferredRefcountingGC -## cls.old = gclltype.use_gc -## def teardown_class(cls): -## gclltype.use_gc = cls.old - - -## class TestDeferredRefcountingGCRunningOnLLinterp(TestMarkSweepGC): -## def setup_class(cls): -## cls.prep_old = gclltype.prepare_graphs_and_create_gc -## gclltype.prepare_graphs_and_create_gc = gclltype.create_gc_run_on_llinterp -## gclltype.use_gc = DeferredRefcountingGC -## cls.old = gclltype.use_gc - -## def teardown_class(cls): -## gclltype.prepare_graphs_and_create_gc = cls.prep_old.im_func -## gclltype.use_gc = cls.old - -## class TestDummyGC(TestMarkSweepGC): -## def setup_class(cls): -## gclltype.use_gc = DummyGC -## cls.old = gclltype.use_gc -## def teardown_class(cls): -## gclltype.use_gc = cls.old - -## class TestDummyGCRunningOnLLinterp(TestMarkSweepGC): -## def setup_class(cls): -## cls.prep_old = gclltype.prepare_graphs_and_create_gc -## gclltype.prepare_graphs_and_create_gc = gclltype.create_gc_run_on_llinterp -## gclltype.use_gc = DummyGC -## cls.old = gclltype.use_gc - -## def teardown_class(cls): -## gclltype.prepare_graphs_and_create_gc = cls.prep_old.im_func -## gclltype.use_gc = cls.old - +import struct from pypy.translator.c import gc from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.memory.gctransform import framework from pypy.rpython.memory.gctransform import stacklessframework from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rpython.memory.support import INT_SIZE from pypy.rpython.memory.gc import X_CLONE, X_POOL, X_POOL_PTR from pypy import conftest +INT_SIZE = struct.calcsize("i") # only for estimates + def rtype(func, inputtypes, specialize=True, gcname='ref', stacklessgc=False): from pypy.translator.translator import TranslationContext Modified: pypy/branch/kill-keepalives-again/pypy/rpython/raddress.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/raddress.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/raddress.py Sun Oct 7 18:56:37 2007 @@ -1,9 +1,8 @@ # rtyping of memory address operations from pypy.annotation.pairtype import pairtype from pypy.annotation import model as annmodel -from pypy.rpython.memory.lladdress import _address from pypy.rpython.lltypesystem.llmemory import NULL, Address, \ - cast_adr_to_int + cast_adr_to_int, fakeaddress from pypy.rpython.rmodel import Repr, IntegerRepr from pypy.rpython.rptr import PtrRepr from pypy.rpython.lltypesystem import lltype @@ -27,7 +26,9 @@ lowleveltype = Address def convert_const(self, value): - assert not isinstance(value, _address) + # note that llarena.fakearenaaddress is not supported as a constant + # in graphs + assert type(value) is fakeaddress return value def ll_str(self, a): Modified: pypy/branch/kill-keepalives-again/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/rbuiltin.py Sun Oct 7 18:56:37 2007 @@ -569,7 +569,6 @@ # _________________________________________________________________ # memory addresses -from pypy.rpython.memory import lladdress from pypy.rpython.lltypesystem import llmemory def rtype_raw_malloc(hop): @@ -594,12 +593,6 @@ v_list = hop.inputargs(llmemory.Address, lltype.Signed) return hop.genop('raw_memclear', v_list) -BUILTIN_TYPER[lladdress.raw_malloc] = rtype_raw_malloc -BUILTIN_TYPER[lladdress.raw_malloc_usage] = rtype_raw_malloc_usage -BUILTIN_TYPER[lladdress.raw_free] = rtype_raw_free -BUILTIN_TYPER[lladdress.raw_memclear] = rtype_raw_memclear -BUILTIN_TYPER[lladdress.raw_memcopy] = rtype_raw_memcopy - BUILTIN_TYPER[llmemory.raw_malloc] = rtype_raw_malloc BUILTIN_TYPER[llmemory.raw_malloc_usage] = rtype_raw_malloc_usage BUILTIN_TYPER[llmemory.raw_free] = rtype_raw_free Modified: pypy/branch/kill-keepalives-again/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/exceptiontransform.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/exceptiontransform.py Sun Oct 7 18:56:37 2007 @@ -7,7 +7,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem import lloperation -from pypy.rpython.memory.lladdress import NULL +from pypy.rpython.lltypesystem.llmemory import NULL from pypy.rpython import rtyper from pypy.rpython import rclass from pypy.rpython.rmodel import inputconst From fijal at codespeak.net Sun Oct 7 23:36:55 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Oct 2007 23:36:55 +0200 (CEST) Subject: [pypy-svn] r47271 - pypy/dist/pypy/doc Message-ID: <20071007213655.7B8A58163@code0.codespeak.net> Author: fijal Date: Sun Oct 7 23:36:54 2007 New Revision: 47271 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: Update todo Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Sun Oct 7 23:36:54 2007 @@ -35,7 +35,6 @@ - rewrite the following rctypes modules using rffi: - - _demo - _ssl - move the following from using old-style C-api @@ -44,3 +43,5 @@ - strtod (RPyton level) - rewrite or kill the extcompiler + + - review CPython regression test suite, enable running tests, fix bugs From fijal at codespeak.net Sun Oct 7 23:37:08 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Oct 2007 23:37:08 +0200 (CEST) Subject: [pypy-svn] r47272 - pypy/dist/pypy/module/_demo Message-ID: <20071007213708.819E18163@code0.codespeak.net> Author: fijal Date: Sun Oct 7 23:37:08 2007 New Revision: 47272 Modified: pypy/dist/pypy/module/_demo/demo.py Log: Move _demo from rctypes to rffi Modified: pypy/dist/pypy/module/_demo/demo.py ============================================================================== --- pypy/dist/pypy/module/_demo/demo.py (original) +++ pypy/dist/pypy/module/_demo/demo.py Sun Oct 7 23:37:08 2007 @@ -2,16 +2,13 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.rpython.rctypes.tool import ctypes_platform -from pypy.rpython.rctypes.tool.libc import libc +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.tool import rffi_platform import sys, math -from ctypes import * -time_t = ctypes_platform.getsimpletype('time_t', '#include ', c_long) +time_t = rffi_platform.getsimpletype('time_t', '#include ', rffi.LONG) -time = libc.time -time.argtypes = [POINTER(time_t)] -time.restype = time_t +time = rffi.llexternal('time', [rffi.VOIDP], time_t, includes=['time.h']) def get(space, name): w_module = space.getbuiltinmodule('_demo') From fijal at codespeak.net Sun Oct 7 23:37:48 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Oct 2007 23:37:48 +0200 (CEST) Subject: [pypy-svn] r47273 - pypy/dist/lib-python/modified-2.4.1/test Message-ID: <20071007213748.306918163@code0.codespeak.net> Author: fijal Date: Sun Oct 7 23:37:47 2007 New Revision: 47273 Added: pypy/dist/lib-python/modified-2.4.1/test/test_bz2.py - copied, changed from r47058, pypy/dist/lib-python/2.4.1/test/test_bz2.py Log: Close file explicitely From fijal at codespeak.net Sun Oct 7 23:38:25 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Oct 2007 23:38:25 +0200 (CEST) Subject: [pypy-svn] r47274 - pypy/dist/pypy/module/struct Message-ID: <20071007213825.B97C88163@code0.codespeak.net> Author: fijal Date: Sun Oct 7 23:38:25 2007 New Revision: 47274 Modified: pypy/dist/pypy/module/struct/app_struct.py Log: Fix docstring Modified: pypy/dist/pypy/module/struct/app_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/app_struct.py (original) +++ pypy/dist/pypy/module/struct/app_struct.py Sun Oct 7 23:38:25 2007 @@ -1,6 +1,6 @@ """ -Application-level definitions for the zlib module. +Application-level definitions for the struct module. NOT_RPYTHON """ From fijal at codespeak.net Sun Oct 7 23:38:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Oct 2007 23:38:54 +0200 (CEST) Subject: [pypy-svn] r47275 - pypy/dist/pypy/rpython/tool Message-ID: <20071007213854.B44538163@code0.codespeak.net> Author: fijal Date: Sun Oct 7 23:38:54 2007 New Revision: 47275 Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py Log: Shuffle stuff around. Helps things like #define _GNU_SOURCE Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Sun Oct 7 23:38:54 2007 @@ -79,9 +79,9 @@ CConfig = self.config print >> f, C_HEADER print >> f + print >> f, getattr(CConfig, '_header_', '') # optional for path in getattr(CConfig, '_includes_', ()): # optional print >> f, '#include <%s>' % (path,) - print >> f, getattr(CConfig, '_header_', '') # optional print >> f def write_entry(self, key, entry): From arigo at codespeak.net Mon Oct 8 11:44:56 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 11:44:56 +0200 (CEST) Subject: [pypy-svn] r47278 - pypy/dist/pypy/module/_file/test Message-ID: <20071008094456.5B86B8175@code0.codespeak.net> Author: arigo Date: Mon Oct 8 11:44:54 2007 New Revision: 47278 Modified: pypy/dist/pypy/module/_file/test/test_file_extra.py Log: This test randomly fails on wyvern, maybe because of a temporary file name clash... Modified: pypy/dist/pypy/module/_file/test/test_file_extra.py ============================================================================== --- pypy/dist/pypy/module/_file/test/test_file_extra.py (original) +++ pypy/dist/pypy/module/_file/test/test_file_extra.py Mon Oct 8 11:44:54 2007 @@ -1,8 +1,10 @@ import os, random, sys -from pypy.tool.udir import udir +import pypy.tool.udir import py from pypy.interpreter.mixedmodule import testmodule +udir = pypy.tool.udir.udir.ensure('test_file_extra', dir=1) + SAMPLE = ''.join([chr(random.randrange(0, 256)) for i in range(12487)]) for extra in ['\r\r', '\r\n', '\n\r', '\n\n']: From arigo at codespeak.net Mon Oct 8 11:58:41 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 11:58:41 +0200 (CEST) Subject: [pypy-svn] r47279 - pypy/dist/pypy/module/_file/test Message-ID: <20071008095841.5F517818F@code0.codespeak.net> Author: arigo Date: Mon Oct 8 11:58:41 2007 New Revision: 47279 Modified: pypy/dist/pypy/module/_file/test/test_file_extra.py Log: Fix an endless loop in the test if readlines() doesn't return enough data. Modified: pypy/dist/pypy/module/_file/test/test_file_extra.py ============================================================================== --- pypy/dist/pypy/module/_file/test/test_file_extra.py (original) +++ pypy/dist/pypy/module/_file/test/test_file_extra.py Mon Oct 8 11:58:41 2007 @@ -110,12 +110,16 @@ stop = 0 samplelines = self.expected_lines() while stop < 5: - morelines = self.file.readlines(random.randrange(0, 300)) + morelines = self.file.readlines(random.randrange(1, 300)) for inputline in morelines: assert inputline == samplelines[0] samplelines.pop(0) if not samplelines: stop += 1 + else: + assert len(morelines) >= 1 # otherwise, this test (and + # real programs) would be prone + # to endless loops def test_seek(self): for i in range(100): From arigo at codespeak.net Mon Oct 8 12:25:28 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 12:25:28 +0200 (CEST) Subject: [pypy-svn] r47280 - pypy/dist/pypy/module/_file Message-ID: <20071008102528.5E8A68192@code0.codespeak.net> Author: arigo Date: Mon Oct 8 12:25:26 2007 New Revision: 47280 Modified: pypy/dist/pypy/module/_file/app_file.py Log: file.readlines(0) in CPython is equivalent to file.readlines(). Modified: pypy/dist/pypy/module/_file/app_file.py ============================================================================== --- pypy/dist/pypy/module/_file/app_file.py (original) +++ pypy/dist/pypy/module/_file/app_file.py Mon Oct 8 12:25:26 2007 @@ -149,7 +149,7 @@ self.stream.unlock() return ''.join(result) - def readlines(self, size=-1): + def readlines(self, size=0): """readlines([size]) -> list of strings, each a line from the file. Call readline() repeatedly and return a list of the lines so read. @@ -161,7 +161,9 @@ raise TypeError("an integer is required") self.stream.lock() try: - if size < 0: + # NB. this implementation is very inefficient for unbuffered + # streams, but ok if self.stream.readline() is efficient. + if size <= 0: result = list(iter(self.stream.readline, "")) else: result = [] From fijal at codespeak.net Mon Oct 8 12:48:05 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Oct 2007 12:48:05 +0200 (CEST) Subject: [pypy-svn] r47281 - pypy/dist/pypy/module/mmap Message-ID: <20071008104805.F2C8C8148@code0.codespeak.net> Author: fijal Date: Mon Oct 8 12:48:05 2007 New Revision: 47281 Modified: pypy/dist/pypy/module/mmap/interp_mmap.py Log: Fix (hopefully) platform check. Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Mon Oct 8 12:48:05 2007 @@ -39,8 +39,8 @@ CConfig.MREMAP_MAYMOVE = ( rffi_platform.DefinedConstantInteger("MREMAP_MAYMOVE")) - CConfig.has_mremap = rffi_platform.Has('mremap()') # a dirty hack, this - # is probably a macro + CConfig.has_mremap = rffi_platform.Has('mremap(NULL, 0, 0, 0)') + # a dirty hack, this is probably a macro elif _MS_WINDOWS: CConfig._includes_ += ("windows.h",) From arigo at codespeak.net Mon Oct 8 12:53:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 12:53:02 +0200 (CEST) Subject: [pypy-svn] r47282 - pypy/dist/pypy/module/zlib Message-ID: <20071008105302.098C2819A@code0.codespeak.net> Author: arigo Date: Mon Oct 8 12:53:02 2007 New Revision: 47282 Modified: pypy/dist/pypy/module/zlib/interp_zlib.py Log: CPython "maximum compatibility" fixes: * support extra undocumented compression options * detect usage of compression objects after a flush(Z_FINISH). Modified: pypy/dist/pypy/module/zlib/interp_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/interp_zlib.py (original) +++ pypy/dist/pypy/module/zlib/interp_zlib.py Mon Oct 8 12:53:02 2007 @@ -102,12 +102,15 @@ """ stream = rzlib.null_stream - def __init__(self, space, level=rzlib.Z_DEFAULT_COMPRESSION): - # XXX CPython actually exposes 4 more undocumented parameters beyond - # level. + def __init__(self, space, level=rzlib.Z_DEFAULT_COMPRESSION, + method=rzlib.Z_DEFLATED, # \ + wbits=rzlib.MAX_WBITS, # \ undocumented + memLevel=rzlib.DEF_MEM_LEVEL, # / parameters + strategy=rzlib.Z_DEFAULT_STRATEGY): # / self.space = space try: - self.stream = rzlib.deflateInit(level) + self.stream = rzlib.deflateInit(level, method, wbits, + memLevel, strategy) except rzlib.RZlibError, e: raise zlib_error(self.space, e.msg) except ValueError: @@ -135,6 +138,9 @@ lock = self.lock lock.acquire(True) try: + if not self.stream: + raise zlib_error(self.space, + "compressor object already flushed") result = rzlib.compress(self.stream, data) finally: lock.release() @@ -160,24 +166,35 @@ lock = self.lock lock.acquire(True) try: + if not self.stream: + raise zlib_error(self.space, + "compressor object already flushed") result = rzlib.compress(self.stream, '', mode) finally: lock.release() except rzlib.RZlibError, e: raise zlib_error(self.space, e.msg) + if mode == rzlib.Z_FINISH: # release the data structures now + rzlib.deflateEnd(self.stream) + self.stream = rzlib.null_stream return self.space.wrap(result) flush.unwrap_spec = ['self', int] -def Compress___new__(space, w_subtype, level=rzlib.Z_DEFAULT_COMPRESSION): +def Compress___new__(space, w_subtype, level=rzlib.Z_DEFAULT_COMPRESSION, + method=rzlib.Z_DEFLATED, # \ + wbits=rzlib.MAX_WBITS, # \ undocumented + memLevel=rzlib.DEF_MEM_LEVEL, # / parameters + strategy=rzlib.Z_DEFAULT_STRATEGY): # / """ Create a new z_stream and call its initializer. """ stream = space.allocate_instance(Compress, w_subtype) stream = space.interp_w(Compress, stream) - Compress.__init__(stream, space, level) + Compress.__init__(stream, space, level, + method, wbits, memLevel, strategy) return space.wrap(stream) -Compress___new__.unwrap_spec = [ObjSpace, W_Root, int] +Compress___new__.unwrap_spec = [ObjSpace, W_Root, int, int, int, int, int] Compress.typedef = TypeDef( From arigo at codespeak.net Mon Oct 8 13:27:18 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 13:27:18 +0200 (CEST) Subject: [pypy-svn] r47283 - in pypy/branch/kill-keepalives-again/pypy/rpython: lltypesystem memory Message-ID: <20071008112718.CAE388188@code0.codespeak.net> Author: arigo Date: Mon Oct 8 13:27:16 2007 New Revision: 47283 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py Log: Some clean-ups in gc.py. A bit of a hack in lltype to allow llarena addresses to be cast to GCREF and back. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py Mon Oct 8 13:27:16 2007 @@ -112,6 +112,9 @@ return llmemory.fakeaddress.__eq__(self, other) def _cast_to_ptr(self, EXPECTED_TYPE): + if EXPECTED_TYPE == llmemory.GCREF: + obj = lltype._opaque(EXPECTED_TYPE.TO, _original_address = self) + return obj._as_ptr() # the first cast determines what object type is at this address if self.offset not in self.arena.objects: self.arena.allocate_object(self.offset, EXPECTED_TYPE.TO) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py Mon Oct 8 13:27:16 2007 @@ -1194,6 +1194,9 @@ ## addr = llmemory.fakeaddress(normalizeptr(_ptr(Ptr(T), parent))) ## addr += llmemory.itemoffsetof(T, parentindex) ## return addr + elif (isinstance(self._obj, _opaque) + and hasattr(self._obj, '_original_address')): + return self._obj._original_address # for llarena.py else: # normal case return llmemory.fakeaddress(normalizeptr(self)) @@ -1754,7 +1757,8 @@ elif isinstance(T, Array): o = _array(T, n, initialization=initialization) elif isinstance(T, OpaqueType): - o = _opaque(T, n, initialization=initialization) + assert n is None + o = _opaque(T, initialization=initialization) else: raise TypeError, "malloc for Structs and Arrays only" if T._gckind != 'gc' and not immortal and flavor.startswith('gc'): Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py Mon Oct 8 13:27:16 2007 @@ -117,7 +117,7 @@ self.get_roots() #this is there so that the annotator thinks get_roots is a function def size_gc_header(self, typeid=0): - return 0 + return self.gcheaderbuilder.size_gc_header def init_gc_object(self, addr, typeid): return @@ -618,9 +618,6 @@ return self.bytes_malloced return -1 - def size_gc_header(self, typeid=0): - return self.gcheaderbuilder.size_gc_header - def init_gc_object(self, addr, typeid): hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) hdr.typeid = typeid << 1 @@ -1001,7 +998,7 @@ class SemiSpaceGC(GCBase): _alloc_flavor_ = "raw" - HDR = lltype.Struct('header', ('forw', lltype.Signed), + HDR = lltype.Struct('header', ('forw', llmemory.Address), ('typeid', lltype.Signed)) def __init__(self, AddressLinkedList, space_size=4096, @@ -1136,8 +1133,8 @@ return obj def trace_and_copy(self, obj): - gc_info = obj - self.size_gc_header() - typeid = gc_info.signed[1] + gc_info = self.header(obj) + typeid = gc_info.typeid ## print "scanning", obj, typeid offsets = self.offsets_to_gc_pointers(typeid) i = 0 @@ -1164,18 +1161,17 @@ i += 1 def is_forwarded(self, obj): - return (obj - self.size_gc_header()).signed[1] < 0 + return self.header(obj).forw != NULL def get_forwarding_address(self, obj): - return (obj - self.size_gc_header()).address[0] + return self.header(obj).forw def set_forwarding_address(self, obj, newobj): - gc_info = obj - self.size_gc_header() - gc_info.signed[1] = -gc_info.signed[1] - 1 - gc_info.address[0] = newobj + gc_info = self.header(obj) + gc_info.forw = newobj def get_size(self, obj): - typeid = (obj - self.size_gc_header()).signed[1] + typeid = self.header(obj).typeid size = self.fixed_size(typeid) if self.is_varsize(typeid): lenaddr = obj + self.varsize_offset_to_length(typeid) @@ -1183,17 +1179,18 @@ size += length * self.varsize_item_sizes(typeid) return size - def size_gc_header(self, typeid=0): - return self.gcheaderbuilder.size_gc_header + def header(self, addr): + addr -= self.gcheaderbuilder.size_gc_header + return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) def init_gc_object(self, addr, typeid): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - #hdr.forw = 0 -- unneeded, the space is initially filled with zero + #hdr.forw = NULL -- unneeded, the space is initially filled with zero hdr.typeid = typeid def init_gc_object_immortal(self, addr, typeid): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.forw = 0 + hdr.forw = NULL hdr.typeid = typeid class DeferredRefcountingGC(GCBase): From cfbolz at codespeak.net Mon Oct 8 13:38:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 8 Oct 2007 13:38:44 +0200 (CEST) Subject: [pypy-svn] r47284 - pypy/dist/pypy/objspace/std Message-ID: <20071008113844.F15C6817E@code0.codespeak.net> Author: cfbolz Date: Mon Oct 8 13:38:44 2007 New Revision: 47284 Modified: pypy/dist/pypy/objspace/std/unicodeobject.py Log: change some things that we now have rtyper support for Modified: pypy/dist/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/unicodeobject.py (original) +++ pypy/dist/pypy/objspace/std/unicodeobject.py Mon Oct 8 13:38:44 2007 @@ -90,16 +90,7 @@ return space.newtuple([W_UnicodeObject(w_uni._value)]) def add__Unicode_Unicode(space, w_left, w_right): - left = w_left._value - right = w_right._value - leftlen = len(left) - rightlen = len(right) - result = [u'\0'] * (leftlen + rightlen) - for i in range(leftlen): - result[i] = left[i] - for i in range(rightlen): - result[i + leftlen] = right[i] - return W_UnicodeObject(result) + return W_UnicodeObject(w_left._value + w_right._value) def add__String_Unicode(space, w_left, w_right): return space.add(space.call_function(space.w_unicode, w_left) , w_right) @@ -265,13 +256,9 @@ try: result_size = ovfcheck(charlen * times) - result = [u'\0'] * result_size + result = chars * times except (OverflowError, MemoryError): raise OperationError(space.w_OverflowError, space.wrap('repeated string is too long')) - for i in range(times): - offset = i * charlen - for j in range(charlen): - result[offset + j] = chars[j] return W_UnicodeObject(result) def mul__ANY_Unicode(space, w_times, w_uni): @@ -381,9 +368,7 @@ while rpos > lpos and u_self[rpos - 1] in u_chars: rpos -= 1 - result = [u'\0'] * (rpos - lpos) - for i in range(rpos - lpos): - result[i] = u_self[lpos + i] + result = u_self[lpos: rpos] return W_UnicodeObject(result) def _strip_none(space, w_self, left, right): @@ -401,9 +386,7 @@ while rpos > lpos and _isspace(u_self[rpos - 1]): rpos -= 1 - result = [u'\0'] * (rpos - lpos) - for i in range(rpos - lpos): - result[i] = u_self[lpos + i] + result = u_self[lpos: rpos] return W_UnicodeObject(result) def unicode_strip__Unicode_None(space, w_self, w_chars): @@ -928,7 +911,7 @@ quote = '"' else: quote = '\'' - result = ['\0'] * (2 + size*6 + 1) + result = ['\0'] * (3 + size*6) result[0] = 'u' result[1] = quote i = 2 @@ -1040,8 +1023,9 @@ import unicodetype register_all(vars(), unicodetype) -# str.strip(unicode) needs to convert self to unicode and call unicode.strip -# we use the following magic to register strip_string_unicode as a String multimethod. +# str.strip(unicode) needs to convert self to unicode and call unicode.strip we +# use the following magic to register strip_string_unicode as a String +# multimethod. class str_methods: import stringtype W_UnicodeObject = W_UnicodeObject From cfbolz at codespeak.net Mon Oct 8 14:14:01 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 8 Oct 2007 14:14:01 +0200 (CEST) Subject: [pypy-svn] r47285 - pypy/dist/pypy/objspace/std Message-ID: <20071008121401.69CA980F2@code0.codespeak.net> Author: cfbolz Date: Mon Oct 8 14:13:59 2007 New Revision: 47285 Modified: pypy/dist/pypy/objspace/std/unicodeobject.py Log: add to asserts for the translator Modified: pypy/dist/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/unicodeobject.py (original) +++ pypy/dist/pypy/objspace/std/unicodeobject.py Mon Oct 8 14:13:59 2007 @@ -368,6 +368,7 @@ while rpos > lpos and u_self[rpos - 1] in u_chars: rpos -= 1 + assert rpos >= 0 result = u_self[lpos: rpos] return W_UnicodeObject(result) @@ -386,6 +387,7 @@ while rpos > lpos and _isspace(u_self[rpos - 1]): rpos -= 1 + assert rpos >= 0 result = u_self[lpos: rpos] return W_UnicodeObject(result) From cfbolz at codespeak.net Mon Oct 8 14:24:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 8 Oct 2007 14:24:38 +0200 (CEST) Subject: [pypy-svn] r47286 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20071008122438.D03CC81A5@code0.codespeak.net> Author: cfbolz Date: Mon Oct 8 14:24:38 2007 New Revision: 47286 Modified: pypy/dist/pypy/objspace/std/rope.py pypy/dist/pypy/objspace/std/ropeobject.py pypy/dist/pypy/objspace/std/test/test_rope.py Log: cleanup ropes a small bit: - add a reverse char iterator, which helps in some places to be more efficient - don't flatten the search string when doing find operations - have prebuilt interp-level rope objects of lenghts 0 and 1 (not only prebuilt W_RopeObjects) Modified: pypy/dist/pypy/objspace/std/rope.py ============================================================================== --- pypy/dist/pypy/objspace/std/rope.py (original) +++ pypy/dist/pypy/objspace/std/rope.py Mon Oct 8 14:24:38 2007 @@ -134,6 +134,9 @@ yield ('"%s" [shape=box,label="length: %s\\n%s"];' % ( id(self), len(self.s), repr(addinfo).replace('"', '').replace("\\", "\\\\"))) +LiteralStringNode.EMPTY = LiteralStringNode("") +LiteralStringNode.PREBUILT = [LiteralStringNode(chr(i)) for i in range(256)] +del i class BinaryConcatNode(StringNode): @@ -395,7 +398,7 @@ def multiply(node, times): if times <= 0: - return LiteralStringNode("") + return LiteralStringNode.EMPTY if times == 1: return node end_length = node.length() * times @@ -443,7 +446,7 @@ for node in nodelist: sizehint += node.length() if sizehint == 0: - return LiteralStringNode("") + return LiteralStringNode.EMPTY # this code is based on the Fibonacci identity: # sum(fib(i) for i in range(n+1)) == fib(n+2) @@ -563,18 +566,17 @@ def find(node, subnode, start=0, stop=-1): len1 = node.length() + len2 = subnode.length() if stop > len1 or stop == -1: stop = len1 - substring = subnode.flatten() # XXX stressful to do it as a node - len2 = len(substring) if len2 == 1: - return find_char(node, substring[0], start, stop) + return find_char(node, subnode.getitem(0), start, stop) if len2 == 0: if (stop - start) < 0: return -1 return start - restart = construct_restart_positions(substring) - return _find(node, substring, start, stop, restart) + restart = construct_restart_positions_node(subnode) + return _find_node(node, subnode, start, stop, restart) def _find(node, substring, start, stop, restart): len2 = len(substring) @@ -608,6 +610,43 @@ i = e return -1 +def _find_node(node, subnode, start, stop, restart): + len2 = subnode.length() + m = start + iter = SeekableCharIterator(node) + iter.seekforward(start) + c = iter.next() + i = 0 + subiter = SeekableCharIterator(subnode) + d = subiter.next() + while m + i < stop: + if c == d: + i += 1 + if i == len2: + return m + d = subiter.next() + if m + i < stop: + c = iter.next() + else: + # mismatch, go back to the last possible starting pos + if i == 0: + m += 1 + if m + i < stop: + c = iter.next() + else: + e = restart[i - 1] + new_m = m + i - e + assert new_m <= m + i + seek = m + i - new_m + if seek: + iter.seekback(m + i - new_m) + c = iter.next() + m = new_m + subiter.seekback(i - e + 1) + d = subiter.next() + i = e + return -1 + def construct_restart_positions(s): l = len(s) restart = [0] * l @@ -710,29 +749,36 @@ except StopIteration: return result -class SeekableFringeIterator(object): - # XXX allow to seek in bigger character steps + +class ReverseFringeIterator(object): def __init__(self, node): self.stack = [node] - self.fringestack = [] - self.fringe = [] def next(self): - if self.fringestack: - result = self.fringestack.pop() - self.fringe.append(result) - return result while self.stack: curr = self.stack.pop() while 1: if isinstance(curr, BinaryConcatNode): - self.stack.append(curr.right) - curr = curr.left + self.stack.append(curr.left) + curr = curr.right else: - self.fringe.append(curr) return curr raise StopIteration +class SeekableFringeIterator(FringeIterator): + def __init__(self, node): + FringeIterator.__init__(self, node) + self.fringestack = [] + self.fringe = [] + + def next(self): + if self.fringestack: + result = self.fringestack.pop() + else: + result = FringeIterator.next(self) + self.fringe.append(result) + return result + def seekback(self): result = self.fringe.pop() self.fringestack.append(result) @@ -756,13 +802,36 @@ break self.index = 0 index = self.index - result = self.node.getitem(index) - if self.index == self.nodelength - 1: + result = node.getitem(index) + if index == self.nodelength - 1: self.node = None else: self.index = index + 1 return result +class ReverseCharIterator(object): + def __init__(self, node): + self.iter = ReverseFringeIterator(node) + self.node = None + self.index = 0 + + def next(self): + node = self.node + index = self.index + if node is None: + while 1: + node = self.node = self.iter.next() + index = self.index = node.length() - 1 + if index != -1: + break + result = node.getitem(index) + if index == 0: + self.node = None + else: + self.index = index - 1 + return result + + class SeekableCharIterator(object): def __init__(self, node): self.iter = SeekableFringeIterator(node) @@ -904,10 +973,7 @@ if not len2: return 1 - if len1 < len2: - cmplen = len1 - else: - cmplen = len2 + cmplen = min(len1, len2) i = 0 iter1 = CharIterator(node1) iter2 = CharIterator(node2) Modified: pypy/dist/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/ropeobject.py (original) +++ pypy/dist/pypy/objspace/std/ropeobject.py Mon Oct 8 14:24:38 2007 @@ -31,8 +31,8 @@ return w_self return W_RopeObject(w_self._node) -W_RopeObject.EMPTY = W_RopeObject(rope.LiteralStringNode("")) -W_RopeObject.PREBUILT = [W_RopeObject(rope.LiteralStringNode(chr(i))) +W_RopeObject.EMPTY = W_RopeObject(rope.LiteralStringNode.EMPTY) +W_RopeObject.PREBUILT = [W_RopeObject(rope.LiteralStringNode.PREBUILT[i]) for i in range(256)] del i @@ -393,8 +393,10 @@ d = u_arg - selfnode.length() if d > 0: fillchar = fillchar[0] # annotator hint: it's a single character - resultnode = rope.concatenate(rope.LiteralStringNode(d * fillchar), - selfnode) + resultnode = rope.concatenate( + rope.multiply(rope.LiteralStringNode.PREBUILT[ord(fillchar)], + d), + selfnode) return W_RopeObject(resultnode) else: return W_RopeObject(selfnode) @@ -410,8 +412,10 @@ d = u_arg - selfnode.length() if d > 0: fillchar = fillchar[0] # annotator hint: it's a single character - resultnode = rope.concatenate(selfnode, - rope.LiteralStringNode(d * fillchar)) + resultnode = rope.concatenate( + selfnode, + rope.multiply(rope.LiteralStringNode.PREBUILT[ord(fillchar)], + d)) return W_RopeObject(resultnode) else: return W_RopeObject(selfnode) @@ -525,7 +529,7 @@ substrings = [by] iter = rope.CharIterator(node) for i in range(upper): - substrings.append(rope.LiteralStringNode(iter.next())) + substrings.append(rope.LiteralStringNode.PREBUILT[ord(iter.next())]) substrings.append(by) substrings.append(rope.getslice_one(node, upper, length)) try: @@ -571,8 +575,8 @@ lpos += 1 if right: - # XXX improve this - while rpos > lpos and node.getitem(rpos - 1) in u_chars: + iter = rope.ReverseCharIterator(node) + while rpos > lpos and iter.next() in u_chars: rpos -= 1 return W_RopeObject(rope.getslice_one(node, lpos, rpos)) @@ -592,8 +596,8 @@ lpos += 1 if right: - # XXX fix this - while rpos > lpos and node.getitem(rpos - 1).isspace(): + iter = rope.ReverseCharIterator(node) + while rpos > lpos and iter.next().isspace(): rpos -= 1 assert rpos >= lpos # annotator hint, don't remove @@ -632,7 +636,7 @@ d = arg - length if d>0: offset = d//2 - fillcharnode = rope.LiteralStringNode(fillchar) + fillcharnode = rope.LiteralStringNode.PREBUILT[ord(fillchar)] pre = rope.multiply(fillcharnode, offset) post = rope.multiply(fillcharnode, (d - offset)) centered = rope.rebalance([pre, node, post]) @@ -724,23 +728,22 @@ def _tabindent(node, tabsize): "calculates distance after the token to the next tabstop" - # XXX implement reverse char iterator length = node.length() distance = tabsize if length: distance = 0 - offset = length + iter = rope.ReverseCharIterator(node) while 1: # no sophisticated linebreak support now # '\r' just for passing adapted CPython test - char = node.getitem(offset - 1) + try: + char = iter.next() + except StopIteration: + break if char == "\n" or char == "\r": break distance += 1 - offset -= 1 - if offset == 0: - break #the same like distance = len(u_token) - (offset + 1) distance = (tabsize - distance) % tabsize @@ -758,7 +761,7 @@ tabsize = space.int_w(w_tabsize) expanded = [] - iter = rope.FindIterator(node, rope.LiteralStringNode("\t")) + iter = rope.FindIterator(node, rope.LiteralStringNode.PREBUILT[ord("\t")]) #split = u_self.split("\t") #u_expanded = oldtoken = split.pop(0) @@ -774,7 +777,7 @@ return w_self.create_if_subclassed() expanded.append(last) while 1: - expanded.append(rope.multiply(rope.LiteralStringNode(" "), + expanded.append(rope.multiply(rope.LiteralStringNode.PREBUILT[ord(" ")], _tabindent(last, tabsize))) try: next = iter.next() @@ -845,7 +848,7 @@ if length >= width: return w_self.create_if_subclassed() - zero = rope.LiteralStringNode("0") + zero = rope.LiteralStringNode.PREBUILT[ord("0")] if length == 0: return W_RopeObject(rope.multiply(zero, width)) @@ -853,7 +856,7 @@ firstchar = node.getitem(0) if length > 0 and (firstchar == '+' or firstchar == '-'): return W_RopeObject(rope.rebalance( - [rope.LiteralStringNode(firstchar), + [rope.LiteralStringNode.PREBUILT[ord(firstchar)], rope.multiply(zero, middle), rope.getslice_one(node, 1, length)])) else: Modified: pypy/dist/pypy/objspace/std/test/test_rope.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_rope.py (original) +++ pypy/dist/pypy/objspace/std/test/test_rope.py Mon Oct 8 14:24:38 2007 @@ -1,3 +1,4 @@ +import py import random from pypy.objspace.std.rope import * @@ -154,6 +155,14 @@ assert c2 == c py.test.raises(StopIteration, iter.next) +def test_reverse_iteration(): + rope, real_st = make_random_string(200) + iter = ReverseCharIterator(rope) + for c in py.builtin.reversed(real_st): + c2 = iter.next() + assert c2 == c + py.test.raises(StopIteration, iter.next) + def test_multiply(): strs = [(LiteralStringNode("a"), "a"), (LiteralStringNode("abc"), "abc"), make_random_string(500)] @@ -218,6 +227,51 @@ assert c2 == "i" py.test.raises(StopIteration, iter.next) +def test_fringe_iterator(): + ABC = LiteralStringNode("abc") + DEF = LiteralStringNode("def") + GHI = LiteralStringNode("ghi") + rope = BinaryConcatNode(BinaryConcatNode(ABC, DEF), GHI) + iter = FringeIterator(rope) + n = iter.next() + assert n is ABC + n = iter.next() + assert n is DEF + n = iter.next() + assert n is GHI + py.test.raises(StopIteration, iter.next) + +def test_seekable_fringe_iterator(): + ABC = LiteralStringNode("abc") + DEF = LiteralStringNode("def") + GHI = LiteralStringNode("ghi") + rope = BinaryConcatNode(BinaryConcatNode(ABC, DEF), GHI) + iter = SeekableFringeIterator(rope) + n = iter.next() + assert n is ABC + n = iter.seekback() + assert n is ABC + n = iter.next() + assert n is ABC + n = iter.next() + assert n is DEF + n = iter.next() + assert n is GHI + n = iter.seekback() + assert n is GHI + n = iter.seekback() + assert n is DEF + n = iter.seekback() + assert n is ABC + n = iter.next() + assert n is ABC + n = iter.next() + assert n is DEF + n = iter.next() + assert n is GHI + py.test.raises(StopIteration, iter.next) + + def test_seekforward(): rope = BinaryConcatNode(BinaryConcatNode(LiteralStringNode("abc"), LiteralStringNode("def")), From arigo at codespeak.net Mon Oct 8 15:04:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 15:04:31 +0200 (CEST) Subject: [pypy-svn] r47287 - in pypy/branch/kill-keepalives-again/pypy/rpython: lltypesystem memory Message-ID: <20071008130431.558FA815C@code0.codespeak.net> Author: arigo Date: Mon Oct 8 15:04:29 2007 New Revision: 47287 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcheader.py Log: Hacking more, but this check-in might be only for reference as I'm getting unsure if it can ever sanely work. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py Mon Oct 8 15:04:29 2007 @@ -27,6 +27,7 @@ for obj in self.objects.itervalues(): obj._free() self.objects.clear() + self.pendinggcheaders = {} if zero: initialbyte = "0" else: @@ -62,6 +63,13 @@ p = lltype.malloc(TYPE, flavor='raw', zero=zero) self.usagemap[offset:offset+size] = array.array('c', 'X' * size) self.objects[offset] = p._obj + if offset in self.pendinggcheaders: + size_gc_header = self.pendinggcheaders.pop(offset) + gcheaderbuilder = size_gc_header.gcheaderbuilder + HDR = gcheaderbuilder.HDR + headeraddr = self.getaddr(offset) - size_gc_header + headerptr = llmemory.cast_adr_to_ptr(headeraddr, lltype.Ptr(HDR)) + gcheaderbuilder.attach_header(p, headerptr) class fakearenaaddress(llmemory.fakeaddress): @@ -83,11 +91,18 @@ return '' % (self.arena, self.offset) def __add__(self, other): - if isinstance(other, llmemory.AddressOffset): - other = llmemory.raw_malloc_usage(other) if isinstance(other, (int, long)): - return self.arena.getaddr(self.offset + other) - return NotImplemented + position = self.offset + other + elif isinstance(other, llmemory.AddressOffset): + position = self.offset + llmemory.raw_malloc_usage(other) + if isinstance(other, llmemory.GCHeaderOffset): + # take the expression 'addr + size_gc_header' as a hint that + # there is at 'addr' a GC header for the object + if position not in self.arena.objects: + self.arena.pendinggcheaders[position] = other + else: + return NotImplemented + return self.arena.getaddr(position) def __sub__(self, other): if isinstance(other, llmemory.AddressOffset): @@ -111,6 +126,18 @@ else: return llmemory.fakeaddress.__eq__(self, other) + def __lt__(self, other): + if isinstance(other, fakearenaaddress): + if self.arena is not other.arena: + return cmp(self.arena._getid(), other.arena._getid()) + else: + return cmp(self.offset, other.offset) + elif isinstance(other, llmemory.fakeaddress) and not other: + return 1 # 'self' > NULL + else: + raise TypeError("comparing a %s and a %s" % ( + self.__class__.__name__, other.__class__.__name__)) + def _cast_to_ptr(self, EXPECTED_TYPE): if EXPECTED_TYPE == llmemory.GCREF: obj = lltype._opaque(EXPECTED_TYPE.TO, _original_address = self) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py Mon Oct 8 15:04:29 2007 @@ -351,6 +351,15 @@ else: return NotImplemented + def __lt__(self, other): + raise TypeError("cannot compare fakeaddresses with '<'") + def __le__(self, other): + return self == other or self < other + def __gt__(self, other): + return not (self == other or self < other) + def __ge__(self, other): + return not (self < other) + def ref(self): if not self: raise NullAddressError Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcheader.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcheader.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcheader.py Mon Oct 8 15:04:29 2007 @@ -28,16 +28,19 @@ def get_header(self, gcptr): return self.obj2header.get(gcptr._as_obj(), None) - def new_header(self, gcptr): + def attach_header(self, gcptr, headerptr): gcobj = gcptr._as_obj() assert gcobj not in self.obj2header # sanity checks assert gcobj._TYPE._gckind == 'gc' assert not isinstance(gcobj._TYPE, lltype.GcOpaqueType) assert not gcobj._parentstructure() - headerptr = lltype.malloc(self.HDR, immortal=True) self.obj2header[gcobj] = headerptr header2obj[headerptr._obj] = gcptr._as_ptr() + + def new_header(self, gcptr): + headerptr = lltype.malloc(self.HDR, immortal=True) + self.attach_header(gcptr, headerptr) return headerptr def _freeze_(self): From fijal at codespeak.net Mon Oct 8 15:39:53 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Oct 2007 15:39:53 +0200 (CEST) Subject: [pypy-svn] r47292 - in pypy/dist/pypy/rlib: . test Message-ID: <20071008133953.DF5F4815C@code0.codespeak.net> Author: fijal Date: Mon Oct 8 15:39:53 2007 New Revision: 47292 Added: pypy/dist/pypy/rlib/libffi.py (contents, props changed) pypy/dist/pypy/rlib/test/test_libffi.py (contents, props changed) Log: Implemend dl* wrappers for rpython. useful for ctypes. Added: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/libffi.py Mon Oct 8 15:39:53 2007 @@ -0,0 +1,70 @@ + +""" Various rpython-level functions for dlopen and libffi wrapping +""" + +from pypy.rpython.tool import rffi_platform +from pypy.rpython.lltypesystem import lltype, rffi + +includes = ['dlfcn.h'] + +class CConfig: + RTLD_LOCAL = rffi_platform.DefinedConstantInteger('RTLD_LOCAL') + RTLD_NOW = rffi_platform.DefinedConstantInteger('RTLD_NOW') + _includes_ = includes + +class cConfig: + pass + +cConfig.__dict__.update(rffi_platform.configure(CConfig)) + +def external(name, args, result): + return rffi.llexternal(name, args, result, includes=includes, + libraries=['dl']) + +c_dlopen = external('dlopen', [rffi.CCHARP, rffi.INT], rffi.VOIDP) +c_dlclose = external('dlclose', [rffi.VOIDP], rffi.INT) +c_dlerror = external('dlerror', [], rffi.CCHARP) +c_dlsym = external('dlsym', [rffi.VOIDP, rffi.CCHARP], rffi.VOIDP) + +RTLD_LOCAL = cConfig.RTLD_LOCAL +RTLD_NOW = cConfig.RTLD_NOW + +def dlerror(): + # XXX this would never work on top of ll2ctypes, because + # ctypes are calling dlerror itself, unsure if I can do much in this + # area (nor I would like to) + res = c_dlerror() + if not res: + return "" + return rffi.charp2str(res) + +def dlopen(name): + """ Wrapper around C-level dlopen + """ + if RTLD_LOCAL is not None: + mode = RTLD_LOCAL | RTLD_NOW + else: + mode = RTLD_NOW + res = c_dlopen(name, mode) + if not res: + raise OSError(dlerror()) + return res + +def dlsym(libhandle, name): + """ Wrapper around C-level dlsym + """ + res = c_dlsym(libhandle, name) + if not res: + raise KeyError(name) + # XXX rffi.cast here... + return res + +class CDLL: + def __init__(self, libname): + self.lib = dlopen(libname) + + def __del__(self): + c_dlclose(self.lib) + + def getpointer(self, name): + return dlsym(self.lib, name) Added: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/test/test_libffi.py Mon Oct 8 15:39:53 2007 @@ -0,0 +1,39 @@ + +""" Tests of libffi wrappers and dl* friends +""" + +from pypy.rpython.test.test_llinterp import interpret +from pypy.rlib.libffi import CDLL, dlopen +from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED +import os, sys +import py + +def setup_module(mod): + if not sys.platform.startswith('linux'): + py.test.skip("Fragile tests, linux only by now") + +class TestDLOperations: + def setup_method(self, meth): + ALLOCATED.clear() + + def teardown_method(self, meth): + assert not ALLOCATED + + def test_dlopen(self): + py.test.raises(OSError, "dlopen('xxxxxxxxxxxx')") + assert dlopen('/lib/libc.so.6') + + def get_libc(self): + return CDLL('/lib/libc.so.6') + + def test_library_open(self): + lib = self.get_libc() + del lib + assert not ALLOCATED + + def test_library_get_func(self): + lib = self.get_libc() + ptr = lib.getpointer('time') + py.test.raises(KeyError, lib.getpointer, 'xxxxxxxxxxxxxxx') + del lib + assert not ALLOCATED From pypy-svn at codespeak.net Mon Oct 8 17:13:06 2007 From: pypy-svn at codespeak.net (VIAGRA ® Official Site) Date: Mon, 8 Oct 2007 17:13:06 +0200 (CEST) Subject: [pypy-svn] October 79% OFF Message-ID: <20071008061249.5806.qmail@ip-92.net-89-2-139.rev.numericable.fr> An HTML attachment was scrubbed... URL: From arigo at codespeak.net Mon Oct 8 17:31:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 17:31:45 +0200 (CEST) Subject: [pypy-svn] r47293 - in pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem: . test Message-ID: <20071008153145.BBA46812A@code0.codespeak.net> Author: arigo Date: Mon Oct 8 17:31:44 2007 New Revision: 47293 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py Log: A cleaner approach: requiring an explicit declaration of the object we expect to find. This is also where the fakearenaaddress -> fakeaddress conversion takes place. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py Mon Oct 8 17:31:44 2007 @@ -14,20 +14,22 @@ pass class Arena(object): + object_arena_location = {} # {container: (arena, offset)} def __init__(self, nbytes, zero): self.nbytes = nbytes self.usagemap = array.array('c') - self.objects = {} + self.objectptrs = {} # {offset: ptr-to-container} self.freed = False self.reset(zero) def reset(self, zero): self.check() - for obj in self.objects.itervalues(): + for ptr in self.objectptrs.itervalues(): + obj = ptr._obj obj._free() - self.objects.clear() - self.pendinggcheaders = {} + del Arena.object_arena_location[obj] + self.objectptrs.clear() if zero: initialbyte = "0" else: @@ -47,29 +49,34 @@ raise ArenaError("Address offset is outside the arena") return fakearenaaddress(self, offset) - def allocate_object(self, offset, TYPE): + def allocate_object(self, offset, size): self.check() - size = llmemory.raw_malloc_usage(llmemory.sizeof(TYPE)) - if offset + size > self.nbytes: + bytes = llmemory.raw_malloc_usage(size) + if offset + bytes > self.nbytes: raise ArenaError("object overflows beyond the end of the arena") zero = True - for c in self.usagemap[offset:offset+size]: + for c in self.usagemap[offset:offset+bytes]: if c == '0': pass elif c == '#': zero = False else: raise ArenaError("new object overlaps a previous object") - p = lltype.malloc(TYPE, flavor='raw', zero=zero) - self.usagemap[offset:offset+size] = array.array('c', 'X' * size) - self.objects[offset] = p._obj - if offset in self.pendinggcheaders: - size_gc_header = self.pendinggcheaders.pop(offset) - gcheaderbuilder = size_gc_header.gcheaderbuilder - HDR = gcheaderbuilder.HDR - headeraddr = self.getaddr(offset) - size_gc_header - headerptr = llmemory.cast_adr_to_ptr(headeraddr, lltype.Ptr(HDR)) - gcheaderbuilder.attach_header(p, headerptr) + assert offset not in self.objectptrs + addr2 = size._raw_malloc([], zero=zero) + self.usagemap[offset:offset+bytes] = array.array('c', 'X' * bytes) + self.objectptrs[offset] = addr2.ptr + Arena.object_arena_location[addr2.ptr._obj] = self, offset + # common case: 'size' starts with a GCHeaderOffset. In this case + # we can also remember that the real object starts after the header. + if (isinstance(size, llmemory.CompositeOffset) and + isinstance(size.offsets[0], llmemory.GCHeaderOffset)): + objaddr = addr2 + size.offsets[0] + objoffset = offset + llmemory.raw_malloc_usage(size.offsets[0]) + assert objoffset not in self.objectptrs + self.objectptrs[objoffset] = objaddr.ptr + Arena.object_arena_location[objaddr.ptr._obj] = self, objoffset + return addr2 class fakearenaaddress(llmemory.fakeaddress): @@ -79,12 +86,11 @@ def _getptr(self): try: - obj = self.arena.objects[self.offset] + return self.arena.objectptrs[self.offset] except KeyError: self.arena.check() raise ArenaError("don't know yet what type of object " "is at offset %d" % (self.offset,)) - return obj._as_ptr() ptr = property(_getptr) def __repr__(self): @@ -95,11 +101,6 @@ position = self.offset + other elif isinstance(other, llmemory.AddressOffset): position = self.offset + llmemory.raw_malloc_usage(other) - if isinstance(other, llmemory.GCHeaderOffset): - # take the expression 'addr + size_gc_header' as a hint that - # there is at 'addr' a GC header for the object - if position not in self.arena.objects: - self.arena.pendinggcheaders[position] = other else: return NotImplemented return self.arena.getaddr(position) @@ -110,9 +111,9 @@ if isinstance(other, (int, long)): return self.arena.getaddr(self.offset - other) if isinstance(other, fakearenaaddress): - if self.other is not other.arena: + if self.arena is not other.arena: raise ArenaError("The two addresses are from different arenas") - return other.offset - self.offset + return self.offset - other.offset return NotImplemented def __nonzero__(self): @@ -121,31 +122,31 @@ def __eq__(self, other): if isinstance(other, fakearenaaddress): return self.arena is other.arena and self.offset == other.offset - elif isinstance(other, llmemory.fakeaddress) and not other: - return False # 'self' can't be equal to NULL + elif isinstance(other, llmemory.fakeaddress): + if other.ptr and other.ptr._obj in Arena.object_arena_location: + arena, offset = Arena.object_arena_location[other.ptr._obj] + return self.arena is arena and self.offset == offset + else: + return False else: return llmemory.fakeaddress.__eq__(self, other) def __lt__(self, other): if isinstance(other, fakearenaaddress): - if self.arena is not other.arena: - return cmp(self.arena._getid(), other.arena._getid()) + arena = other.arena + offset = other.offset + elif isinstance(other, llmemory.fakeaddress): + if other.ptr and other.ptr._obj in Arena.object_arena_location: + arena, offset = Arena.object_arena_location[other.ptr._obj] else: - return cmp(self.offset, other.offset) - elif isinstance(other, llmemory.fakeaddress) and not other: - return 1 # 'self' > NULL + return 1 # arbitrarily, 'self' > any address not in any arena else: raise TypeError("comparing a %s and a %s" % ( self.__class__.__name__, other.__class__.__name__)) - - def _cast_to_ptr(self, EXPECTED_TYPE): - if EXPECTED_TYPE == llmemory.GCREF: - obj = lltype._opaque(EXPECTED_TYPE.TO, _original_address = self) - return obj._as_ptr() - # the first cast determines what object type is at this address - if self.offset not in self.arena.objects: - self.arena.allocate_object(self.offset, EXPECTED_TYPE.TO) - return llmemory.fakeaddress._cast_to_ptr(self, EXPECTED_TYPE) + if self.arena is arena: + return cmp(self.offset, offset) + else: + return cmp(self.arena._getid(), arena._getid()) def _cast_to_int(self): return self.arena._getid() + self.offset @@ -176,3 +177,12 @@ assert arena_addr.offset == 0 assert myarenasize == arena_addr.arena.nbytes arena_addr.arena.reset(zero) + +def arena_reserve(addr, size): + """Mark some bytes in an arena as reserved, and returns addr. + For debugging this can check that reserved ranges of bytes don't + overlap. The size must be symbolic; in non-translated version + the returned 'addr' is not a fakearenaaddress but a fakeaddress + inside a fresh lltype object.""" + assert isinstance(addr, fakearenaaddress) + return addr.arena.allocate_object(addr.offset, size) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py Mon Oct 8 17:31:44 2007 @@ -1194,9 +1194,6 @@ ## addr = llmemory.fakeaddress(normalizeptr(_ptr(Ptr(T), parent))) ## addr += llmemory.itemoffsetof(T, parentindex) ## return addr - elif (isinstance(self._obj, _opaque) - and hasattr(self._obj, '_original_address')): - return self._obj._original_address # for llarena.py else: # normal case return llmemory.fakeaddress(normalizeptr(self)) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py Mon Oct 8 17:31:44 2007 @@ -2,6 +2,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr from pypy.rpython.lltypesystem.llarena import arena_malloc, arena_reset +from pypy.rpython.lltypesystem.llarena import arena_reserve from pypy.rpython.lltypesystem.llarena import ArenaError def test_arena(): @@ -13,12 +14,14 @@ assert a != llmemory.NULL assert a + 3 != llmemory.NULL + arena_reserve(a, llmemory.sizeof(S)) s1_ptr1 = cast_adr_to_ptr(a, SPTR) s1_ptr1.x = 1 s1_ptr2 = cast_adr_to_ptr(a, SPTR) assert s1_ptr2.x == 1 assert s1_ptr1 == s1_ptr2 + arena_reserve(a + ssize + 1, llmemory.sizeof(S)) s2_ptr1 = cast_adr_to_ptr(a + ssize + 1, SPTR) py.test.raises(lltype.UninitializedMemoryAccess, 's2_ptr1.x') s2_ptr1.x = 2 @@ -31,20 +34,29 @@ S2 = lltype.Struct('S2', ('y',lltype.Char)) S2PTR = lltype.Ptr(S2) - py.test.raises(TypeError, cast_adr_to_ptr, a, S2PTR) + py.test.raises(lltype.InvalidCast, cast_adr_to_ptr, a, S2PTR) py.test.raises(ArenaError, cast_adr_to_ptr, a+1, SPTR) py.test.raises(ArenaError, cast_adr_to_ptr, a+ssize, SPTR) py.test.raises(ArenaError, cast_adr_to_ptr, a+2*ssize, SPTR) py.test.raises(ArenaError, cast_adr_to_ptr, a+2*ssize+1, SPTR) + py.test.raises(ArenaError, arena_reserve, a+1, llmemory.sizeof(S)) + py.test.raises(ArenaError, arena_reserve, a+ssize, llmemory.sizeof(S)) + py.test.raises(ArenaError, arena_reserve, a+2*ssize, llmemory.sizeof(S)) + py.test.raises(ArenaError, arena_reserve, a+2*ssize+1, llmemory.sizeof(S)) arena_reset(a, myarenasize, True) + py.test.raises(ArenaError, cast_adr_to_ptr, a, SPTR) + arena_reserve(a, llmemory.sizeof(S)) s1_ptr1 = cast_adr_to_ptr(a, SPTR) assert s1_ptr1.x == 0 s1_ptr1.x = 5 + arena_reserve(a + ssize, llmemory.sizeof(S2)) s2_ptr1 = cast_adr_to_ptr(a + ssize, S2PTR) assert s2_ptr1.y == '\x00' s2_ptr1.y = 'X' assert cast_adr_to_ptr(a + 0, SPTR).x == 5 assert cast_adr_to_ptr((a + ssize + 1) - 1, S2PTR).y == 'X' + + assert (a + 4) - (a + 1) == 3 From arigo at codespeak.net Mon Oct 8 17:35:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 17:35:52 +0200 (CEST) Subject: [pypy-svn] r47294 - pypy/branch/kill-keepalives-again/pypy/rpython/memory Message-ID: <20071008153552.4D6EE8121@code0.codespeak.net> Author: arigo Date: Mon Oct 8 17:35:52 2007 New Revision: 47294 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py Log: * Use the new llarena interface. * Fixes in the "are we running out of space" logic. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py Mon Oct 8 17:35:52 2007 @@ -1048,13 +1048,15 @@ raise NotImplementedError("weakptr in SemiSpaceGC") size_gc_header = self.gcheaderbuilder.size_gc_header totalsize = size_gc_header + size - if can_collect and self.free + totalsize > self.top_of_space: + if raw_malloc_usage(totalsize) > self.top_of_space - self.free: + if not can_collect: + raise memoryError self.collect() #XXX need to increase the space size if the object is too big #for bonus points do big objects differently - if self.free + totalsize > self.top_of_space: + if raw_malloc_usage(totalsize) > self.top_of_space - self.free: raise memoryError - result = self.free + result = llarena.arena_reserve(self.free, totalsize) self.init_gc_object(result, typeid) self.free += totalsize return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) @@ -1070,14 +1072,15 @@ totalsize = ovfcheck(nonvarsize + varsize) except OverflowError: raise memoryError - # XXX we can't use ovfcheck() for self.free + totalsize... - if can_collect and self.free + totalsize > self.top_of_space: + if raw_malloc_usage(totalsize) > self.top_of_space - self.free: + if not can_collect: + raise memoryError self.collect() #XXX need to increase the space size if the object is too big #for bonus points do big objects differently - if self.free + totalsize > self.top_of_space: + if raw_malloc_usage(totalsize) > self.top_of_space - self.free: raise memoryError - result = self.free + result = llarena.arena_reserve(self.free, totalsize) self.init_gc_object(result, typeid) (result + size_gc_header + offset_to_length).signed[0] = length self.free += totalsize From arigo at codespeak.net Mon Oct 8 18:34:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 18:34:34 +0200 (CEST) Subject: [pypy-svn] r47296 - in pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem: . test Message-ID: <20071008163434.94BEF814F@code0.codespeak.net> Author: arigo Date: Mon Oct 8 18:34:34 2007 New Revision: 47296 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py Log: Fix fakearenaaddress comparison. Write paranoid test. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py Mon Oct 8 18:34:34 2007 @@ -139,14 +139,15 @@ if other.ptr and other.ptr._obj in Arena.object_arena_location: arena, offset = Arena.object_arena_location[other.ptr._obj] else: - return 1 # arbitrarily, 'self' > any address not in any arena + # arbitrarily, 'self' > any address not in any arena + return False else: raise TypeError("comparing a %s and a %s" % ( self.__class__.__name__, other.__class__.__name__)) if self.arena is arena: - return cmp(self.offset, offset) + return self.offset < offset else: - return cmp(self.arena._getid(), arena._getid()) + return self.arena._getid() < arena._getid() def _cast_to_int(self): return self.arena._getid() + self.offset Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py Mon Oct 8 18:34:34 2007 @@ -25,7 +25,7 @@ def _raw_malloc(self, rest, zero): raise NotImplementedError("_raw_malloc(%r, %r)" % (self, rest)) - def raw_memcopy(self, srcadr, dstsrc): + def raw_memcopy(self, srcadr, dstadr): raise NotImplementedError("raw_memcopy(%r)" % (self,)) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py Mon Oct 8 18:34:34 2007 @@ -60,3 +60,47 @@ assert cast_adr_to_ptr((a + ssize + 1) - 1, S2PTR).y == 'X' assert (a + 4) - (a + 1) == 3 + + +def lt(x, y): + if x < y: + assert (x < y) and (y > x) + assert (x <= y) and (y >= x) + assert not (x == y) and not (y == x) + assert (x != y) and (y != x) + assert not (x > y) and not (y < x) + assert not (x >= y) and not (y <= x) + return True + else: + assert (x >= y) and (y <= x) + assert (x == y) == (not (x != y)) == (y == x) == (not (y != x)) + assert (x > y) == (not (x == y)) == (y < x) + return False + +def eq(x, y): + if x == y: + assert not (x != y) and not (y != x) + assert not (x < y) and not (y > x) + assert not (x > y) and not (y < x) + assert (x <= y) and (y >= x) + assert (x >= y) and (y <= x) + return True + else: + assert (x != y) and (y != x) + assert ((x < y) == (x <= y) == (not (x > y)) == (not (x >= y)) == + (y > x) == (y >= x) == (not (y < x)) == (not (y <= x))) + return False + + +def test_address_order(): + a = arena_malloc(20, False) + assert eq(a, a) + assert lt(a, a+1) + assert lt(a+5, a+20) + + b = arena_malloc(20, False) + if a > b: + a, b = a, b + assert lt(a, b) + assert lt(a+19, b) + assert lt(a, b+19) From arigo at codespeak.net Mon Oct 8 18:37:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 18:37:17 +0200 (CEST) Subject: [pypy-svn] r47297 - in pypy/branch/kill-keepalives-again/pypy/rpython: . memory Message-ID: <20071008163717.DDC138150@code0.codespeak.net> Author: arigo Date: Mon Oct 8 18:37:17 2007 New Revision: 47297 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcwrapper.py Log: Modify llinterp.find_roots() to return a list of "addresses" to its local variables. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py Mon Oct 8 18:37:17 2007 @@ -123,6 +123,7 @@ log.traceback(line) def find_roots(self): + """Return a list of the addresses of the roots.""" #log.findroots("starting") frame = self.active_frame roots = [] @@ -455,14 +456,17 @@ def find_roots(self, roots): #log.findroots(self.curr_block.inputargs) - PyObjPtr = lltype.Ptr(lltype.PyObject) - for arg in self.curr_block.inputargs: - if (isinstance(arg, Variable) and - isinstance(getattr(arg, 'concretetype', PyObjPtr), lltype.Ptr)): - roots.append(self.getval(arg)) + vars = [] + for v in self.curr_block.inputargs: + if isinstance(v, Variable): + vars.append(v) for op in self.curr_block.operations[:self.curr_operation_index]: - if isinstance(getattr(op.result, 'concretetype', PyObjPtr), lltype.Ptr): - roots.append(self.getval(op.result)) + vars.append(op.result) + + for v in vars: + TYPE = getattr(v, 'concretetype', None) + if isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': + roots.append(_address_of_local_var(self, v)) # __________________________________________________________ # misc LL operation implementations @@ -1209,6 +1213,30 @@ addcls(cls) return result +class _address_of_local_var(object): + _TYPE = llmemory.Address + def __init__(self, frame, v): + self._frame = frame + self._v = v + def _getaddress(self): + return _address_of_local_var_accessor(self._frame, self._v) + address = property(_getaddress) + +class _address_of_local_var_accessor(object): + def __init__(self, frame, v): + self.frame = frame + self.v = v + def __getitem__(self, index): + if index != 0: + raise IndexError("address of local vars only support [0] indexing") + p = self.frame.getval(self.v) + return llmemory.cast_ptr_to_adr(p) + def __setitem__(self, index, newvalue): + if index != 0: + raise IndexError("address of local vars only support [0] indexing") + p = llmemory.cast_adr_to_ptr(newvalue, self.v.concretetype) + self.frame.setvar(self.v, p) + # by default we route all logging messages to nothingness # e.g. tests can then switch on logging to get more help # for failing tests Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py Mon Oct 8 18:37:17 2007 @@ -88,6 +88,9 @@ def statistics(self, index): return -1 + def size_gc_header(self, typeid=0): + return self.gcheaderbuilder.size_gc_header + def x_swap_pool(self, newpool): return newpool @@ -116,9 +119,6 @@ def collect(self): self.get_roots() #this is there so that the annotator thinks get_roots is a function - def size_gc_header(self, typeid=0): - return self.gcheaderbuilder.size_gc_header - def init_gc_object(self, addr, typeid): return init_gc_object_immortal = init_gc_object @@ -1120,8 +1120,8 @@ ## print "already copied to", self.get_forwarding_address(obj) return self.get_forwarding_address(obj) else: - newaddr = self.free - totalsize = self.get_size(obj) + self.size_gc_header() + totalsize = self.size_gc_header() + self.get_size(obj) + newaddr = llarena.arena_reserve(self.free, totalsize) raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize) self.free += totalsize newobj = newaddr + self.size_gc_header() Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcwrapper.py Mon Oct 8 18:37:17 2007 @@ -37,11 +37,8 @@ ll = self.AddressLinkedList() for addrofaddr in self.constantroots: ll.append(addrofaddr) - for root in self.llinterp.find_roots(): - if lltype.typeOf(root).TO._gckind == 'gc': - addrofaddr = llmemory.raw_malloc(sizeofaddr) - addrofaddr.address[0] = llmemory.cast_ptr_to_adr(root) - ll.append(addrofaddr) + for addrofaddr in self.llinterp.find_roots(): + ll.append(addrofaddr) return ll # ____________________________________________________________ From fijal at codespeak.net Mon Oct 8 19:03:25 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Oct 2007 19:03:25 +0200 (CEST) Subject: [pypy-svn] r47298 - in pypy/dist/pypy/rlib: . test Message-ID: <20071008170325.71B5480EA@code0.codespeak.net> Author: fijal Date: Mon Oct 8 19:03:24 2007 New Revision: 47298 Modified: pypy/dist/pypy/rlib/libffi.py pypy/dist/pypy/rlib/test/test_libffi.py Log: First external function call from rpython level passes! Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Mon Oct 8 19:03:24 2007 @@ -5,21 +5,40 @@ from pypy.rpython.tool import rffi_platform from pypy.rpython.lltypesystem import lltype, rffi -includes = ['dlfcn.h'] +includes = ['dlfcn.h', 'ffi.h'] + +FFI_TYPE_P = lltype.Ptr(lltype.ForwardReference()) +FFI_TYPE_PP = rffi.CArrayPtr(FFI_TYPE_P) class CConfig: + _includes_ = includes + RTLD_LOCAL = rffi_platform.DefinedConstantInteger('RTLD_LOCAL') RTLD_NOW = rffi_platform.DefinedConstantInteger('RTLD_NOW') - _includes_ = includes + + FFI_OK = rffi_platform.ConstantInteger('FFI_OK') + FFI_BAD_TYPEDEF = rffi_platform.ConstantInteger('FFI_BAD_TYPEDEF') + FFI_DEFAULT_ABI = rffi_platform.ConstantInteger('FFI_DEFAULT_ABI') + + size_t = rffi_platform.SimpleType("size_t", rffi.ULONG) + + ffi_type = rffi_platform.Struct('ffi_type', [('size', rffi.ULONG), + ('alignment', rffi.USHORT), + ('type', rffi.USHORT), + ('elements', FFI_TYPE_PP)]) + # XXX elements goes here, for structures class cConfig: pass cConfig.__dict__.update(rffi_platform.configure(CConfig)) +FFI_TYPE_P.TO.become(cConfig.ffi_type) +size_t = cConfig.size_t + def external(name, args, result): return rffi.llexternal(name, args, result, includes=includes, - libraries=['dl']) + libraries=['dl', 'ffi']) c_dlopen = external('dlopen', [rffi.CCHARP, rffi.INT], rffi.VOIDP) c_dlclose = external('dlclose', [rffi.VOIDP], rffi.INT) @@ -28,6 +47,23 @@ RTLD_LOCAL = cConfig.RTLD_LOCAL RTLD_NOW = cConfig.RTLD_NOW +FFI_OK = cConfig.FFI_OK +FFI_BAD_TYPEDEF = cConfig.FFI_BAD_TYPEDEF +FFI_DEFAULT_ABI = cConfig.FFI_DEFAULT_ABI +FFI_CIFP = rffi.COpaquePtr('ffi_cif', includes=includes) + +c_ffi_prep_cif = external('ffi_prep_cif', [FFI_CIFP, rffi.USHORT, rffi.UINT, + FFI_TYPE_P, FFI_TYPE_PP], rffi.INT) +c_ffi_call = external('ffi_call', [FFI_CIFP, rffi.VOIDP, rffi.VOIDP, + rffi.CArrayPtr(rffi.VOIDP)], lltype.Void) + +# XXX hardcode this values by now, we need some new logic/thinking for that + +ffi_type_sint = lltype.malloc(FFI_TYPE_P.TO, flavor='raw', immortal=True) +ffi_type_sint.c_size = rffi.cast(size_t, 4) +ffi_type_sint.c_alignment = rffi.cast(rffi.USHORT, 4) +ffi_type_sint.c_type = rffi.cast(rffi.USHORT, 10) +ffi_type_sint.c_elements = lltype.nullptr(FFI_TYPE_PP.TO) def dlerror(): # XXX this would never work on top of ll2ctypes, because @@ -59,6 +95,31 @@ # XXX rffi.cast here... return res +class FuncPtr: + def __init__(self, func_sym): + # xxx args here + if not func_sym: + raise OSError("NULL func_sym") + self.func_sym = func_sym + self.ll_cif = lltype.malloc(FFI_CIFP.TO, flavor='raw') + res = c_ffi_prep_cif(self.ll_cif, rffi.cast(rffi.USHORT, FFI_DEFAULT_ABI), + rffi.cast(rffi.UINT, 0), ffi_type_sint, lltype.nullptr(FFI_TYPE_PP.TO)) + if not res == FFI_OK: + raise OSError("Wrong typedef") + + def call(self, args): + # allocated result should be padded and stuff + PTR_T = lltype.Ptr(rffi.CFixedArray(rffi.INT, 1)) + result = lltype.malloc(PTR_T.TO, flavor='raw') + c_ffi_call(self.ll_cif, self.func_sym, rffi.cast(rffi.VOIDP, result), + lltype.nullptr(rffi.CCHARPP.TO)) + res = result[0] + lltype.free(result, flavor='raw') + return res + + def __del__(self): + lltype.free(self.ll_cif, flavor='raw') + class CDLL: def __init__(self, libname): self.lib = dlopen(libname) @@ -67,4 +128,4 @@ c_dlclose(self.lib) def getpointer(self, name): - return dlsym(self.lib, name) + return FuncPtr(dlsym(self.lib, name)) Modified: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_libffi.py (original) +++ pypy/dist/pypy/rlib/test/test_libffi.py Mon Oct 8 19:03:24 2007 @@ -17,7 +17,8 @@ ALLOCATED.clear() def teardown_method(self, meth): - assert not ALLOCATED + pass + #assert not ALLOCATED, not yet def test_dlopen(self): py.test.raises(OSError, "dlopen('xxxxxxxxxxxx')") @@ -29,11 +30,20 @@ def test_library_open(self): lib = self.get_libc() del lib - assert not ALLOCATED def test_library_get_func(self): lib = self.get_libc() ptr = lib.getpointer('time') py.test.raises(KeyError, lib.getpointer, 'xxxxxxxxxxxxxxx') del lib - assert not ALLOCATED + + def test_library_func_call(self): + lib = self.get_libc() + ptr = lib.getpointer('rand') + zeroes = 0 + for i in range(100): + res = ptr.call([]) + if not res: + zeroes += 1 + assert not zeroes + # not very hard check, but something :] From arigo at codespeak.net Mon Oct 8 19:03:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 19:03:34 +0200 (CEST) Subject: [pypy-svn] r47299 - in pypy/branch/kill-keepalives-again/pypy/rpython: lltypesystem lltypesystem/test memory memory/test Message-ID: <20071008170334.B09FD812A@code0.codespeak.net> Author: arigo Date: Mon Oct 8 19:03:34 2007 New Revision: 47299 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_gc.py Log: the SemiSpace tests pass! the change is to introduce some guessing in 'fakearenaaddress + symbolic_offset'. See comments. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py Mon Oct 8 19:03:34 2007 @@ -20,6 +20,7 @@ self.nbytes = nbytes self.usagemap = array.array('c') self.objectptrs = {} # {offset: ptr-to-container} + self.objectsizes = {} # {offset: size} self.freed = False self.reset(zero) @@ -30,6 +31,7 @@ obj._free() del Arena.object_arena_location[obj] self.objectptrs.clear() + self.objectsizes.clear() if zero: initialbyte = "0" else: @@ -64,17 +66,21 @@ raise ArenaError("new object overlaps a previous object") assert offset not in self.objectptrs addr2 = size._raw_malloc([], zero=zero) - self.usagemap[offset:offset+bytes] = array.array('c', 'X' * bytes) + pattern = 'X' + 'x'*(bytes-1) + self.usagemap[offset:offset+bytes] = array.array('c', pattern) self.objectptrs[offset] = addr2.ptr + self.objectsizes[offset] = bytes Arena.object_arena_location[addr2.ptr._obj] = self, offset # common case: 'size' starts with a GCHeaderOffset. In this case # we can also remember that the real object starts after the header. if (isinstance(size, llmemory.CompositeOffset) and isinstance(size.offsets[0], llmemory.GCHeaderOffset)): objaddr = addr2 + size.offsets[0] - objoffset = offset + llmemory.raw_malloc_usage(size.offsets[0]) + hdrbytes = llmemory.raw_malloc_usage(size.offsets[0]) + objoffset = offset + hdrbytes assert objoffset not in self.objectptrs self.objectptrs[objoffset] = objaddr.ptr + self.objectsizes[objoffset] = bytes - hdrbytes Arena.object_arena_location[objaddr.ptr._obj] = self, objoffset return addr2 @@ -100,7 +106,16 @@ if isinstance(other, (int, long)): position = self.offset + other elif isinstance(other, llmemory.AddressOffset): - position = self.offset + llmemory.raw_malloc_usage(other) + # this is really some Do What I Mean logic. There are two + # possible meanings: either we want to go past the current + # object in the arena, or we want to take the address inside + # the current object. Try to guess... + bytes = llmemory.raw_malloc_usage(other) + if (self.offset in self.arena.objectsizes and + bytes < self.arena.objectsizes[self.offset]): + # looks like we mean "inside the object" + return llmemory.fakeaddress.__add__(self, other) + position = self.offset + bytes else: return NotImplemented return self.arena.getaddr(position) @@ -183,7 +198,6 @@ """Mark some bytes in an arena as reserved, and returns addr. For debugging this can check that reserved ranges of bytes don't overlap. The size must be symbolic; in non-translated version - the returned 'addr' is not a fakearenaaddress but a fakeaddress - inside a fresh lltype object.""" + this is used to know what type of lltype object to allocate.""" assert isinstance(addr, fakearenaaddress) - return addr.arena.allocate_object(addr.offset, size) + addr.arena.allocate_object(addr.offset, size) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py Mon Oct 8 19:03:34 2007 @@ -104,3 +104,15 @@ assert lt(a, b) assert lt(a+19, b) assert lt(a, b+19) + + +def test_look_inside_object(): + S = lltype.Struct('S', ('x',lltype.Signed)) + SPTR = lltype.Ptr(S) + a = arena_malloc(50, False) + b = a + 4 + arena_reserve(b, llmemory.sizeof(S)) + (b + llmemory.offsetof(S, 'x')).signed[0] = 123 + assert llmemory.cast_adr_to_ptr(b, SPTR).x == 123 + llmemory.cast_adr_to_ptr(b, SPTR).x += 1 + assert (b + llmemory.offsetof(S, 'x')).signed[0] == 124 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py Mon Oct 8 19:03:34 2007 @@ -1056,7 +1056,8 @@ #for bonus points do big objects differently if raw_malloc_usage(totalsize) > self.top_of_space - self.free: raise memoryError - result = llarena.arena_reserve(self.free, totalsize) + result = self.free + llarena.arena_reserve(result, totalsize) self.init_gc_object(result, typeid) self.free += totalsize return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) @@ -1080,7 +1081,8 @@ #for bonus points do big objects differently if raw_malloc_usage(totalsize) > self.top_of_space - self.free: raise memoryError - result = llarena.arena_reserve(self.free, totalsize) + result = self.free + llarena.arena_reserve(result, totalsize) self.init_gc_object(result, typeid) (result + size_gc_header + offset_to_length).signed[0] = length self.free += totalsize @@ -1120,8 +1122,9 @@ ## print "already copied to", self.get_forwarding_address(obj) return self.get_forwarding_address(obj) else: + newaddr = self.free totalsize = self.size_gc_header() + self.get_size(obj) - newaddr = llarena.arena_reserve(self.free, totalsize) + llarena.arena_reserve(newaddr, totalsize) raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize) self.free += totalsize newobj = newaddr + self.size_gc_header() Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/test/test_gc.py Mon Oct 8 19:03:34 2007 @@ -87,8 +87,6 @@ class TestSemiSpaceGC(GCTest): from pypy.rpython.memory.gc import SemiSpaceGC as GCClass - def setup_class(cls): - py.test.skip("in-progress") class TestDeferredRefcountingGC(GCTest): from pypy.rpython.memory.gc import DeferredRefcountingGC as GCClass From arigo at codespeak.net Mon Oct 8 19:15:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 19:15:23 +0200 (CEST) Subject: [pypy-svn] r47300 - pypy/branch/kill-keepalives-again/pypy/rpython/memory Message-ID: <20071008171523.844C880FF@code0.codespeak.net> Author: arigo Date: Mon Oct 8 19:15:22 2007 New Revision: 47300 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcwrapper.py Log: Move the malloc() method used for tests to the GCBase class. This allows us to share this increasingly long-winded logic. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gc.py Mon Oct 8 19:15:22 2007 @@ -91,6 +91,34 @@ def size_gc_header(self, typeid=0): return self.gcheaderbuilder.size_gc_header + def malloc(self, typeid, length=0, zero=False): + """For testing. The interface used by the gctransformer is + the four malloc_[fixed,var]size[_clear]() functions. + """ + size = self.fixed_size(typeid) + needs_finalizer = bool(self.getfinalizer(typeid)) + contains_weakptr = self.weakpointer_offset(typeid) != -1 + assert not (needs_finalizer and contains_weakptr) + if self.is_varsize(typeid): + assert not contains_weakptr + itemsize = self.varsize_item_sizes(typeid) + offset_to_length = self.varsize_offset_to_length(typeid) + if zero: + malloc_varsize = self.malloc_varsize_clear + else: + malloc_varsize = self.malloc_varsize + ref = malloc_varsize(typeid, length, size, itemsize, + offset_to_length, True, needs_finalizer) + else: + if zero: + malloc_fixedsize = self.malloc_fixedsize_clear + else: + malloc_fixedsize = self.malloc_fixedsize + ref = malloc_fixedsize(typeid, size, True, needs_finalizer, + contains_weakptr) + # lots of cast and reverse-cast around... + return llmemory.cast_ptr_to_adr(ref) + def x_swap_pool(self, newpool): return newpool @@ -107,13 +135,16 @@ self.get_roots = get_roots #self.set_query_functions(None, None, None, None, None, None, None) - def malloc(self, typeid, length=0): + def malloc(self, typeid, length=0, zero=False): size = self.fixed_size(typeid) if self.is_varsize(typeid): size += length * self.varsize_item_sizes(typeid) result = raw_malloc(size) if not result: raise memoryError + if zero: + raw_memclear(result, size) + # XXX set the length field? return result def collect(self): @@ -176,24 +207,6 @@ self.collect_in_progress = False self.prev_collect_end_time = 0.0 - def malloc(self, typeid, length=0): - size = self.fixed_size(typeid) - needs_finalizer = bool(self.getfinalizer(typeid)) - contains_weakptr = self.weakpointer_offset(typeid) != -1 - assert not (needs_finalizer and contains_weakptr) - if self.is_varsize(typeid): - assert not contains_weakptr - itemsize = self.varsize_item_sizes(typeid) - offset_to_length = self.varsize_offset_to_length(typeid) - ref = self.malloc_varsize(typeid, length, size, itemsize, - offset_to_length, True, needs_finalizer) - else: - ref = self.malloc_fixedsize(typeid, size, True, needs_finalizer, - contains_weakptr) - # XXX lots of cast and reverse-cast around, but this malloc() - # should eventually be killed - return llmemory.cast_ptr_to_adr(ref) - def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: @@ -1026,19 +1039,6 @@ self.tospace = NULL llarena.arena_free(self.fromspace) self.fromspace = NULL - - def malloc(self, typeid, length=0): - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - itemsize = self.varsize_item_sizes(typeid) - offset_to_length = self.varsize_offset_to_length(typeid) - ref = self.malloc_varsize(typeid, length, size, itemsize, - offset_to_length, True) - else: - ref = self.malloc_fixedsize(typeid, size, True) - # XXX lots of cast and reverse-cast around, but this malloc() - # should eventually be killed - return llmemory.cast_ptr_to_adr(ref) def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): Modified: pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/memory/gcwrapper.py Mon Oct 8 19:15:22 2007 @@ -48,7 +48,7 @@ def malloc(self, TYPE, n=None, flavor='gc', zero=False): if flavor == 'gc': typeid = self.get_type_id(TYPE) - addr = self.gc.malloc(typeid, n) + addr = self.gc.malloc(typeid, n, zero=zero) return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) else: return lltype.malloc(TYPE, n, flavor=flavor, zero=zero) From fijal at codespeak.net Mon Oct 8 19:20:21 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Oct 2007 19:20:21 +0200 (CEST) Subject: [pypy-svn] r47301 - pypy/branch/bz2-module-rffi Message-ID: <20071008172021.AFFFF8108@code0.codespeak.net> Author: fijal Date: Mon Oct 8 19:20:19 2007 New Revision: 47301 Removed: pypy/branch/bz2-module-rffi/ Log: Remove dead branch From arigo at codespeak.net Mon Oct 8 19:20:41 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 19:20:41 +0200 (CEST) Subject: [pypy-svn] r47302 - pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test Message-ID: <20071008172041.5F4008108@code0.codespeak.net> Author: arigo Date: Mon Oct 8 19:20:40 2007 New Revision: 47302 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py Log: Typo. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py Mon Oct 8 19:20:40 2007 @@ -100,7 +100,7 @@ b = arena_malloc(20, False) if a > b: - a, b = a, b + a, b = b, a assert lt(a, b) assert lt(a+19, b) assert lt(a, b+19) From arigo at codespeak.net Mon Oct 8 19:30:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 19:30:20 +0200 (CEST) Subject: [pypy-svn] r47303 - pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test Message-ID: <20071008173020.574F580AB@code0.codespeak.net> Author: arigo Date: Mon Oct 8 19:30:20 2007 New Revision: 47303 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llmemory.py Log: Fix these tests. (Mostly by killing them, ha ha ha -<:-]) Modified: pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llmemory.py Mon Oct 8 19:30:20 2007 @@ -57,22 +57,6 @@ sizeof(array, 1) sizeof(varstruct, 2) -def test_cast_ptr_to_adr(): - from pypy.rpython.memory.test.test_llinterpsim import interpret - class A(object): - pass - def f(x): - if x: - a = A() - else: - a = None - adr_a = cast_ptr_to_adr(a) - return bool(adr_a) - res = interpret(f, [1]) - assert res - res = interpret(f, [0]) - assert not res - def test_confusion_with_fixedarray_item_0(): A = lltype.FixedSizeArray(lltype.Signed, 5) B = lltype.FixedSizeArray(A, 3) @@ -104,18 +88,6 @@ s = cast_adr_to_ptr(adr_s, lltype.Ptr(S)) assert s.x == 42 -def test_cast_adr_to_ptr(): - from pypy.rpython.memory.test.test_llinterpsim import interpret - S = lltype.GcStruct("S", ("x", lltype.Signed)) - Sptr = lltype.Ptr(S) - def f(): - s1 = lltype.malloc(S) - adr = cast_ptr_to_adr(s1) - s2 = cast_adr_to_ptr(adr, Sptr) - return s1 == s2 - res = interpret(f, []) - assert res - def test_fakeaddress_equality(): S = lltype.GcStruct('S', ('x', lltype.Signed)) T = lltype.GcStruct('T', ('y', lltype.Signed)) @@ -144,20 +116,6 @@ a_t, a_s = map(cast_ptr_to_adr, [s, t]) assert a_t == a_s -def test_cast_adr_to_int(): - from pypy.rpython.memory.test.test_llinterpsim import interpret - S = lltype.GcStruct("S", ("x", lltype.Signed)) - Sptr = lltype.Ptr(S) - def f(): - s1 = lltype.malloc(S) - adr = cast_ptr_to_adr(s1) - i = cast_adr_to_int(adr) - i2 = lltype.cast_ptr_to_int(s1) - return i == i2 - assert f() - res = interpret(f, []) - assert res - def test_fakeaccessor(): S = lltype.GcStruct("S", ("x", lltype.Signed), ("y", lltype.Signed)) s = lltype.malloc(S) @@ -285,18 +243,16 @@ T = lltype.GcStruct('T', ('z', lltype.Signed)) S = lltype.Struct('S', ('x', lltype.Signed), ('y', lltype.Ptr(T))) - from pypy.rpython.memory import lladdress # GRUMBLE! - size = sizeof(S) def test_read_uninit(): - adr = lladdress.raw_malloc(size) + adr = raw_malloc(size) s = cast_adr_to_ptr(adr, lltype.Ptr(S)) return s.x py.test.raises(lltype.UninitializedMemoryAccess, "interpret(test_read_uninit, [])") def test_read_init(): - adr = lladdress.raw_malloc(size) - lladdress.raw_memclear(adr, size) + adr = raw_malloc(size) + raw_memclear(adr, size) s = cast_adr_to_ptr(adr, lltype.Ptr(S)) return s.x res = interpret(test_read_init, []) From arigo at codespeak.net Mon Oct 8 19:34:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 19:34:54 +0200 (CEST) Subject: [pypy-svn] r47304 - pypy/branch/kill-keepalives-again/pypy/rpython Message-ID: <20071008173454.3DED880ED@code0.codespeak.net> Author: arigo Date: Mon Oct 8 19:34:53 2007 New Revision: 47304 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py Log: Fix stack mallocs. Modified: pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py Mon Oct 8 19:34:53 2007 @@ -269,7 +269,7 @@ if nextblock is None: self.llinterpreter.active_frame = self.f_back for obj in self.alloca_objects: - self.heap.free(obj, flavor='raw') + obj._obj._free() return args finally: if tracer: From arigo at codespeak.net Mon Oct 8 19:38:19 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 19:38:19 +0200 (CEST) Subject: [pypy-svn] r47305 - pypy/branch/kill-keepalives-again/pypy/rpython/test Message-ID: <20071008173819.529F380ED@code0.codespeak.net> Author: arigo Date: Mon Oct 8 19:38:18 2007 New Revision: 47305 Modified: pypy/branch/kill-keepalives-again/pypy/rpython/test/test_llinterp.py Log: Unsure about this test. Why did we use to get an AttributeError instead of the RuntimError: object already freed? Modified: pypy/branch/kill-keepalives-again/pypy/rpython/test/test_llinterp.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/rpython/test/test_llinterp.py (original) +++ pypy/branch/kill-keepalives-again/pypy/rpython/test/test_llinterp.py Mon Oct 8 19:38:18 2007 @@ -441,8 +441,9 @@ return globala.next.i interp, graph = get_interpreter(h, []) fgraph = graph.startblock.operations[0].args[0].value._obj.graph + assert fgraph.startblock.operations[0].opname == 'malloc' fgraph.startblock.operations[0].args[1] = inputconst(Void, {'flavor': "stack"}) - py.test.raises(AttributeError, "interp.eval_graph(graph, [])") + py.test.raises(RuntimeError, "interp.eval_graph(graph, [])") #__________________________________________________________________ # example functions for testing the LLInterpreter From fijal at codespeak.net Mon Oct 8 19:50:52 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Oct 2007 19:50:52 +0200 (CEST) Subject: [pypy-svn] r47306 - pypy/dist/pypy/rlib Message-ID: <20071008175052.C3C788136@code0.codespeak.net> Author: fijal Date: Mon Oct 8 19:50:52 2007 New Revision: 47306 Modified: pypy/dist/pypy/rlib/libffi.py Log: Perform some tasks automatically instead of manually Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Mon Oct 8 19:50:52 2007 @@ -12,6 +12,7 @@ class CConfig: _includes_ = includes + _libraries_ = ['ffi'] RTLD_LOCAL = rffi_platform.DefinedConstantInteger('RTLD_LOCAL') RTLD_NOW = rffi_platform.DefinedConstantInteger('RTLD_NOW') @@ -26,7 +27,27 @@ ('alignment', rffi.USHORT), ('type', rffi.USHORT), ('elements', FFI_TYPE_PP)]) - # XXX elements goes here, for structures + +def add_simple_type(type_name): + for name in ['size', 'alignment', 'type']: + setattr(CConfig, type_name + '_' + name, + rffi_platform.ConstantInteger(type_name + '.' + name)) + +def configure_simple_type(type_name): + l = lltype.malloc(FFI_TYPE_P.TO, flavor='raw', immortal=True) + for tp, name in [(size_t, 'size'), + (rffi.USHORT, 'alignment'), + (rffi.USHORT, 'type')]: + value = getattr(cConfig, '%s_%s' % (type_name, name)) + setattr(l, 'c_' + name, rffi.cast(tp, value)) + l.c_elements = lltype.nullptr(FFI_TYPE_PP.TO) + return l + +base_names = ['double', 'uchar', 'schar', 'sshort', 'ushort', 'uint', 'sint', + 'ulong', 'slong', 'float', 'pointer', 'void'] +type_names = ['ffi_type_%s' % name for name in base_names] +for i in type_names: + add_simple_type(i) class cConfig: pass @@ -36,6 +57,9 @@ FFI_TYPE_P.TO.become(cConfig.ffi_type) size_t = cConfig.size_t +for name in type_names: + locals()[name] = configure_simple_type(name) + def external(name, args, result): return rffi.llexternal(name, args, result, includes=includes, libraries=['dl', 'ffi']) @@ -49,7 +73,7 @@ RTLD_NOW = cConfig.RTLD_NOW FFI_OK = cConfig.FFI_OK FFI_BAD_TYPEDEF = cConfig.FFI_BAD_TYPEDEF -FFI_DEFAULT_ABI = cConfig.FFI_DEFAULT_ABI +FFI_DEFAULT_ABI = rffi.cast(rffi.USHORT, cConfig.FFI_DEFAULT_ABI) FFI_CIFP = rffi.COpaquePtr('ffi_cif', includes=includes) c_ffi_prep_cif = external('ffi_prep_cif', [FFI_CIFP, rffi.USHORT, rffi.UINT, @@ -57,14 +81,6 @@ c_ffi_call = external('ffi_call', [FFI_CIFP, rffi.VOIDP, rffi.VOIDP, rffi.CArrayPtr(rffi.VOIDP)], lltype.Void) -# XXX hardcode this values by now, we need some new logic/thinking for that - -ffi_type_sint = lltype.malloc(FFI_TYPE_P.TO, flavor='raw', immortal=True) -ffi_type_sint.c_size = rffi.cast(size_t, 4) -ffi_type_sint.c_alignment = rffi.cast(rffi.USHORT, 4) -ffi_type_sint.c_type = rffi.cast(rffi.USHORT, 10) -ffi_type_sint.c_elements = lltype.nullptr(FFI_TYPE_PP.TO) - def dlerror(): # XXX this would never work on top of ll2ctypes, because # ctypes are calling dlerror itself, unsure if I can do much in this @@ -102,8 +118,9 @@ raise OSError("NULL func_sym") self.func_sym = func_sym self.ll_cif = lltype.malloc(FFI_CIFP.TO, flavor='raw') - res = c_ffi_prep_cif(self.ll_cif, rffi.cast(rffi.USHORT, FFI_DEFAULT_ABI), - rffi.cast(rffi.UINT, 0), ffi_type_sint, lltype.nullptr(FFI_TYPE_PP.TO)) + res = c_ffi_prep_cif(self.ll_cif, FFI_DEFAULT_ABI, + rffi.cast(rffi.UINT, 0), + ffi_type_sint, lltype.nullptr(FFI_TYPE_PP.TO)) if not res == FFI_OK: raise OSError("Wrong typedef") From arigo at codespeak.net Mon Oct 8 19:55:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 19:55:04 +0200 (CEST) Subject: [pypy-svn] r47307 - pypy/branch/kill-keepalives-again/pypy/translator/c/test Message-ID: <20071008175504.715F58131@code0.codespeak.net> Author: arigo Date: Mon Oct 8 19:55:04 2007 New Revision: 47307 Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lladdresses.py Log: Fix import. Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lladdresses.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lladdresses.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_lladdresses.py Mon Oct 8 19:55:04 2007 @@ -1,4 +1,4 @@ -from pypy.rpython.memory.lladdress import * +from pypy.rpython.lltypesystem.llmemory import * from pypy.annotation.model import SomeAddress, SomeChar from pypy.translator.c.test.test_genc import compile from pypy.rlib.objectmodel import free_non_gc_object From arigo at codespeak.net Mon Oct 8 20:20:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 20:20:54 +0200 (CEST) Subject: [pypy-svn] r47308 - pypy/branch/kill-keepalives-again/pypy/translator/c/test Message-ID: <20071008182054.BA4D980DB@code0.codespeak.net> Author: arigo Date: Mon Oct 8 20:20:53 2007 New Revision: 47308 Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_symbolic.py pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_tasklets.py Log: Fix imports. Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_symbolic.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_symbolic.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_symbolic.py Mon Oct 8 20:20:53 2007 @@ -1,7 +1,6 @@ from pypy.translator.interactive import Translation from pypy import conftest from pypy.rpython.lltypesystem import llmemory, lltype -from pypy.rpython.memory import lladdress from pypy.rlib.objectmodel import ComputedIntSymbolic def getcompiled(f, args): @@ -83,11 +82,11 @@ sizeofs = llmemory.sizeof(STRUCT) offsety = llmemory.offsetof(STRUCT, 'y') def f(): - adr = lladdress.raw_malloc(sizeofs) + adr = llmemory.raw_malloc(sizeofs) s = llmemory.cast_adr_to_ptr(adr, STRUCTPTR) s.y = 5 # does not crash result = (adr + offsety).signed[0] * 10 + int(offsety < sizeofs) - lladdress.raw_free(adr) + llmemory.raw_free(adr) return result fn, t = getcompiled(f, []) res = fn() Modified: pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_tasklets.py ============================================================================== --- pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_tasklets.py (original) +++ pypy/branch/kill-keepalives-again/pypy/translator/c/test/test_tasklets.py Mon Oct 8 20:20:53 2007 @@ -1,7 +1,7 @@ import py import os -from pypy.rpython.memory.lladdress import NULL +from pypy.rpython.lltypesystem.llmemory import NULL from pypy.rlib.rstack import yield_current_frame_to_caller # ____________________________________________________________ From fijal at codespeak.net Mon Oct 8 20:30:24 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Oct 2007 20:30:24 +0200 (CEST) Subject: [pypy-svn] r47309 - in pypy/dist/pypy/rlib: . test Message-ID: <20071008183024.1D97880DC@code0.codespeak.net> Author: fijal Date: Mon Oct 8 20:30:23 2007 New Revision: 47309 Modified: pypy/dist/pypy/rlib/libffi.py pypy/dist/pypy/rlib/test/test_libffi.py Log: Calling pow via libffi via wrapper works! Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Mon Oct 8 20:30:23 2007 @@ -60,6 +60,24 @@ for name in type_names: locals()[name] = configure_simple_type(name) +TYPE_MAP = { + rffi.DOUBLE : ffi_type_double, + rffi.FLOAT : ffi_type_float, + rffi.UCHAR : ffi_type_uchar, + rffi.CHAR : ffi_type_schar, + rffi.SHORT : ffi_type_sshort, + rffi.USHORT : ffi_type_ushort, + rffi.UINT : ffi_type_uint, + rffi.INT : ffi_type_sint, + rffi.ULONG : ffi_type_ulong, + rffi.LONG : ffi_type_slong, + lltype.Void : ffi_type_void, + # some shortcuts + None : ffi_type_void, + float : ffi_type_double, + int : ffi_type_sint, + } + def external(name, args, result): return rffi.llexternal(name, args, result, includes=includes, libraries=['dl', 'ffi']) @@ -76,10 +94,12 @@ FFI_DEFAULT_ABI = rffi.cast(rffi.USHORT, cConfig.FFI_DEFAULT_ABI) FFI_CIFP = rffi.COpaquePtr('ffi_cif', includes=includes) +VOIDPP = rffi.CArrayPtr(rffi.VOIDP) + c_ffi_prep_cif = external('ffi_prep_cif', [FFI_CIFP, rffi.USHORT, rffi.UINT, FFI_TYPE_P, FFI_TYPE_PP], rffi.INT) c_ffi_call = external('ffi_call', [FFI_CIFP, rffi.VOIDP, rffi.VOIDP, - rffi.CArrayPtr(rffi.VOIDP)], lltype.Void) + VOIDPP], lltype.Void) def dlerror(): # XXX this would never work on top of ll2ctypes, because @@ -112,30 +132,55 @@ return res class FuncPtr: - def __init__(self, func_sym): - # xxx args here + def __init__(self, func_sym, argtypes, restype): + argnum = len(argtypes) + TP = rffi.CFixedArray(FFI_TYPE_P, argnum) + self.ll_argtypes = lltype.malloc(TP, flavor='raw') + self.argtypes = argtypes + for i, argtype in enumerate(argtypes): + self.ll_argtypes[i] = TYPE_MAP[argtype] + TP = rffi.CFixedArray(rffi.VOIDP, argnum) + self.ll_args = lltype.malloc(TP, flavor='raw') + for i in range(argnum): + # XXX + TP = rffi.CFixedArray(argtypes[i], 1) + self.ll_args[i] = rffi.cast(rffi.VOIDP, + lltype.malloc(TP, flavor='raw')) + self.restype = restype + if restype is not None: + TP = rffi.CFixedArray(restype, 1) + self.ll_res = lltype.malloc(TP, flavor='raw') if not func_sym: raise OSError("NULL func_sym") self.func_sym = func_sym self.ll_cif = lltype.malloc(FFI_CIFP.TO, flavor='raw') res = c_ffi_prep_cif(self.ll_cif, FFI_DEFAULT_ABI, - rffi.cast(rffi.UINT, 0), - ffi_type_sint, lltype.nullptr(FFI_TYPE_PP.TO)) + rffi.cast(rffi.UINT, argnum), + TYPE_MAP[restype], + rffi.cast(FFI_TYPE_PP, self.ll_argtypes)) if not res == FFI_OK: raise OSError("Wrong typedef") + __init__._annspecialcase_ = 'specialize:arg(2, 3)' def call(self, args): # allocated result should be padded and stuff PTR_T = lltype.Ptr(rffi.CFixedArray(rffi.INT, 1)) - result = lltype.malloc(PTR_T.TO, flavor='raw') - c_ffi_call(self.ll_cif, self.func_sym, rffi.cast(rffi.VOIDP, result), - lltype.nullptr(rffi.CCHARPP.TO)) - res = result[0] - lltype.free(result, flavor='raw') - return res + for i in range(len(args)): + TP = lltype.Ptr(rffi.CFixedArray(self.argtypes[i], 1)) + addr = rffi.cast(TP, self.ll_args[i]) + addr[0] = args[i] + c_ffi_call(self.ll_cif, self.func_sym, + rffi.cast(rffi.VOIDP, self.ll_res), + rffi.cast(VOIDPP, self.ll_args)) + return self.ll_res[0] + call._annspecialcase_ = 'specialize:argtype(1)' def __del__(self): + lltype.free(self.ll_argtypes, flavor='raw') + lltype.free(self.ll_args, flavor='raw') lltype.free(self.ll_cif, flavor='raw') + if self.restype is not None: + lltype.free(self.ll_res, flavor='raw') class CDLL: def __init__(self, libname): @@ -144,5 +189,5 @@ def __del__(self): c_dlclose(self.lib) - def getpointer(self, name): - return FuncPtr(dlsym(self.lib, name)) + def getpointer(self, name, argtypes, restype): + return FuncPtr(dlsym(self.lib, name), argtypes, restype) Modified: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_libffi.py (original) +++ pypy/dist/pypy/rlib/test/test_libffi.py Mon Oct 8 20:30:23 2007 @@ -5,6 +5,7 @@ from pypy.rpython.test.test_llinterp import interpret from pypy.rlib.libffi import CDLL, dlopen from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED +from pypy.rpython.lltypesystem import rffi import os, sys import py @@ -33,13 +34,13 @@ def test_library_get_func(self): lib = self.get_libc() - ptr = lib.getpointer('time') - py.test.raises(KeyError, lib.getpointer, 'xxxxxxxxxxxxxxx') + ptr = lib.getpointer('time', [], None) + py.test.raises(KeyError, lib.getpointer, 'xxxxxxxxxxxxxxx', [], None) del lib def test_library_func_call(self): lib = self.get_libc() - ptr = lib.getpointer('rand') + ptr = lib.getpointer('rand', [], rffi.INT) zeroes = 0 for i in range(100): res = ptr.call([]) @@ -47,3 +48,9 @@ zeroes += 1 assert not zeroes # not very hard check, but something :] + + def test_call_args(self): + libm = CDLL('libm.so') + pow = libm.getpointer('pow', [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE) + assert pow.call((2.0, 2.0)) == 4.0 + assert pow.call((3.0, 3.0)) == 27.0 From fijal at codespeak.net Mon Oct 8 20:45:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Oct 2007 20:45:51 +0200 (CEST) Subject: [pypy-svn] r47310 - in pypy/dist/pypy/rlib: . test Message-ID: <20071008184551.D43128118@code0.codespeak.net> Author: fijal Date: Mon Oct 8 20:45:51 2007 New Revision: 47310 Modified: pypy/dist/pypy/rlib/libffi.py pypy/dist/pypy/rlib/test/test_libffi.py Log: Move slightly into rpython direction. Need help though Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Mon Oct 8 20:45:51 2007 @@ -4,6 +4,7 @@ from pypy.rpython.tool import rffi_platform from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib.unroll import unrolling_iterable includes = ['dlfcn.h', 'ffi.h'] @@ -73,9 +74,6 @@ rffi.LONG : ffi_type_slong, lltype.Void : ffi_type_void, # some shortcuts - None : ffi_type_void, - float : ffi_type_double, - int : ffi_type_sint, } def external(name, args, result): @@ -137,7 +135,8 @@ TP = rffi.CFixedArray(FFI_TYPE_P, argnum) self.ll_argtypes = lltype.malloc(TP, flavor='raw') self.argtypes = argtypes - for i, argtype in enumerate(argtypes): + for i in unrolling_iterable(range(len(argtypes))): + argtype = argtypes[i] self.ll_argtypes[i] = TYPE_MAP[argtype] TP = rffi.CFixedArray(rffi.VOIDP, argnum) self.ll_args = lltype.malloc(TP, flavor='raw') @@ -191,3 +190,4 @@ def getpointer(self, name, argtypes, restype): return FuncPtr(dlsym(self.lib, name), argtypes, restype) + getpointer._annspecialcase_ = 'specialize:arg(2, 3)' Modified: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_libffi.py (original) +++ pypy/dist/pypy/rlib/test/test_libffi.py Mon Oct 8 20:45:51 2007 @@ -5,7 +5,7 @@ from pypy.rpython.test.test_llinterp import interpret from pypy.rlib.libffi import CDLL, dlopen from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED -from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem import rffi, lltype import os, sys import py @@ -34,8 +34,8 @@ def test_library_get_func(self): lib = self.get_libc() - ptr = lib.getpointer('time', [], None) - py.test.raises(KeyError, lib.getpointer, 'xxxxxxxxxxxxxxx', [], None) + ptr = lib.getpointer('time', [], lltype.Void) + py.test.raises(KeyError, lib.getpointer, 'xxxxxxxxxxxxxxx', [], lltype.Void) del lib def test_library_func_call(self): @@ -54,3 +54,12 @@ pow = libm.getpointer('pow', [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE) assert pow.call((2.0, 2.0)) == 4.0 assert pow.call((3.0, 3.0)) == 27.0 + + def test_compile(self): + py.test.skip("in-progress") + def f(x, y): + libm = CDLL('libm.so') + c_pow = libm.getpointer('pow', (rffi.DOUBLE, rffi.DOUBLE), rffi.DOUBLE) + return c_pow.call((x, y)) + + interpret(f, [2.0, 4.0]) From fijal at codespeak.net Mon Oct 8 20:52:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Oct 2007 20:52:41 +0200 (CEST) Subject: [pypy-svn] r47311 - pypy/dist/pypy/rlib Message-ID: <20071008185241.9E8B580F5@code0.codespeak.net> Author: fijal Date: Mon Oct 8 20:52:41 2007 New Revision: 47311 Modified: pypy/dist/pypy/rlib/libffi.py Log: More rpython. Still complains about broken casts Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Mon Oct 8 20:52:41 2007 @@ -117,7 +117,7 @@ mode = RTLD_NOW res = c_dlopen(name, mode) if not res: - raise OSError(dlerror()) + raise OSError(-1, dlerror()) return res def dlsym(libhandle, name): @@ -129,57 +129,60 @@ # XXX rffi.cast here... return res -class FuncPtr: - def __init__(self, func_sym, argtypes, restype): - argnum = len(argtypes) - TP = rffi.CFixedArray(FFI_TYPE_P, argnum) - self.ll_argtypes = lltype.malloc(TP, flavor='raw') - self.argtypes = argtypes - for i in unrolling_iterable(range(len(argtypes))): - argtype = argtypes[i] - self.ll_argtypes[i] = TYPE_MAP[argtype] - TP = rffi.CFixedArray(rffi.VOIDP, argnum) - self.ll_args = lltype.malloc(TP, flavor='raw') - for i in range(argnum): - # XXX - TP = rffi.CFixedArray(argtypes[i], 1) - self.ll_args[i] = rffi.cast(rffi.VOIDP, - lltype.malloc(TP, flavor='raw')) - self.restype = restype - if restype is not None: - TP = rffi.CFixedArray(restype, 1) - self.ll_res = lltype.malloc(TP, flavor='raw') - if not func_sym: - raise OSError("NULL func_sym") - self.func_sym = func_sym - self.ll_cif = lltype.malloc(FFI_CIFP.TO, flavor='raw') - res = c_ffi_prep_cif(self.ll_cif, FFI_DEFAULT_ABI, - rffi.cast(rffi.UINT, argnum), - TYPE_MAP[restype], - rffi.cast(FFI_TYPE_PP, self.ll_argtypes)) - if not res == FFI_OK: - raise OSError("Wrong typedef") - __init__._annspecialcase_ = 'specialize:arg(2, 3)' - - def call(self, args): - # allocated result should be padded and stuff - PTR_T = lltype.Ptr(rffi.CFixedArray(rffi.INT, 1)) - for i in range(len(args)): - TP = lltype.Ptr(rffi.CFixedArray(self.argtypes[i], 1)) - addr = rffi.cast(TP, self.ll_args[i]) - addr[0] = args[i] - c_ffi_call(self.ll_cif, self.func_sym, - rffi.cast(rffi.VOIDP, self.ll_res), - rffi.cast(VOIDPP, self.ll_args)) - return self.ll_res[0] - call._annspecialcase_ = 'specialize:argtype(1)' +def new_funcptr(argtypes, restype): + argnum = len(argtypes) - def __del__(self): - lltype.free(self.ll_argtypes, flavor='raw') - lltype.free(self.ll_args, flavor='raw') - lltype.free(self.ll_cif, flavor='raw') - if self.restype is not None: - lltype.free(self.ll_res, flavor='raw') + argtypes_iterable = unrolling_iterable(enumerate(argtypes)) + + class FuncPtr: + def __init__(self, func_sym): + TP = rffi.CFixedArray(FFI_TYPE_P, argnum) + self.ll_argtypes = lltype.malloc(TP, flavor='raw') + self.argtypes = argtypes + for i, argtype in argtypes_iterable: + self.ll_argtypes[i] = TYPE_MAP[argtype] + TP = rffi.CFixedArray(rffi.VOIDP, argnum) + self.ll_args = lltype.malloc(TP, flavor='raw') + for i, argtype in argtypes_iterable: + # XXX + TP = rffi.CFixedArray(argtypes[i], 1) + self.ll_args[i] = rffi.cast(rffi.VOIDP, + lltype.malloc(TP, flavor='raw')) + self.restype = restype + if restype is not None: + TP = rffi.CFixedArray(restype, 1) + self.ll_res = lltype.malloc(TP, flavor='raw') + if not func_sym: + raise OSError(-1, "NULL func_sym") + self.func_sym = func_sym + self.ll_cif = lltype.malloc(FFI_CIFP.TO, flavor='raw') + res = c_ffi_prep_cif(self.ll_cif, FFI_DEFAULT_ABI, + rffi.cast(rffi.UINT, argnum), + TYPE_MAP[restype], + rffi.cast(FFI_TYPE_PP, self.ll_argtypes)) + if not res == FFI_OK: + raise OSError(-1, "Wrong typedef") + + def call(self, args): + # allocated result should be padded and stuff + PTR_T = lltype.Ptr(rffi.CFixedArray(rffi.INT, 1)) + for i, argtype in argtypes_iterable: + TP = lltype.Ptr(rffi.CFixedArray(argtype, 1)) + addr = rffi.cast(TP, self.ll_args[i]) + addr[0] = args[i] + c_ffi_call(self.ll_cif, self.func_sym, + rffi.cast(rffi.VOIDP, self.ll_res), + rffi.cast(VOIDPP, self.ll_args)) + return self.ll_res[0] + + def __del__(self): + lltype.free(self.ll_argtypes, flavor='raw') + lltype.free(self.ll_args, flavor='raw') + lltype.free(self.ll_cif, flavor='raw') + if self.restype is not None: + lltype.free(self.ll_res, flavor='raw') + return FuncPtr +new_funcptr._annspecialcase_ = 'specialize:memo' class CDLL: def __init__(self, libname): @@ -189,5 +192,6 @@ c_dlclose(self.lib) def getpointer(self, name, argtypes, restype): - return FuncPtr(dlsym(self.lib, name), argtypes, restype) + funcptr = new_funcptr(argtypes, restype) + return funcptr(dlsym(self.lib, name)) getpointer._annspecialcase_ = 'specialize:arg(2, 3)' From fijal at codespeak.net Mon Oct 8 20:54:20 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Oct 2007 20:54:20 +0200 (CEST) Subject: [pypy-svn] r47312 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071008185420.5F2A080DD@code0.codespeak.net> Author: fijal Date: Mon Oct 8 20:54:19 2007 New Revision: 47312 Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: A helper Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Mon Oct 8 20:54:19 2007 @@ -212,11 +212,15 @@ def CFixedArray(tp, size): return lltype.FixedSizeArray(tp, size) +CFixedArray._annspecialcase_ = 'specialize:memo' def CArray(tp): return lltype.Array(tp, hints={'nolength': True}) CArray._annspecialcase_ = 'specialize:memo' +def CArrayPtr(tp): + return lltype.Ptr(CArray(tp)) + def COpaque(name, hints=None, **kwds): if hints is None: hints = {} From arigo at codespeak.net Mon Oct 8 21:17:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Oct 2007 21:17:12 +0200 (CEST) Subject: [pypy-svn] r47313 - in pypy/dist/pypy: annotation config config/test doc/config module/_stackless rlib rpython rpython/lltypesystem rpython/lltypesystem/test rpython/memory rpython/rctypes rpython/rctypes/test rpython/test translator translator/backendopt translator/c Message-ID: <20071008191712.AEEA4812B@code0.codespeak.net> Author: arigo Date: Mon Oct 8 21:17:10 2007 New Revision: 47313 Added: pypy/dist/pypy/annotation/ - copied from r47309, pypy/branch/kill-keepalives-again/pypy/annotation/ pypy/dist/pypy/config/test/ - copied from r47309, pypy/branch/kill-keepalives-again/pypy/config/test/ pypy/dist/pypy/config/translationoption.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/config/translationoption.py pypy/dist/pypy/doc/config/translation.frameworkgc.txt - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/doc/config/translation.frameworkgc.txt pypy/dist/pypy/doc/config/translation.gc.txt - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/doc/config/translation.gc.txt pypy/dist/pypy/doc/config/translation.stacklessgc.txt - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/doc/config/translation.stacklessgc.txt pypy/dist/pypy/module/_stackless/ - copied from r47309, pypy/branch/kill-keepalives-again/pypy/module/_stackless/ pypy/dist/pypy/rlib/rgc.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rlib/rgc.py pypy/dist/pypy/rpython/llinterp.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/llarena.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llarena.py pypy/dist/pypy/rpython/lltypesystem/llheap.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llheap.py pypy/dist/pypy/rpython/lltypesystem/llmemory.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/lltypesystem/lltype.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/lltype.py pypy/dist/pypy/rpython/lltypesystem/opimpl.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/opimpl.py pypy/dist/pypy/rpython/lltypesystem/rdict.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/rdict.py pypy/dist/pypy/rpython/lltypesystem/rstr.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/rstr.py pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llarena.py pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_llmemory.py pypy/dist/pypy/rpython/lltypesystem/test/test_lltype.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/lltypesystem/test/test_lltype.py pypy/dist/pypy/rpython/memory/ - copied from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/memory/ pypy/dist/pypy/rpython/raddress.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/raddress.py pypy/dist/pypy/rpython/rbuiltin.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/rbuiltin.py pypy/dist/pypy/rpython/rctypes/rarray.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rarray.py pypy/dist/pypy/rpython/rctypes/rchar_p.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rchar_p.py pypy/dist/pypy/rpython/rctypes/rmodel.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rmodel.py pypy/dist/pypy/rpython/rctypes/rvoid_p.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/rvoid_p.py pypy/dist/pypy/rpython/rctypes/test/test_rfunc.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/rctypes/test/test_rfunc.py pypy/dist/pypy/rpython/rptr.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/rptr.py pypy/dist/pypy/rpython/test/ - copied from r47309, pypy/branch/kill-keepalives-again/pypy/rpython/test/ pypy/dist/pypy/translator/backendopt/ - copied from r47309, pypy/branch/kill-keepalives-again/pypy/translator/backendopt/ pypy/dist/pypy/translator/c/ - copied from r47309, pypy/branch/kill-keepalives-again/pypy/translator/c/ pypy/dist/pypy/translator/exceptiontransform.py - copied unchanged from r47309, pypy/branch/kill-keepalives-again/pypy/translator/exceptiontransform.py Log: (arigo, cfbolz): Merge the kill-keepalives-again branch: - change the way interior pointers into structures and arrays are handled. Interior pointers are not allowed to cross function barriers any more. To achieve this, some new operations (setinteriorfield, getinteriorfield, getinteriorarraysize) are introduced - refactor the way the framework GCs are tested and move the semispace GC to the same type-safe approach the mark-n-sweep GC already uses. From cfbolz at codespeak.net Mon Oct 8 21:20:49 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 8 Oct 2007 21:20:49 +0200 (CEST) Subject: [pypy-svn] r47314 - pypy/branch/kill-keepalives-again Message-ID: <20071008192049.B04E1812C@code0.codespeak.net> Author: cfbolz Date: Mon Oct 8 21:20:49 2007 New Revision: 47314 Removed: pypy/branch/kill-keepalives-again/ Log: delete merged branch From cfbolz at codespeak.net Mon Oct 8 21:50:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 8 Oct 2007 21:50:53 +0200 (CEST) Subject: [pypy-svn] r47315 - in pypy/dist/pypy: annotation annotation/test jit/hintannotator jit/timeshifter rlib rpython rpython/lltypesystem rpython/module rpython/numpy rpython/ootypesystem rpython/rctypes translator/cli Message-ID: <20071008195053.01EE4812E@code0.codespeak.net> Author: cfbolz Date: Mon Oct 8 21:50:52 2007 New Revision: 47315 Removed: pypy/dist/pypy/annotation/pairtype.py Modified: pypy/dist/pypy/annotation/annrpython.py pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/description.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/annotation/test/test_pairtype.py pypy/dist/pypy/jit/hintannotator/model.py pypy/dist/pypy/jit/timeshifter/hrtyper.py pypy/dist/pypy/rlib/rmarshal.py pypy/dist/pypy/rpython/controllerentry.py pypy/dist/pypy/rpython/lltypesystem/rbuiltin.py pypy/dist/pypy/rpython/lltypesystem/rclass.py pypy/dist/pypy/rpython/lltypesystem/rdict.py pypy/dist/pypy/rpython/lltypesystem/rlist.py pypy/dist/pypy/rpython/lltypesystem/rpbc.py pypy/dist/pypy/rpython/lltypesystem/rslice.py pypy/dist/pypy/rpython/lltypesystem/rstr.py pypy/dist/pypy/rpython/lltypesystem/rtuple.py pypy/dist/pypy/rpython/module/ll_os_stat.py pypy/dist/pypy/rpython/numpy/aarray.py pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/ootypesystem/rbltregistry.py pypy/dist/pypy/rpython/ootypesystem/rclass.py pypy/dist/pypy/rpython/ootypesystem/rdict.py pypy/dist/pypy/rpython/ootypesystem/rlist.py pypy/dist/pypy/rpython/ootypesystem/rootype.py pypy/dist/pypy/rpython/ootypesystem/rpbc.py pypy/dist/pypy/rpython/raddress.py pypy/dist/pypy/rpython/rbool.py pypy/dist/pypy/rpython/rbuiltin.py pypy/dist/pypy/rpython/rctypes/afunc.py pypy/dist/pypy/rpython/rctypes/atype.py pypy/dist/pypy/rpython/rctypes/rarray.py pypy/dist/pypy/rpython/rctypes/rchar_p.py pypy/dist/pypy/rpython/rctypes/rmodel.py pypy/dist/pypy/rpython/rctypes/rpointer.py pypy/dist/pypy/rpython/rctypes/rprimitive.py pypy/dist/pypy/rpython/rctypes/rpyobject.py pypy/dist/pypy/rpython/rctypes/rstringbuf.py pypy/dist/pypy/rpython/rctypes/rtype.py pypy/dist/pypy/rpython/rctypes/rvoid_p.py pypy/dist/pypy/rpython/rdict.py pypy/dist/pypy/rpython/rexternalobj.py pypy/dist/pypy/rpython/rfloat.py pypy/dist/pypy/rpython/rgeneric.py pypy/dist/pypy/rpython/rint.py pypy/dist/pypy/rpython/rlist.py pypy/dist/pypy/rpython/rmodel.py pypy/dist/pypy/rpython/robject.py pypy/dist/pypy/rpython/rpbc.py pypy/dist/pypy/rpython/rptr.py pypy/dist/pypy/rpython/rrange.py pypy/dist/pypy/rpython/rstr.py pypy/dist/pypy/rpython/rtuple.py pypy/dist/pypy/rpython/rtyper.py pypy/dist/pypy/rpython/typesystem.py pypy/dist/pypy/translator/cli/dotnet.py Log: continue the move of pypy.annotation.pairtype to pypy.tool.pairtype: fix imports and kill the import forwarding file. Modified: pypy/dist/pypy/annotation/annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/annrpython.py (original) +++ pypy/dist/pypy/annotation/annrpython.py Mon Oct 8 21:50:52 2007 @@ -3,7 +3,7 @@ from types import ClassType, FunctionType from pypy.tool.ansi_print import ansi_log, raise_nicer_exception from pypy.annotation import model as annmodel -from pypy.annotation.pairtype import pair +from pypy.tool.pairtype import pair from pypy.annotation.bookkeeper import Bookkeeper, getbookkeeper from pypy.annotation import signature from pypy.objspace.flow.model import Variable, Constant Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Mon Oct 8 21:50:52 2007 @@ -4,7 +4,7 @@ import py import operator -from pypy.annotation.pairtype import pair, pairtype +from pypy.tool.pairtype import pair, pairtype from pypy.annotation.model import SomeObject, SomeInteger, SomeBool, s_Bool from pypy.annotation.model import SomeString, SomeChar, SomeList, SomeDict from pypy.annotation.model import SomeUnicodeCodePoint Modified: pypy/dist/pypy/annotation/description.py ============================================================================== --- pypy/dist/pypy/annotation/description.py (original) +++ pypy/dist/pypy/annotation/description.py Mon Oct 8 21:50:52 2007 @@ -4,7 +4,7 @@ from pypy.interpreter.argument import rawshape from pypy.interpreter.argument import ArgErr from pypy.tool.sourcetools import valid_identifier -from pypy.annotation.pairtype import extendabletype +from pypy.tool.pairtype import extendabletype class CallFamily: """A family of Desc objects that could be called from common call sites. Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Mon Oct 8 21:50:52 2007 @@ -30,7 +30,7 @@ from types import BuiltinFunctionType, MethodType, FunctionType import pypy.tool.instancemethod -from pypy.annotation.pairtype import pair, extendabletype +from pypy.tool.pairtype import pair, extendabletype from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong, base_int from pypy.rlib.rarithmetic import r_singlefloat Modified: pypy/dist/pypy/annotation/test/test_pairtype.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_pairtype.py (original) +++ pypy/dist/pypy/annotation/test/test_pairtype.py Mon Oct 8 21:50:52 2007 @@ -1,5 +1,5 @@ -from pypy.annotation.pairtype import pairtype, pair, extendabletype +from pypy.tool.pairtype import pairtype, pair, extendabletype def test_binop(): ### Binary operation example Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Mon Oct 8 21:50:52 2007 @@ -1,5 +1,5 @@ from pypy.annotation import model as annmodel -from pypy.annotation.pairtype import pair, pairtype +from pypy.tool.pairtype import pair, pairtype from pypy.jit.hintannotator.bookkeeper import getbookkeeper from pypy.rpython.lltypesystem import lltype, lloperation Modified: pypy/dist/pypy/jit/timeshifter/hrtyper.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/hrtyper.py (original) +++ pypy/dist/pypy/jit/timeshifter/hrtyper.py Mon Oct 8 21:50:52 2007 @@ -6,7 +6,7 @@ from pypy.translator.backendopt.ssa import SSA_to_SSI from pypy.annotation import model as annmodel from pypy.annotation import listdef -from pypy.annotation.pairtype import pair, pairtype +from pypy.tool.pairtype import pair, pairtype from pypy.rpython.annlowlevel import PseudoHighLevelCallable from pypy.rpython.annlowlevel import cast_instance_to_base_ptr from pypy.rpython.annlowlevel import cast_base_ptr_to_instance Modified: pypy/dist/pypy/rlib/rmarshal.py ============================================================================== --- pypy/dist/pypy/rlib/rmarshal.py (original) +++ pypy/dist/pypy/rlib/rmarshal.py Mon Oct 8 21:50:52 2007 @@ -5,7 +5,7 @@ from pypy.annotation import model as annmodel from pypy.annotation.signature import annotation from pypy.annotation.listdef import ListDef, TooLateForChange -from pypy.annotation.pairtype import pair, pairtype +from pypy.tool.pairtype import pair, pairtype from pypy.rlib.rarithmetic import formatd, r_longlong, intmask from pypy.rlib.rarithmetic import break_up_float, parts_to_float from pypy.rlib.unroll import unrolling_iterable Modified: pypy/dist/pypy/rpython/controllerentry.py ============================================================================== --- pypy/dist/pypy/rpython/controllerentry.py (original) +++ pypy/dist/pypy/rpython/controllerentry.py Mon Oct 8 21:50:52 2007 @@ -1,5 +1,5 @@ from pypy.annotation import model as annmodel -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation.bookkeeper import getbookkeeper from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.annlowlevel import cachedtype Modified: pypy/dist/pypy/rpython/lltypesystem/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rbuiltin.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import rclass Modified: pypy/dist/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rclass.py Mon Oct 8 21:50:52 2007 @@ -1,6 +1,6 @@ import sys import types -from pypy.annotation.pairtype import pairtype, pair +from pypy.tool.pairtype import pairtype, pair from pypy.objspace.flow.model import Constant from pypy.rpython.error import TyperError from pypy.rpython.rmodel import Repr, inputconst, warning, mangle Modified: pypy/dist/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rdict.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.objspace.flow.model import Constant from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\ Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rlist.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype, pair +from pypy.tool.pairtype import pairtype, pair from pypy.annotation import model as annmodel from pypy.rpython.error import TyperError from pypy.rpython.rmodel import Repr, IntegerRepr, inputconst Modified: pypy/dist/pypy/rpython/lltypesystem/rpbc.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rpbc.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rpbc.py Mon Oct 8 21:50:52 2007 @@ -1,6 +1,6 @@ import types import sys -from pypy.annotation.pairtype import pairtype, pair +from pypy.tool.pairtype import pairtype, pair from pypy.annotation import model as annmodel from pypy.annotation import description from pypy.objspace.flow.model import Constant, Variable Modified: pypy/dist/pypy/rpython/lltypesystem/rslice.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rslice.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rslice.py Mon Oct 8 21:50:52 2007 @@ -1,7 +1,7 @@ from pypy.rpython.rslice import AbstractSliceRepr from pypy.rpython.lltypesystem.lltype import \ GcStruct, Signed, Ptr, Void, malloc, PyObject, nullptr -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.robject import PyObjRepr, pyobj_repr from pypy.rpython.rmodel import inputconst, PyObjPtr, IntegerRepr Modified: pypy/dist/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rstr.py Mon Oct 8 21:50:52 2007 @@ -1,5 +1,5 @@ from weakref import WeakValueDictionary -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rlib.objectmodel import malloc_zero_filled, we_are_translated from pypy.rlib.objectmodel import debug_assert Modified: pypy/dist/pypy/rpython/lltypesystem/rtuple.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rtuple.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rtuple.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.rmodel import inputconst from pypy.rpython.robject import PyObjRepr, pyobj_repr from pypy.rpython.rtuple import AbstractTupleRepr, AbstractTupleIteratorRepr Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Mon Oct 8 21:50:52 2007 @@ -4,7 +4,7 @@ """ import os, sys from pypy.annotation import model as annmodel -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.tool.sourcetools import func_with_new_name from pypy.rpython import extregistry from pypy.rpython.extfunc import register_external Modified: pypy/dist/pypy/rpython/numpy/aarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/aarray.py (original) +++ pypy/dist/pypy/rpython/numpy/aarray.py Mon Oct 8 21:50:52 2007 @@ -1,5 +1,5 @@ from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.annotation.pairtype import pair, pairtype +from pypy.tool.pairtype import pair, pairtype from pypy.annotation.model import SomeExternalObject, SomeList, SomeImpossibleValue from pypy.annotation.model import SomeObject, SomeInteger, SomeFloat, SomeString, SomeChar, SomeTuple, SomeSlice from pypy.tool.error import AnnotatorError Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Mon Oct 8 21:50:52 2007 @@ -2,7 +2,7 @@ from pypy.annotation.model import lltype_to_annotation, SomeObject, SomeInteger from pypy.rpython.numpy import aarray from pypy.rpython.numpy.aarray import SomeArray -from pypy.annotation.pairtype import pairtype, pair +from pypy.tool.pairtype import pairtype, pair from pypy.rlib.unroll import unrolling_iterable from pypy.annotation import listdef from pypy.rlib.objectmodel import debug_assert Modified: pypy/dist/pypy/rpython/ootypesystem/rbltregistry.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rbltregistry.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rbltregistry.py Mon Oct 8 21:50:52 2007 @@ -3,7 +3,7 @@ from pypy.rpython.ootypesystem import ootype, bltregistry from pypy.rpython.rmodel import Repr from pypy.annotation.signature import annotation -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype class ExternalInstanceRepr(Repr): Modified: pypy/dist/pypy/rpython/ootypesystem/rclass.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rclass.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rclass.py Mon Oct 8 21:50:52 2007 @@ -7,7 +7,7 @@ from pypy.rpython.rclass import AbstractClassRepr, AbstractInstanceRepr, \ getinstancerepr, getclassrepr, get_type_repr from pypy.rpython.ootypesystem import ootype -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.tool.sourcetools import func_with_new_name CLASSTYPE = ootype.Instance("Object_meta", ootype.ROOT, Modified: pypy/dist/pypy/rpython/ootypesystem/rdict.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rdict.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rdict.py Mon Oct 8 21:50:52 2007 @@ -1,5 +1,5 @@ from pypy.rpython.error import TyperError -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.objspace.flow.model import Constant from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\ Modified: pypy/dist/pypy/rpython/ootypesystem/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rlist.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rlist.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.rlist import AbstractBaseListRepr, AbstractListRepr, \ AbstractListIteratorRepr, rtype_newlist, rtype_alloc_and_set from pypy.rpython.rmodel import Repr, IntegerRepr Modified: pypy/dist/pypy/rpython/ootypesystem/rootype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rootype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rootype.py Mon Oct 8 21:50:52 2007 @@ -2,7 +2,7 @@ from pypy.rpython.rmodel import Repr from pypy.rpython.ootypesystem import ootype from pypy.rpython.ootypesystem.ootype import Void, Class -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype class __extend__(annmodel.SomeOOClass): def rtyper_makerepr(self, rtyper): Modified: pypy/dist/pypy/rpython/ootypesystem/rpbc.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rpbc.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rpbc.py Mon Oct 8 21:50:52 2007 @@ -11,7 +11,7 @@ from pypy.rpython.ootypesystem.rclass import mangle from pypy.annotation import model as annmodel from pypy.annotation import description -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.objspace.flow.model import Constant, Variable import types Modified: pypy/dist/pypy/rpython/raddress.py ============================================================================== --- pypy/dist/pypy/rpython/raddress.py (original) +++ pypy/dist/pypy/rpython/raddress.py Mon Oct 8 21:50:52 2007 @@ -1,5 +1,5 @@ # rtyping of memory address operations -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem.llmemory import NULL, Address, \ cast_adr_to_int, fakeaddress Modified: pypy/dist/pypy/rpython/rbool.py ============================================================================== --- pypy/dist/pypy/rpython/rbool.py (original) +++ pypy/dist/pypy/rpython/rbool.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem.lltype import Signed, Unsigned, Bool, Float from pypy.rpython.error import TyperError Modified: pypy/dist/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/rbuiltin.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype, rclass, llmemory Modified: pypy/dist/pypy/rpython/rctypes/afunc.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/afunc.py (original) +++ pypy/dist/pypy/rpython/rctypes/afunc.py Mon Oct 8 21:50:52 2007 @@ -1,6 +1,6 @@ from pypy.annotation.model import SomeCTypesObject from pypy.annotation import model as annmodel -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rpython.rctypes.implementation import CTypesEntry from pypy.rpython.lltypesystem import lltype Modified: pypy/dist/pypy/rpython/rctypes/atype.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/atype.py (original) +++ pypy/dist/pypy/rpython/rctypes/atype.py Mon Oct 8 21:50:52 2007 @@ -4,7 +4,7 @@ from pypy.annotation.model import SomeCTypesObject from pypy.annotation.model import SomeBuiltin, SomeInteger, SomeString -from pypy.annotation.pairtype import pair, pairtype +from pypy.tool.pairtype import pair, pairtype from pypy.rpython.extregistry import ExtRegistryEntry Modified: pypy/dist/pypy/rpython/rctypes/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rarray.py (original) +++ pypy/dist/pypy/rpython/rctypes/rarray.py Mon Oct 8 21:50:52 2007 @@ -4,7 +4,7 @@ from pypy.rpython.rmodel import IntegerRepr, inputconst from pypy.rpython.rslice import AbstractSliceRepr from pypy.rpython.lltypesystem import lltype -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.rctypes.rmodel import CTypesRefRepr, CTypesValueRepr from pypy.rpython.rctypes.rmodel import genreccopy_arrayitem, reccopy, C_ZERO from pypy.rpython.rctypes.rmodel import unsafe_getfield Modified: pypy/dist/pypy/rpython/rctypes/rchar_p.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rchar_p.py (original) +++ pypy/dist/pypy/rpython/rctypes/rchar_p.py Mon Oct 8 21:50:52 2007 @@ -6,7 +6,7 @@ from pypy.rpython.rctypes.rmodel import unsafe_getfield from pypy.rpython.rctypes.rarray import ArrayRepr from pypy.rpython.rctypes.rstringbuf import StringBufRepr -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from ctypes import c_char, c_char_p, cast Modified: pypy/dist/pypy/rpython/rctypes/rmodel.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rmodel.py (original) +++ pypy/dist/pypy/rpython/rctypes/rmodel.py Mon Oct 8 21:50:52 2007 @@ -2,7 +2,7 @@ from pypy.rpython.error import TyperError from pypy.rpython.lltypesystem import lltype, llmemory from pypy.annotation.model import SomeCTypesObject -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype class CTypesRepr(Repr): Modified: pypy/dist/pypy/rpython/rctypes/rpointer.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rpointer.py (original) +++ pypy/dist/pypy/rpython/rctypes/rpointer.py Mon Oct 8 21:50:52 2007 @@ -1,7 +1,7 @@ from pypy.rpython.rmodel import IntegerRepr, inputconst from pypy.rpython.error import TyperError from pypy.rpython.lltypesystem import lltype -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.rctypes.rmodel import CTypesValueRepr, genreccopy from pypy.annotation.model import SomeCTypesObject from pypy.objspace.flow.model import Constant Modified: pypy/dist/pypy/rpython/rctypes/rprimitive.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rprimitive.py (original) +++ pypy/dist/pypy/rpython/rctypes/rprimitive.py Mon Oct 8 21:50:52 2007 @@ -1,7 +1,7 @@ from pypy.rpython.rmodel import inputconst from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.rstr import CharRepr, UniCharRepr -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.rmodel import IntegerRepr, FloatRepr from pypy.rpython.error import TyperError from pypy.rpython.rctypes.rmodel import CTypesValueRepr Modified: pypy/dist/pypy/rpython/rctypes/rpyobject.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rpyobject.py (original) +++ pypy/dist/pypy/rpython/rctypes/rpyobject.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.rmodel import inputconst from pypy.rpython.lltypesystem import lltype from pypy.rpython.rctypes.rmodel import CTypesValueRepr Modified: pypy/dist/pypy/rpython/rctypes/rstringbuf.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rstringbuf.py (original) +++ pypy/dist/pypy/rpython/rctypes/rstringbuf.py Mon Oct 8 21:50:52 2007 @@ -1,5 +1,5 @@ from pypy.rpython.lltypesystem import lltype -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.rmodel import IntegerRepr, inputconst from pypy.rpython.rctypes.rmodel import CTypesRefRepr from pypy.objspace.flow.model import Constant Modified: pypy/dist/pypy/rpython/rctypes/rtype.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rtype.py (original) +++ pypy/dist/pypy/rpython/rctypes/rtype.py Mon Oct 8 21:50:52 2007 @@ -1,5 +1,5 @@ from pypy.objspace.flow.model import Constant -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.lltypesystem import lltype from pypy.rpython.rmodel import Repr, IntegerRepr, inputconst from pypy.rpython.error import TyperError Modified: pypy/dist/pypy/rpython/rctypes/rvoid_p.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rvoid_p.py (original) +++ pypy/dist/pypy/rpython/rctypes/rvoid_p.py Mon Oct 8 21:50:52 2007 @@ -1,7 +1,7 @@ from pypy.rpython.rctypes.rmodel import CTypesValueRepr, C_ZERO from pypy.rpython.rctypes.rmodel import unsafe_getfield from pypy.rpython.rctypes.rstringbuf import StringBufRepr -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.rstr import AbstractStringRepr from pypy.rpython.lltypesystem.rstr import string_repr from pypy.rpython.rctypes.rchar_p import CCharPRepr Modified: pypy/dist/pypy/rpython/rdict.py ============================================================================== --- pypy/dist/pypy/rpython/rdict.py (original) +++ pypy/dist/pypy/rpython/rdict.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype Modified: pypy/dist/pypy/rpython/rexternalobj.py ============================================================================== --- pypy/dist/pypy/rpython/rexternalobj.py (original) +++ pypy/dist/pypy/rpython/rexternalobj.py Mon Oct 8 21:50:52 2007 @@ -8,7 +8,7 @@ from pypy.objspace.flow.model import Constant, Variable from pypy.rpython import extregistry from pypy.annotation.signature import annotation -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype # ExternalObjects Modified: pypy/dist/pypy/rpython/rfloat.py ============================================================================== --- pypy/dist/pypy/rpython/rfloat.py (original) +++ pypy/dist/pypy/rpython/rfloat.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem.lltype import \ Signed, Unsigned, SignedLongLong, Bool, Float, Void, pyobjectptr Modified: pypy/dist/pypy/rpython/rgeneric.py ============================================================================== --- pypy/dist/pypy/rpython/rgeneric.py (original) +++ pypy/dist/pypy/rpython/rgeneric.py Mon Oct 8 21:50:52 2007 @@ -2,7 +2,7 @@ from pypy.rpython.rmodel import Repr from pypy.rpython.rpbc import AbstractFunctionsPBCRepr,\ AbstractMethodsPBCRepr -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.lltypesystem import lltype class AbstractGenericCallableRepr(Repr): Modified: pypy/dist/pypy/rpython/rint.py ============================================================================== --- pypy/dist/pypy/rpython/rint.py (original) +++ pypy/dist/pypy/rpython/rint.py Mon Oct 8 21:50:52 2007 @@ -1,5 +1,5 @@ import sys -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.objspace.flow.objspace import op_appendices from pypy.rpython.lltypesystem.lltype import Signed, Unsigned, Bool, Float, \ Modified: pypy/dist/pypy/rpython/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/rlist.py (original) +++ pypy/dist/pypy/rpython/rlist.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype, pair +from pypy.tool.pairtype import pairtype, pair from pypy.objspace.flow.model import Constant from pypy.annotation import model as annmodel from pypy.rpython.error import TyperError Modified: pypy/dist/pypy/rpython/rmodel.py ============================================================================== --- pypy/dist/pypy/rpython/rmodel.py (original) +++ pypy/dist/pypy/rpython/rmodel.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype, extendabletype, pair +from pypy.tool.pairtype import pairtype, extendabletype, pair from pypy.annotation import model as annmodel from pypy.annotation import description from pypy.objspace.flow.model import Constant Modified: pypy/dist/pypy/rpython/robject.py ============================================================================== --- pypy/dist/pypy/rpython/robject.py (original) +++ pypy/dist/pypy/rpython/robject.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem.lltype import \ PyObject, Ptr, Void, pyobjectptr, nullptr, Bool Modified: pypy/dist/pypy/rpython/rpbc.py ============================================================================== --- pypy/dist/pypy/rpython/rpbc.py (original) +++ pypy/dist/pypy/rpython/rpbc.py Mon Oct 8 21:50:52 2007 @@ -1,6 +1,6 @@ import types import sys -from pypy.annotation.pairtype import pair, pairtype +from pypy.tool.pairtype import pair, pairtype from pypy.annotation import model as annmodel from pypy.annotation import description from pypy.objspace.flow.model import Constant Modified: pypy/dist/pypy/rpython/rptr.py ============================================================================== --- pypy/dist/pypy/rpython/rptr.py (original) +++ pypy/dist/pypy/rpython/rptr.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.objspace.flow import model as flowmodel from pypy.rpython.lltypesystem import lltype Modified: pypy/dist/pypy/rpython/rrange.py ============================================================================== --- pypy/dist/pypy/rpython/rrange.py (original) +++ pypy/dist/pypy/rpython/rrange.py Mon Oct 8 21:50:52 2007 @@ -1,4 +1,4 @@ -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rpython.lltypesystem.lltype import Signed, Void, Ptr from pypy.rpython.rmodel import Repr, IntegerRepr, IteratorRepr Modified: pypy/dist/pypy/rpython/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/rstr.py (original) +++ pypy/dist/pypy/rpython/rstr.py Mon Oct 8 21:50:52 2007 @@ -1,5 +1,5 @@ from pypy.tool.staticmethods import StaticMethods -from pypy.annotation.pairtype import pairtype, pair +from pypy.tool.pairtype import pairtype, pair from pypy.annotation import model as annmodel from pypy.rpython.error import TyperError from pypy.rpython.rmodel import IntegerRepr, IteratorRepr Modified: pypy/dist/pypy/rpython/rtuple.py ============================================================================== --- pypy/dist/pypy/rpython/rtuple.py (original) +++ pypy/dist/pypy/rpython/rtuple.py Mon Oct 8 21:50:52 2007 @@ -1,5 +1,5 @@ import operator -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.objspace.flow.model import Constant from pypy.rpython.error import TyperError Modified: pypy/dist/pypy/rpython/rtyper.py ============================================================================== --- pypy/dist/pypy/rpython/rtyper.py (original) +++ pypy/dist/pypy/rpython/rtyper.py Mon Oct 8 21:50:52 2007 @@ -14,7 +14,7 @@ from __future__ import generators import os import py -from pypy.annotation.pairtype import pair +from pypy.tool.pairtype import pair from pypy.annotation import model as annmodel from pypy.objspace.flow.model import Variable, Constant from pypy.objspace.flow.model import SpaceOperation, c_last_exception Modified: pypy/dist/pypy/rpython/typesystem.py ============================================================================== --- pypy/dist/pypy/rpython/typesystem.py (original) +++ pypy/dist/pypy/rpython/typesystem.py Mon Oct 8 21:50:52 2007 @@ -1,7 +1,7 @@ """typesystem.py -- Typesystem-specific operations for RTyper.""" -from pypy.annotation.pairtype import extendabletype +from pypy.tool.pairtype import extendabletype from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem import lltype @@ -178,7 +178,7 @@ # Multiple dispatch on type system and high-level annotation -from pypy.annotation.pairtype import pairtype +from pypy.tool.pairtype import pairtype from pypy.annotation.model import SomeObject class __extend__(pairtype(TypeSystem, SomeObject)): Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Mon Oct 8 21:50:52 2007 @@ -1,6 +1,6 @@ import types -from pypy.annotation.pairtype import pair, pairtype +from pypy.tool.pairtype import pair, pairtype from pypy.annotation.model import SomeObject, SomeInstance, SomeOOInstance, SomeInteger, s_None,\ s_ImpossibleValue, lltype_to_annotation, annotation_to_lltype, SomeChar, SomeString, SomePBC from pypy.annotation.binaryop import _make_none_union From cfbolz at codespeak.net Mon Oct 8 21:53:31 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 8 Oct 2007 21:53:31 +0200 (CEST) Subject: [pypy-svn] r47316 - in pypy/dist/pypy: annotation/test tool/test Message-ID: <20071008195331.30EDF812E@code0.codespeak.net> Author: cfbolz Date: Mon Oct 8 21:53:30 2007 New Revision: 47316 Added: pypy/dist/pypy/tool/test/test_pairtype.py - copied unchanged from r47315, pypy/dist/pypy/annotation/test/test_pairtype.py Removed: pypy/dist/pypy/annotation/test/test_pairtype.py Log: move tests of pairtype to tool/test too From cfbolz at codespeak.net Mon Oct 8 23:37:04 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 8 Oct 2007 23:37:04 +0200 (CEST) Subject: [pypy-svn] r47320 - pypy/dist/pypy/rpython/memory Message-ID: <20071008213704.B1F348139@code0.codespeak.net> Author: cfbolz Date: Mon Oct 8 23:37:02 2007 New Revision: 47320 Modified: pypy/dist/pypy/rpython/memory/gc.py Log: remove some rests of the memory simulator approach to doing things Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Mon Oct 8 23:37:02 2007 @@ -20,32 +20,6 @@ class GCError(Exception): pass -def get_dummy_annotate(gc, AddressLinkedList): - def dummy_annotate(): - gc.setup() - gc.get_roots = dummy_get_roots1 #prevent the get_roots attribute to - gc.get_roots = dummy_get_roots2 #be constants - a = gc.malloc(1, 2) - b = gc.malloc(2, 3) - gc.write_barrier(raw_malloc(1), raw_malloc(2), raw_malloc(1)) - gc.collect() - return a - b - - def dummy_get_roots1(): - ll = AddressLinkedList() - ll.append(NULL) - ll.append(raw_malloc(10)) - ll.pop() #make the annotator see pop - return ll - - def dummy_get_roots2(): - ll = AddressLinkedList() - ll.append(raw_malloc(10)) - ll.append(NULL) - ll.pop() #make the annotator see pop - return ll - return dummy_annotate, dummy_get_roots1, dummy_get_roots2 - gc_interface = { "malloc": lltype.FuncType((lltype.Signed, lltype.Signed), llmemory.Address), From cfbolz at codespeak.net Mon Oct 8 23:52:36 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 8 Oct 2007 23:52:36 +0200 (CEST) Subject: [pypy-svn] r47321 - pypy/dist/pypy/rpython/memory Message-ID: <20071008215236.66E20813C@code0.codespeak.net> Author: cfbolz Date: Mon Oct 8 23:52:35 2007 New Revision: 47321 Modified: pypy/dist/pypy/rpython/memory/gc.py Log: seems this stuff is no longer needed either Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Mon Oct 8 23:52:35 2007 @@ -17,17 +17,6 @@ X_CLONE_PTR = lltype.Ptr(X_CLONE) -class GCError(Exception): - pass - - -gc_interface = { - "malloc": lltype.FuncType((lltype.Signed, lltype.Signed), llmemory.Address), - "collect": lltype.FuncType((), lltype.Void), - "write_barrier": lltype.FuncType((llmemory.Address, ) * 3, lltype.Void), - } - - class GCBase(object): _alloc_flavor_ = "raw" @@ -102,31 +91,6 @@ def x_become(self, target_addr, source_addr): raise RuntimeError("no support for x_become in the GC") -class DummyGC(GCBase): - _alloc_flavor_ = "raw" - - def __init__(self, AddressLinkedList, dummy=None, get_roots=None): - self.get_roots = get_roots - #self.set_query_functions(None, None, None, None, None, None, None) - - def malloc(self, typeid, length=0, zero=False): - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - size += length * self.varsize_item_sizes(typeid) - result = raw_malloc(size) - if not result: - raise memoryError - if zero: - raw_memclear(result, size) - # XXX set the length field? - return result - - def collect(self): - self.get_roots() #this is there so that the annotator thinks get_roots is a function - - def init_gc_object(self, addr, typeid): - return - init_gc_object_immortal = init_gc_object DEBUG_PRINT = False memoryError = MemoryError() From cfbolz at codespeak.net Tue Oct 9 00:01:43 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 9 Oct 2007 00:01:43 +0200 (CEST) Subject: [pypy-svn] r47323 - pypy/dist/pypy/rpython/memory Message-ID: <20071008220143.37138814B@code0.codespeak.net> Author: cfbolz Date: Tue Oct 9 00:01:42 2007 New Revision: 47323 Modified: pypy/dist/pypy/rpython/memory/gc.py Log: more dead code Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Tue Oct 9 00:01:42 2007 @@ -40,11 +40,6 @@ def write_barrier(self, addr, addr_to, addr_struct): addr_to.address[0] = addr - def free_memory(self): - #this will never be called at runtime, just during setup - "NOT_RPYTHON" - pass - def setup(self): pass @@ -971,13 +966,6 @@ debug_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace - def free_memory(self): - "NOT_RPYTHON" - llarena.arena_free(self.tospace) - self.tospace = NULL - llarena.arena_free(self.fromspace) - self.fromspace = NULL - def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): if has_finalizer: From fijal at codespeak.net Tue Oct 9 09:28:07 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Oct 2007 09:28:07 +0200 (CEST) Subject: [pypy-svn] r47324 - pypy/dist/demo/distribution Message-ID: <20071009072807.C1C7E818F@code0.codespeak.net> Author: fijal Date: Tue Oct 9 09:28:06 2007 New Revision: 47324 Modified: pypy/dist/demo/distribution/client.py Log: Fix exampl. Now this is really remote traceback. Modified: pypy/dist/demo/distribution/client.py ============================================================================== --- pypy/dist/demo/distribution/client.py (original) +++ pypy/dist/demo/distribution/client.py Tue Oct 9 09:28:06 2007 @@ -26,7 +26,7 @@ # XXX next one does not work, while it should. Too much mangling with remote # traceback frames probably try: - x.meth() # wrong argument numbers + x.meth(1, 2) # non-callable argument, AssertionError except: import sys e, c, tb = sys.exc_info() From cfbolz at codespeak.net Tue Oct 9 10:57:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 9 Oct 2007 10:57:30 +0200 (CEST) Subject: [pypy-svn] r47325 - in pypy/dist/pypy/rpython/memory: . test Message-ID: <20071009085730.E57AD81AC@code0.codespeak.net> Author: cfbolz Date: Tue Oct 9 10:57:29 2007 New Revision: 47325 Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py pypy/dist/pypy/rpython/memory/test/test_gc.py Log: make collections possible Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Tue Oct 9 10:57:29 2007 @@ -61,6 +61,11 @@ # XXX use write_barrier - but we need the address of the GcStruct array[index] = newitem + # XXX do we need a barrier for setinteriorfield too? + + def collect(self): + self.gc.collect() + # ____________________________________________________________ Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Tue Oct 9 10:57:29 2007 @@ -4,6 +4,9 @@ #from pypy.rpython.memory.support import INT_SIZE from pypy.rpython.memory import gcwrapper from pypy.rpython.test.test_llinterp import get_interpreter +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rlib.objectmodel import we_are_translated def stdout_ignore_ll_functions(msg): @@ -82,6 +85,24 @@ assert res == concat(100) #assert simulator.current_size - curr < 16000 * INT_SIZE / 4 + + def test_collect(self): + #curr = simulator.current_size + def concat(j): + lst = [] + for i in range(j): + lst.append(str(i)) + result = len("".join(lst)) + if we_are_translated(): + # can't call llop.gc__collect directly + llop.gc__collect(lltype.Void) + return result + res = self.interpret(concat, [100]) + assert res == concat(100) + #assert simulator.current_size - curr < 16000 * INT_SIZE / 4 + + + class TestMarkSweepGC(GCTest): from pypy.rpython.memory.gc import MarkSweepGC as GCClass From arigo at codespeak.net Tue Oct 9 11:22:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 11:22:22 +0200 (CEST) Subject: [pypy-svn] r47327 - in pypy/dist/pypy: rpython/lltypesystem rpython/lltypesystem/test translator/c Message-ID: <20071009092222.6BC2081A8@code0.codespeak.net> Author: arigo Date: Tue Oct 9 11:22:21 2007 New Revision: 47327 Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py pypy/dist/pypy/translator/c/primitive.py Log: Remove the previous arena support from llmemory. Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Tue Oct 9 11:22:21 2007 @@ -635,57 +635,3 @@ assert lltype.typeOf(source) == Address assert lltype.typeOf(dest) == Address size.raw_memcopy(source, dest) - -# ____________________________________________________________ - -ARENA_ITEM = lltype.OpaqueType('ArenaItem') - -class _arena(object): - #_cache = weakref.WeakKeyDictionary() # {obj: _arenaitem} - - def __init__(self, rng, zero): - self.rng = rng - self.zero = zero - self.items = [] - - def getitemaddr(self, n): - while len(self.items) <= n: - self.items.append(_arenaitem(self, len(self.items))) - return fakeaddress(self.items[n]._as_ptr()) - -class _arenaitem(lltype._container): - _TYPE = ARENA_ITEM - - def __init__(self, arena, nr): - self.arena = arena - self.nr = nr - self.reserved_size = None - - def reserve(self, size): - if self.reserved_size is None: - # xxx check that we are not larger than unitsize*n - itemadr = raw_malloc(size) - self.container = itemadr.ptr._obj - #_arena._cache[itemadr.ptr._obj] = self - else: - assert size == self.reserved_size - -class ArenaRange(AddressOffset): - def __init__(self, unitsize, n): - self.unitsize = unitsize - self.n = n - - def _raw_malloc(self, rest, zero=False): - assert not rest - arena = _arena(self, zero=zero) - return arena.getitemaddr(0) - -def arena(TYPE, n): - return ArenaRange(sizeof(TYPE), n) - -def bump(adr, size): - baseptr = cast_adr_to_ptr(adr, lltype.Ptr(ARENA_ITEM)) - baseptr._obj.reserve(size) - arena = baseptr._obj.arena - nr = baseptr._obj.nr - return arena.getitemaddr(nr + 1) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py Tue Oct 9 11:22:21 2007 @@ -498,26 +498,6 @@ assert (adr + offsetof(S, 't') + offsetof(T, 'x')).signed[0] == 3 -def test_arena_bump_ptr(): - S = lltype.Struct('S', ('x',lltype.Signed)) - SPTR = lltype.Ptr(S) - badr = start = raw_malloc(arena(S, 4)) - s_adr = badr - badr = bump(badr, sizeof(S)) - s_ptr = cast_adr_to_ptr(s_adr, SPTR) - s_ptr.x = 1 - s2_adr = badr - badr = bump(badr, sizeof(S)) - s2_ptr = cast_adr_to_ptr(s2_adr, SPTR) - s2_ptr.x = 2 - badr = start - s_ptr = cast_adr_to_ptr(badr, SPTR) - assert s_ptr.x == 1 - badr = bump(badr, sizeof(S)) - s2_ptr = cast_adr_to_ptr(badr, SPTR) - assert s2_ptr.x == 2 - # release(start) - def test_weakref(): S1 = lltype.GcStruct('S1', ('x',lltype.Signed)) S = lltype.GcStruct('S', ('s1', S1)) Modified: pypy/dist/pypy/translator/c/primitive.py ============================================================================== --- pypy/dist/pypy/translator/c/primitive.py (original) +++ pypy/dist/pypy/translator/c/primitive.py Tue Oct 9 11:22:21 2007 @@ -7,7 +7,7 @@ from pypy.rpython.lltypesystem.llmemory import Address, \ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ CompositeOffset, ArrayLengthOffset, \ - GCHeaderOffset, ArenaRange + GCHeaderOffset from pypy.translator.c.support import cdecl, barebonearray # ____________________________________________________________ @@ -48,8 +48,6 @@ return '0' elif type(value) == GCHeaderOffset: return '0' - elif isinstance(value, ArenaRange): - return '(%d * %d)' % (value.n, name_signed(value.unitsize, db)) elif isinstance(value, CDefinedIntSymbolic): return str(value.expr) elif isinstance(value, ComputedIntSymbolic): From cfbolz at codespeak.net Tue Oct 9 11:45:11 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 9 Oct 2007 11:45:11 +0200 (CEST) Subject: [pypy-svn] r47328 - in pypy/dist/pypy/rpython/memory: . test Message-ID: <20071009094511.79A8381AB@code0.codespeak.net> Author: cfbolz Date: Tue Oct 9 11:45:11 2007 New Revision: 47328 Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py pypy/dist/pypy/rpython/memory/test/test_gc.py Log: enable finalizer test and fix gcwrapper to make it pass Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Tue Oct 9 11:45:11 2007 @@ -1,4 +1,5 @@ from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython import llinterp from pypy.rpython.memory.support import get_address_linked_list from pypy.rpython.memory import gctypelayout from pypy.objspace.flow.model import Constant @@ -16,7 +17,7 @@ self.gc.setup() def prepare_graphs(self, flowgraphs): - layoutbuilder = DirectRunLayoutBuilder() + layoutbuilder = DirectRunLayoutBuilder(self.llinterp) self.get_type_id = layoutbuilder.get_type_id self.gc.set_query_functions(*layoutbuilder.get_query_functions()) @@ -71,8 +72,29 @@ class DirectRunLayoutBuilder(gctypelayout.TypeLayoutBuilder): + def __init__(self, llinterp): + self.llinterp = llinterp + super(DirectRunLayoutBuilder, self).__init__() + def make_finalizer_funcptr_for_type(self, TYPE): - return None # XXX + from pypy.rpython.memory.gctransform.support import get_rtti, \ + type_contains_pyobjs + rtti = get_rtti(TYPE) + if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'): + destrptr = rtti._obj.destructor_funcptr + DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0] + destrgraph = destrptr._obj.graph + else: + return None + + assert not type_contains_pyobjs(TYPE), "not implemented" + def ll_finalizer(addr): + try: + v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) + self.llinterp.eval_graph(destrgraph, [v]) + except llinterp.LLException: + print "a destructor raised an exception, ignoring it" + return ll_finalizer def collect_constants(graphs): Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Tue Oct 9 11:45:11 2007 @@ -101,6 +101,29 @@ assert res == concat(100) #assert simulator.current_size - curr < 16000 * INT_SIZE / 4 + def test_finalizer(self): + class B(object): + pass + b = B() + b.nextid = 0 + b.num_deleted = 0 + class A(object): + def __init__(self): + self.id = b.nextid + b.nextid += 1 + def __del__(self): + b.num_deleted += 1 + def f(x): + a = A() + i = 0 + while i < x: + i += 1 + a = A() + llop.gc__collect(lltype.Void) + llop.gc__collect(lltype.Void) + return b.num_deleted + res = self.interpret(f, [5]) + assert res == 6 class TestMarkSweepGC(GCTest): @@ -108,6 +131,8 @@ class TestSemiSpaceGC(GCTest): from pypy.rpython.memory.gc import SemiSpaceGC as GCClass + def test_finalizer(self): + py.test.skip("implement me") class TestDeferredRefcountingGC(GCTest): from pypy.rpython.memory.gc import DeferredRefcountingGC as GCClass From cfbolz at codespeak.net Tue Oct 9 11:54:46 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 9 Oct 2007 11:54:46 +0200 (CEST) Subject: [pypy-svn] r47329 - in pypy/dist/pypy/rpython/memory: . test Message-ID: <20071009095446.3B0E6817D@code0.codespeak.net> Author: cfbolz Date: Tue Oct 9 11:54:45 2007 New Revision: 47329 Modified: pypy/dist/pypy/rpython/memory/gc.py pypy/dist/pypy/rpython/memory/test/test_gc.py Log: implement finalizers in the semispace GC Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Tue Oct 9 11:54:45 2007 @@ -957,6 +957,9 @@ self.free = NULL self.get_roots = get_roots self.gcheaderbuilder = GCHeaderBuilder(self.HDR) + self.AddressLinkedList = AddressLinkedList + self.objects_with_finalizers = AddressLinkedList() + self.run_finalizers = AddressLinkedList() def setup(self): self.tospace = llarena.arena_malloc(self.space_size, True) @@ -968,8 +971,6 @@ def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): - if has_finalizer: - raise NotImplementedError("finalizers in SemiSpaceGC") if contains_weakptr: raise NotImplementedError("weakptr in SemiSpaceGC") size_gc_header = self.gcheaderbuilder.size_gc_header @@ -986,6 +987,8 @@ llarena.arena_reserve(result, totalsize) self.init_gc_object(result, typeid) self.free += totalsize + if has_finalizer: + self.objects_with_finalizers.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, @@ -1012,6 +1015,8 @@ self.init_gc_object(result, typeid) (result + size_gc_header + offset_to_length).signed[0] = length self.free += totalsize + if has_finalizer: + self.objects_with_finalizers.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) # for now, the spaces are filled with zeroes in advance @@ -1031,14 +1036,31 @@ root = roots.pop() if root == NULL: break -## print "root", root, root.address[0] root.address[0] = self.copy(root.address[0]) free_non_gc_object(roots) + scan = self.scan_copied(scan) + # walk over list of objects with finalizers + # if it is not copied, add it to the list of to-be-called finalizers + # and copy it, to me make the finalizer runnable + new_with_finalizer = self.AddressLinkedList() + while self.objects_with_finalizers.non_empty(): + obj = self.objects_with_finalizers.pop() + if self.is_forwarded(obj): + new_with_finalizer.append(self.get_forwarding_address(obj)) + else: + self.run_finalizers.append(self.copy(obj)) + scan = self.scan_copied(scan) + self.objects_with_finalizers.delete() + self.objects_with_finalizers = new_with_finalizer + llarena.arena_reset(fromspace, self.space_size, True) + self.execute_finalizers() + + def scan_copied(self, scan): while scan < self.free: curr = scan + self.size_gc_header() self.trace_and_copy(curr) scan += self.get_size(curr) + self.size_gc_header() - llarena.arena_reset(fromspace, self.space_size, True) + return scan def copy(self, obj): if not self.fromspace <= obj < self.fromspace + self.space_size: @@ -1125,6 +1147,13 @@ hdr.forw = NULL hdr.typeid = typeid + def execute_finalizers(self): + while self.run_finalizers.non_empty(): + obj = self.run_finalizers.pop() + hdr = self.header(obj) + finalizer = self.getfinalizer(hdr.typeid) + finalizer(obj) + class DeferredRefcountingGC(GCBase): _alloc_flavor_ = "raw" Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Tue Oct 9 11:54:45 2007 @@ -131,8 +131,6 @@ class TestSemiSpaceGC(GCTest): from pypy.rpython.memory.gc import SemiSpaceGC as GCClass - def test_finalizer(self): - py.test.skip("implement me") class TestDeferredRefcountingGC(GCTest): from pypy.rpython.memory.gc import DeferredRefcountingGC as GCClass From arigo at codespeak.net Tue Oct 9 12:09:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 12:09:49 +0200 (CEST) Subject: [pypy-svn] r47332 - in pypy/dist/pypy: annotation rpython/lltypesystem rpython/test Message-ID: <20071009100949.4528D81A8@code0.codespeak.net> Author: arigo Date: Tue Oct 9 12:09:47 2007 New Revision: 47332 Modified: pypy/dist/pypy/annotation/specialize.py pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/test/test_normalizecalls.py Log: Allow llmemory.sizeof(T) calls to be constant-folded during translation. This requires supporting default arguments in specialize:memo. Modified: pypy/dist/pypy/annotation/specialize.py ============================================================================== --- pypy/dist/pypy/annotation/specialize.py (original) +++ pypy/dist/pypy/annotation/specialize.py Tue Oct 9 12:09:47 2007 @@ -222,6 +222,7 @@ entrypoint = make_subhelper(args_so_far = ()) self.graph = annotator.translator.buildflowgraph(entrypoint) + self.graph.defaults = self.funcdesc.defaults # schedule this new graph for being annotated args_s = [] Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Tue Oct 9 12:09:47 2007 @@ -293,7 +293,8 @@ itemoffsetof(TYPE._flds[TYPE._arrayfld], n) else: raise Exception("don't know how to take the size of a %r"%TYPE) - +sizeof._annspecialcase_ = 'specialize:memo' # only for n == None + def offsetof(TYPE, fldname): assert fldname in TYPE._flds return FieldOffset(TYPE, fldname) Modified: pypy/dist/pypy/rpython/test/test_normalizecalls.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_normalizecalls.py (original) +++ pypy/dist/pypy/rpython/test/test_normalizecalls.py Tue Oct 9 12:09:47 2007 @@ -249,4 +249,27 @@ assert sub2_graph.getreturnvar().concretetype == lltype.Signed assert sub3_graph.getreturnvar().concretetype == lltype.Signed assert dummyfn_graph.getreturnvar().concretetype == lltype.Signed - + + def test_call_memoized_function_with_defaults(self): + class Freezing: + def _freeze_(self): + return True + fr1 = Freezing(); fr1.x = 1 + fr2 = Freezing(); fr2.x = 2 + def getorbuild(key1, key2=fr2, flag3=True): + return key1.x * 100 + key2.x * 10 + flag3 + getorbuild._annspecialcase_ = "specialize:memo" + + def f1(i): + if i > 0: + fr = fr1 + else: + fr = fr2 + if i % 2: + return getorbuild(fr) + else: + return getorbuild(fr, fr2, False) + + for i in [-7, -2, 100, 5]: + res = interpret(f1, [i]) + assert res == f1(i) From arigo at codespeak.net Tue Oct 9 12:26:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 12:26:32 +0200 (CEST) Subject: [pypy-svn] r47333 - in pypy/dist/pypy/rpython: . lltypesystem lltypesystem/test Message-ID: <20071009102632.63DCA81A8@code0.codespeak.net> Author: arigo Date: Tue Oct 9 12:26:31 2007 New Revision: 47333 Modified: pypy/dist/pypy/rpython/extfunc.py pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/llarena.py pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py Log: Annotate and translate the arena functions. Extended the register_external() logic so that if we specify both llimpl and llfakeimpl, the latter is called by the llinterpreter and the former is translated to C. Modified: pypy/dist/pypy/rpython/extfunc.py ============================================================================== --- pypy/dist/pypy/rpython/extfunc.py (original) +++ pypy/dist/pypy/rpython/extfunc.py Tue Oct 9 12:26:31 2007 @@ -191,6 +191,8 @@ '_name': self.name, '_safe_not_sandboxed': self.safe_not_sandboxed, } + if hasattr(self, fake_method_name): + impl._llfnobjattrs_['_fakeimpl'] = fakeimpl obj = rtyper.getannmixlevel().delayedfunction( impl, signature_args, hop.s_result) else: Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Tue Oct 9 12:26:31 2007 @@ -439,8 +439,12 @@ def invoke_callable_with_pyexceptions(self, fptr, *args): obj = self.llinterpreter.typer.type_system.deref(fptr) + if hasattr(obj, '_fakeimpl'): + f = obj._fakeimpl + else: + f = obj._callable try: - return obj._callable(*args) + return f(*args) except LLException, e: raise except Exception, e: @@ -606,6 +610,8 @@ def perform_call(self, f, ARGS, args): fobj = self.llinterpreter.typer.type_system.deref(f) + if hasattr(fobj, '_fakeimpl'): + return self.invoke_callable_with_pyexceptions(f, *args) has_callable = getattr(fobj, '_callable', None) is not None if has_callable and getattr(fobj._callable, 'suggested_primitive', False): return self.invoke_callable_with_pyexceptions(f, *args) Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llarena.py Tue Oct 9 12:26:31 2007 @@ -169,11 +169,10 @@ # ____________________________________________________________ # -# Public interface: arena_malloc(), arena_free() and arena_reset() -# which directly correspond to lloperations. Although the operations -# are similar to raw_malloc(), raw_free() and raw_memclear(), the -# backend can choose a different implementation for arenas, one that -# is more suited to very large chunks of memory. +# Public interface: arena_malloc(), arena_free(), arena_reset() +# are similar to raw_malloc(), raw_free() and raw_memclear(), but +# work with fakearenaaddresses on which arbitrary arithmetic is +# possible even on top of the llinterpreter. def arena_malloc(nbytes, zero): """Allocate and return a new arena, optionally zero-initialized.""" @@ -201,3 +200,80 @@ this is used to know what type of lltype object to allocate.""" assert isinstance(addr, fakearenaaddress) addr.arena.allocate_object(addr.offset, size) + +# ____________________________________________________________ +# +# Translation support: the functions above turn into the code below. +# We can tweak these implementations to be more suited to very large +# chunks of memory. + +import os, sys +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.extfunc import register_external + +if os.name == 'posix': + READ_MAX = (sys.maxint//4) + 1 # upper bound on reads to avoid surprises + os_read = rffi.llexternal('read', + [rffi.INT, llmemory.Address, rffi.SIZE_T], + rffi.SIZE_T) + + def clear_large_memory_chunk(baseaddr, size): + # on Linux at least, reading from /dev/zero is the fastest way + # to clear arenas, because the kernel knows that it doesn't + # need to even allocate the pages before they are used + try: + fd = os.open('/dev/zero', os.O_RDONLY, 0644) + except OSError: + pass + else: + while size > 0: + count = os_read(fd, baseaddr, min(READ_MAX, size)) + count = rffi.cast(lltype.Signed, count) + if count < 0: + break + size -= count + baseaddr += count + os.close(fd) + + if size > 0: # reading from /dev/zero failed, fallback + llmemory.raw_memclear(baseaddr, size) + +else: + # XXX any better implementation on Windows? + clear_large_memory_chunk = llmemory.raw_memclear + + +def llimpl_arena_malloc(nbytes, zero): + addr = llmemory.raw_malloc(nbytes) + if zero: + clear_large_memory_chunk(addr, nbytes) + return addr +register_external(arena_malloc, [int, bool], llmemory.Address, + 'll_arena.arena_malloc', + llimpl=llimpl_arena_malloc, + llfakeimpl=arena_malloc, + sandboxsafe=True) + +def llimpl_arena_free(arena_addr): + llmemory.raw_free(arena_addr) +register_external(arena_free, [llmemory.Address], None, 'll_arena.arena_free', + llimpl=llimpl_arena_free, + llfakeimpl=arena_free, + sandboxsafe=True) + +def llimpl_arena_reset(arena_addr, myarenasize, zero): + if zero: + clear_large_memory_chunk(arena_addr, myarenasize) +register_external(arena_reset, [llmemory.Address, int, bool], None, + 'll_arena.arena_reset', + llimpl=llimpl_arena_reset, + llfakeimpl=arena_reset, + sandboxsafe=True) + +def llimpl_arena_reserve(addr, size): + pass +register_external(arena_reserve, [llmemory.Address, int], None, + 'll_arena.arena_reserve', + llimpl=llimpl_arena_reserve, + llfakeimpl=arena_reserve, + sandboxsafe=True) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py Tue Oct 9 12:26:31 2007 @@ -2,7 +2,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr from pypy.rpython.lltypesystem.llarena import arena_malloc, arena_reset -from pypy.rpython.lltypesystem.llarena import arena_reserve +from pypy.rpython.lltypesystem.llarena import arena_reserve, arena_free from pypy.rpython.lltypesystem.llarena import ArenaError def test_arena(): @@ -106,13 +106,32 @@ assert lt(a, b+19) +SX = lltype.Struct('S', ('x',lltype.Signed)) + def test_look_inside_object(): - S = lltype.Struct('S', ('x',lltype.Signed)) - SPTR = lltype.Ptr(S) - a = arena_malloc(50, False) + SPTR = lltype.Ptr(SX) + myarenasize = 50 + a = arena_malloc(myarenasize, False) b = a + 4 - arena_reserve(b, llmemory.sizeof(S)) - (b + llmemory.offsetof(S, 'x')).signed[0] = 123 + arena_reserve(b, llmemory.sizeof(SX)) + (b + llmemory.offsetof(SX, 'x')).signed[0] = 123 assert llmemory.cast_adr_to_ptr(b, SPTR).x == 123 llmemory.cast_adr_to_ptr(b, SPTR).x += 1 - assert (b + llmemory.offsetof(S, 'x')).signed[0] == 124 + assert (b + llmemory.offsetof(SX, 'x')).signed[0] == 124 + arena_reset(a, myarenasize, True) + arena_reserve(b, llmemory.sizeof(SX)) + assert llmemory.cast_adr_to_ptr(b, SPTR).x == 0 + arena_free(a) + return 42 + + +def test_llinterpreted(): + from pypy.rpython.test.test_llinterp import interpret + res = interpret(test_look_inside_object, []) + assert res == 42 + +def test_compiled(): + from pypy.translator.c.test.test_genc import compile + fn = compile(test_look_inside_object, []) + res = fn() + assert res == 42 From arigo at codespeak.net Tue Oct 9 12:39:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 12:39:00 +0200 (CEST) Subject: [pypy-svn] r47334 - in pypy/dist/pypy: config rpython/memory rpython/memory/gctransform Message-ID: <20071009103900.3E8FF81AD@code0.codespeak.net> Author: arigo Date: Tue Oct 9 12:38:59 2007 New Revision: 47334 Modified: pypy/dist/pypy/config/translationoption.py pypy/dist/pypy/rpython/memory/gc.py pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: Link the --frameworkgc option to logic that really chooses the gc. Doesn't mean that the SemiSpaceGC translates yet. Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Tue Oct 9 12:38:59 2007 @@ -53,7 +53,7 @@ ("translation.stackless", True)]), ChoiceOption("frameworkgc", "Select one of our custom GCs", ["marksweep", "semispace"], - "marksweep", requires={ + "marksweep", cmdline="--frameworkgc", requires={ "marksweep": [("translation.gc", "framework")], "semispace": [("translation.gc", "framework")], }), Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Tue Oct 9 12:38:59 2007 @@ -1282,3 +1282,18 @@ def size_gc_header(self, typeid=0): XXX +# ____________________________________________________________ + +def choose_gc_from_config(config): + """Return a (GCClass, GC_PARAMS) from the given config object. + """ + config.translation.gc = "framework" + if config.translation.frameworkgc == "marksweep": + GC_PARAMS = {'start_heap_size': 8*1024*1024} # XXX adjust + return MarkSweepGC, GC_PARAMS + elif config.translation.frameworkgc == "semispace": + GC_PARAMS = {'space_size': 32*1024*1024} # XXX fixed at 32MB + return SemiSpaceGC, GC_PARAMS + else: + raise ValueError("unknown value for frameworkgc: %r" % ( + config.translation.frameworkgc,)) Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Tue Oct 9 12:38:59 2007 @@ -31,14 +31,12 @@ extra_static_slots = 0 root_stack_depth = 163840 - from pypy.rpython.memory.gc import MarkSweepGC as GCClass - GC_PARAMS = {'start_heap_size': 8*1024*1024} # XXX adjust - def __init__(self, translator): from pypy.rpython.memory.support import get_address_linked_list + from pypy.rpython.memory.gc import choose_gc_from_config super(FrameworkGCTransformer, self).__init__(translator, inline=True) AddressLinkedList = get_address_linked_list() - GCClass = self.GCClass + GCClass, GC_PARAMS = choose_gc_from_config(translator.config) self.FINALIZERTYPE = lltype.Ptr(ADDRESS_VOID_FUNC) class GCData(object): # types of the GC information tables @@ -113,7 +111,7 @@ sizeofaddr = llmemory.sizeof(llmemory.Address) StackRootIterator = self.build_stack_root_iterator() - gcdata.gc = GCClass(AddressLinkedList, get_roots=StackRootIterator, **self.GC_PARAMS) + gcdata.gc = GCClass(AddressLinkedList, get_roots=StackRootIterator, **GC_PARAMS) def frameworkgc_setup(): # run-time initialization code From cfbolz at codespeak.net Tue Oct 9 12:39:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 9 Oct 2007 12:39:23 +0200 (CEST) Subject: [pypy-svn] r47335 - pypy/dist/pypy/rpython/memory/test Message-ID: <20071009103923.366AD818F@code0.codespeak.net> Author: cfbolz Date: Tue Oct 9 12:39:22 2007 New Revision: 47335 Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py Log: two more passing finalizer tests Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Tue Oct 9 12:39:22 2007 @@ -125,6 +125,64 @@ res = self.interpret(f, [5]) assert res == 6 + def test_finalizer_calls_malloc(self): + class B(object): + pass + b = B() + b.nextid = 0 + b.num_deleted = 0 + class A(object): + def __init__(self): + self.id = b.nextid + b.nextid += 1 + def __del__(self): + b.num_deleted += 1 + C() + class C(A): + def __del__(self): + b.num_deleted += 1 + def f(x): + a = A() + i = 0 + while i < x: + i += 1 + a = A() + llop.gc__collect(lltype.Void) + llop.gc__collect(lltype.Void) + return b.num_deleted + res = self.interpret(f, [5]) + assert res == 12 + + def test_finalizer_resurrects(self): + class B(object): + pass + b = B() + b.nextid = 0 + b.num_deleted = 0 + class A(object): + def __init__(self): + self.id = b.nextid + b.nextid += 1 + def __del__(self): + b.num_deleted += 1 + b.a = self + def f(x): + a = A() + i = 0 + while i < x: + i += 1 + a = A() + llop.gc__collect(lltype.Void) + llop.gc__collect(lltype.Void) + aid = b.a.id + b.a = None + # check that __del__ is not called again + llop.gc__collect(lltype.Void) + llop.gc__collect(lltype.Void) + return b.num_deleted * 10 + aid + 100 * (b.a is None) + res = self.interpret(f, [5]) + assert 160 <= res <= 165 + class TestMarkSweepGC(GCTest): from pypy.rpython.memory.gc import MarkSweepGC as GCClass From arigo at codespeak.net Tue Oct 9 13:16:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 13:16:50 +0200 (CEST) Subject: [pypy-svn] r47336 - pypy/dist/pypy/rpython/memory Message-ID: <20071009111650.E2EFF80C3@code0.codespeak.net> Author: arigo Date: Tue Oct 9 13:16:47 2007 New Revision: 47336 Modified: pypy/dist/pypy/rpython/memory/gc.py Log: Trivial fixes to have the SemiSpaceGC translate. Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Tue Oct 9 13:16:47 2007 @@ -949,25 +949,21 @@ def __init__(self, AddressLinkedList, space_size=4096, get_roots=None): - self.bytes_malloced = 0 self.space_size = space_size - self.tospace = NULL - self.top_of_space = NULL - self.fromspace = NULL - self.free = NULL self.get_roots = get_roots self.gcheaderbuilder = GCHeaderBuilder(self.HDR) self.AddressLinkedList = AddressLinkedList - self.objects_with_finalizers = AddressLinkedList() - self.run_finalizers = AddressLinkedList() def setup(self): + self.bytes_malloced = 0 self.tospace = llarena.arena_malloc(self.space_size, True) debug_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size self.fromspace = llarena.arena_malloc(self.space_size, True) debug_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace + self.objects_with_finalizers = self.AddressLinkedList() + self.run_finalizers = self.AddressLinkedList() def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): From arigo at codespeak.net Tue Oct 9 13:29:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 13:29:22 +0200 (CEST) Subject: [pypy-svn] r47337 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071009112922.EDE9781B0@code0.codespeak.net> Author: arigo Date: Tue Oct 9 13:29:22 2007 New Revision: 47337 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: This change broke various tests. For testing, let the transformer subclass still override the GCClass and GC_PARAMS explicitly. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Tue Oct 9 13:29:22 2007 @@ -36,7 +36,15 @@ from pypy.rpython.memory.gc import choose_gc_from_config super(FrameworkGCTransformer, self).__init__(translator, inline=True) AddressLinkedList = get_address_linked_list() - GCClass, GC_PARAMS = choose_gc_from_config(translator.config) + if hasattr(self, 'GC_PARAMS'): + # for tests: the GC choice can be specified as class attributes + from pypy.rpython.memory.gc import MarkSweepGC + GCClass = getattr(self, 'GCClass', MarkSweepGC) + GC_PARAMS = self.GC_PARAMS + else: + # for regular translation: pick the GC from the config + GCClass, GC_PARAMS = choose_gc_from_config(translator.config) + self.FINALIZERTYPE = lltype.Ptr(ADDRESS_VOID_FUNC) class GCData(object): # types of the GC information tables From arigo at codespeak.net Tue Oct 9 13:32:07 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 13:32:07 +0200 (CEST) Subject: [pypy-svn] r47338 - pypy/dist/pypy/rpython/memory/test Message-ID: <20071009113207.8458A81AF@code0.codespeak.net> Author: arigo Date: Tue Oct 9 13:32:07 2007 New Revision: 47338 Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: Fix the test. Still failing. Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Tue Oct 9 13:32:07 2007 @@ -627,5 +627,5 @@ class gcpolicy(gc.StacklessFrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): from pypy.rpython.memory.gc import SemiSpaceGC as GCClass - GC_PARAMS = {'space_size': llmemory.arena(lltype.Signed, 512)} + GC_PARAMS = {'space_size': 2048} root_stack_depth = 200 From arigo at codespeak.net Tue Oct 9 13:46:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 13:46:45 +0200 (CEST) Subject: [pypy-svn] r47339 - in pypy/dist/pypy: rpython translator/c Message-ID: <20071009114645.44B5E81B6@code0.codespeak.net> Author: arigo Date: Tue Oct 9 13:46:44 2007 New Revision: 47339 Modified: pypy/dist/pypy/rpython/extfunc.py pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/typesystem.py pypy/dist/pypy/translator/c/node.py Log: Attempt a different approach than '_fakeimpl': reuse the existing suggested_primitive logic. We get a suggested_primitive external function with a real graph. The llinterp calls the _callable which is the llfakeimpl from register_external(); the optimizations should leave the call alone because of the suggested_primitive flag; but (with this change) the C backend will simply compile the llimpl graph. Modified: pypy/dist/pypy/rpython/extfunc.py ============================================================================== --- pypy/dist/pypy/rpython/extfunc.py (original) +++ pypy/dist/pypy/rpython/extfunc.py Tue Oct 9 13:46:44 2007 @@ -192,7 +192,11 @@ '_safe_not_sandboxed': self.safe_not_sandboxed, } if hasattr(self, fake_method_name): - impl._llfnobjattrs_['_fakeimpl'] = fakeimpl + # we use the suggested_primitive flag to ask the llinterp + # to call the fakeimpl directly. It also disables inlining + # and other optimizations that would remove the call. + impl._llfnobjattrs_['_callable'] = fakeimpl + fakeimpl.suggested_primitive = True obj = rtyper.getannmixlevel().delayedfunction( impl, signature_args, hop.s_result) else: Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Tue Oct 9 13:46:44 2007 @@ -439,12 +439,8 @@ def invoke_callable_with_pyexceptions(self, fptr, *args): obj = self.llinterpreter.typer.type_system.deref(fptr) - if hasattr(obj, '_fakeimpl'): - f = obj._fakeimpl - else: - f = obj._callable try: - return f(*args) + return obj._callable(*args) except LLException, e: raise except Exception, e: @@ -610,8 +606,6 @@ def perform_call(self, f, ARGS, args): fobj = self.llinterpreter.typer.type_system.deref(f) - if hasattr(fobj, '_fakeimpl'): - return self.invoke_callable_with_pyexceptions(f, *args) has_callable = getattr(fobj, '_callable', None) is not None if has_callable and getattr(fobj._callable, 'suggested_primitive', False): return self.invoke_callable_with_pyexceptions(f, *args) Modified: pypy/dist/pypy/rpython/typesystem.py ============================================================================== --- pypy/dist/pypy/rpython/typesystem.py (original) +++ pypy/dist/pypy/rpython/typesystem.py Tue Oct 9 13:46:44 2007 @@ -69,7 +69,10 @@ name = fnobjattrs.pop('_name', name) else: fnobjattrs = {} - return constr(FT, name, graph = graph, _callable = graph.func, + # _callable is normally graph.func, but can be overridden: + # see fakeimpl in extfunc.py + _callable = fnobjattrs.pop('_callable', graph.func) + return constr(FT, name, graph = graph, _callable = _callable, **fnobjattrs) else: return constr(FT, name, graph = graph) Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Tue Oct 9 13:46:44 2007 @@ -818,9 +818,6 @@ return sandbox_stub(fnobj, db) db.externalfuncs[fnobj._callable] = fnobj return [] - elif getattr(fnobj._callable, 'suggested_primitive', False): - raise ValueError, "trying to compile suggested primitive %r" % ( - fnobj._callable,) elif hasattr(fnobj, 'graph'): if sandbox and sandbox != "if_external": # apply the sandbox transformation From arigo at codespeak.net Tue Oct 9 14:13:36 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 14:13:36 +0200 (CEST) Subject: [pypy-svn] r47340 - pypy/dist/pypy/translator/c/src Message-ID: <20071009121336.9308081B4@code0.codespeak.net> Author: arigo Date: Tue Oct 9 14:13:34 2007 New Revision: 47340 Modified: pypy/dist/pypy/translator/c/src/g_prerequisite.h Log: this must be defined before including other headers in order to get a few extra functions like mremap() Modified: pypy/dist/pypy/translator/c/src/g_prerequisite.h ============================================================================== --- pypy/dist/pypy/translator/c/src/g_prerequisite.h (original) +++ pypy/dist/pypy/translator/c/src/g_prerequisite.h Tue Oct 9 14:13:34 2007 @@ -2,6 +2,9 @@ /**************************************************************/ /*** this is included before any code produced by genc.py ***/ +#define _GNU_SOURCE /* this must be defined before including other headers + in order to get a few extra functions like mremap() */ + /* XXX for now we always include Python.h even to produce stand-alone * executables (which are *not* linked against CPython then), * to get the convenient macro definitions From arigo at codespeak.net Tue Oct 9 14:14:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 14:14:45 +0200 (CEST) Subject: [pypy-svn] r47341 - pypy/dist/pypy/rpython/memory/test Message-ID: <20071009121445.EE92E81B4@code0.codespeak.net> Author: arigo Date: Tue Oct 9 14:14:45 2007 New Revision: 47341 Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: Fix test. Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Tue Oct 9 14:14:45 2007 @@ -83,6 +83,7 @@ class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc import MarkSweepGC as GCClass GC_PARAMS = {'start_heap_size': 4096 } root_stack_depth = 200 gcname = "framework" From cfbolz at codespeak.net Tue Oct 9 14:50:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 9 Oct 2007 14:50:00 +0200 (CEST) Subject: [pypy-svn] r47342 - in pypy/dist/pypy/rpython: . lltypesystem Message-ID: <20071009125000.30A6C8167@code0.codespeak.net> Author: cfbolz Date: Tue Oct 9 14:49:59 2007 New Revision: 47342 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/llheap.py Log: redirect the weakref operations to the heap too Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Tue Oct 9 14:49:59 2007 @@ -716,17 +716,17 @@ op_cast_opaque_ptr.need_result_type = True def op_weakref_create(self, obj): - return llmemory.weakref_create(obj) + return self.heap.weakref_create(obj) def op_weakref_deref(self, PTRTYPE, obj): - return llmemory.weakref_deref(PTRTYPE, obj) + return self.heap.weakref_deref(PTRTYPE, obj) op_weakref_deref.need_result_type = True def op_cast_ptr_to_weakrefptr(self, obj): - return llmemory.cast_ptr_to_weakrefptr(obj) + return self.heap.cast_ptr_to_weakrefptr(obj) def op_cast_weakrefptr_to_ptr(self, PTRTYPE, obj): - return llmemory.cast_weakrefptr_to_ptr(PTRTYPE, obj) + return self.heap.cast_weakrefptr_to_ptr(PTRTYPE, obj) op_cast_weakrefptr_to_ptr.need_result_type = True def op_gc__collect(self): Modified: pypy/dist/pypy/rpython/lltypesystem/llheap.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llheap.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llheap.py Tue Oct 9 14:49:59 2007 @@ -3,7 +3,9 @@ from pypy.rpython.lltypesystem.lltype import pyobjectptr, malloc, free from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free from pypy.rpython.lltypesystem.llmemory import raw_memclear, raw_memcopy -from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage +from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage, \ + weakref_create, weakref_deref, cast_weakrefptr_to_ptr, \ + cast_ptr_to_weakrefptr setfield = setattr from operator import setitem as setarrayitem From arigo at codespeak.net Tue Oct 9 15:02:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 15:02:45 +0200 (CEST) Subject: [pypy-svn] r47345 - in pypy/dist/pypy/rpython: . memory memory/gctransform memory/test Message-ID: <20071009130245.DEDE381B1@code0.codespeak.net> Author: arigo Date: Tue Oct 9 15:02:44 2007 New Revision: 47345 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/memory/gc.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: Revive gc_reload_possibly_moved. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Tue Oct 9 15:02:44 2007 @@ -764,8 +764,14 @@ def op_gc_unprotect(self, obj): raise NotImplementedError("gc_unprotect") - def op_gc_reload_possibly_moved(self, newaddr, ptr): - raise NotImplementedError("gc_reload_possibly_moved") + def op_gc_reload_possibly_moved(self, v_newaddr, v_ptr): + assert v_newaddr.concretetype is llmemory.Address + assert isinstance(v_ptr.concretetype, lltype.Ptr) + assert v_ptr.concretetype.TO._gckind == 'gc' + newaddr = self.getval(v_newaddr) + p = llmemory.cast_adr_to_ptr(newaddr, v_ptr.concretetype) + self.setvar(v_ptr, p) + op_gc_reload_possibly_moved.specialform = True def op_gc_set_max_heap_size(self, maxsize): raise NotImplementedError("gc_set_max_heap_size") Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Tue Oct 9 15:02:44 2007 @@ -19,6 +19,7 @@ class GCBase(object): _alloc_flavor_ = "raw" + moving_gc = False def set_query_functions(self, is_varsize, getfinalizer, offsets_to_gc_pointers, @@ -943,6 +944,7 @@ class SemiSpaceGC(GCBase): _alloc_flavor_ = "raw" + moving_gc = True HDR = lltype.Struct('header', ('forw', llmemory.Address), ('typeid', lltype.Signed)) Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Tue Oct 9 15:02:44 2007 @@ -168,33 +168,26 @@ self.frameworkgc_setup_ptr = getfn(frameworkgc_setup, [], annmodel.s_None) if StackRootIterator.need_root_stack: - self.pop_root_ptr = getfn(StackRootIterator.pop_root, [], - annmodel.s_None, - inline = True) - self.push_root_ptr = getfn(StackRootIterator.push_root, - [annmodel.SomeAddress()], - annmodel.s_None, - inline = True) + #self.pop_root_ptr = getfn(StackRootIterator.pop_root, [], + # annmodel.SomeAddress(), + # inline = True) + #self.push_root_ptr = getfn(StackRootIterator.push_root, + # [annmodel.SomeAddress()], + # annmodel.s_None, + # inline = True) self.incr_stack_ptr = getfn(StackRootIterator.incr_stack, [annmodel.SomeInteger()], annmodel.SomeAddress(), inline = True) self.decr_stack_ptr = getfn(StackRootIterator.decr_stack, [annmodel.SomeInteger()], - annmodel.s_None, - inline = True) - self.save_addr_ptr = getfn(StackRootIterator.save_addr, - [annmodel.SomeAddress(), - annmodel.SomeInteger(), - annmodel.SomeAddress()], - annmodel.s_None, + annmodel.SomeAddress(), inline = True) else: - self.push_root_ptr = None - self.pop_root_ptr = None + #self.push_root_ptr = None + #self.pop_root_ptr = None self.incr_stack_ptr = None self.decr_stack_ptr = None - self.save_addr_ptr = None self.weakref_deref_ptr = self.inittime_helper( ll_weakref_deref, [llmemory.WeakRefPtr], llmemory.Address) @@ -292,12 +285,10 @@ return top incr_stack = staticmethod(incr_stack) - def save_addr(top, k, addr): - top.address[k] = addr - save_addr = staticmethod(save_addr) - def decr_stack(n): - gcdata.root_stack_top -= n*sizeofaddr + top = gcdata.root_stack_top - n*sizeofaddr + gcdata.root_stack_top = top + return top decr_stack = staticmethod(decr_stack) def push_root(addr): @@ -307,7 +298,9 @@ push_root = staticmethod(push_root) def pop_root(): - gcdata.root_stack_top -= sizeofaddr + top = gcdata.root_stack_top - sizeofaddr + gcdata.root_stack_top = top + return top.address[0] pop_root = staticmethod(pop_root) def __init__(self): @@ -532,27 +525,43 @@ pass def push_roots(self, hop): - if self.push_root_ptr is None: + if self.incr_stack_ptr is None: return livevars = [var for var in self.livevars if not var_ispyobj(var)] + if not livevars: + return c_len = rmodel.inputconst(lltype.Signed, len(livevars) ) base_addr = hop.genop("direct_call", [self.incr_stack_ptr, c_len ], resulttype=llmemory.Address) + c_type = rmodel.inputconst(lltype.Void, llmemory.Address) for k,var in enumerate(livevars): c_k = rmodel.inputconst(lltype.Signed, k) v_adr = gen_cast(hop.llops, llmemory.Address, var) - hop.genop("direct_call", [self.save_addr_ptr, base_addr, c_k, v_adr]) + hop.genop("raw_store", [base_addr, c_type, c_k, v_adr]) def pop_roots(self, hop): - if self.pop_root_ptr is None: + if self.decr_stack_ptr is None: return livevars = [var for var in self.livevars if not var_ispyobj(var)] + if not livevars: + return c_len = rmodel.inputconst(lltype.Signed, len(livevars) ) - hop.genop("direct_call", [self.decr_stack_ptr, c_len ] ) -## for var in livevars[::-1]: -## # XXX specific to non-moving collectors -## hop.genop("direct_call", [self.pop_root_ptr]) -## #hop.genop("gc_reload_possibly_moved", [var]) + base_addr = hop.genop("direct_call", [self.decr_stack_ptr, c_len ], + resulttype=llmemory.Address) + if self.gcdata.gc.moving_gc: + # for moving collectors, reload the roots into the local variables + c_type = rmodel.inputconst(lltype.Void, llmemory.Address) + for k,var in enumerate(livevars): + c_k = rmodel.inputconst(lltype.Signed, k) + v_newaddr = hop.genop("raw_load", [base_addr, c_type, c_k], + resulttype=llmemory.Address) + hop.genop("gc_reload_possibly_moved", [v_newaddr, var]) + + def compute_borrowed_vars(self, graph): + # XXX temporary workaround, should be done more correctly + if self.gcdata.gc.moving_gc: + return lambda v: False + return super(FrameworkGCTransformer, self).compute_borrowed_vars(graph) class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder): Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Tue Oct 9 15:02:44 2007 @@ -625,7 +625,7 @@ def setup_class(cls): py.test.skip("in-progress") - class gcpolicy(gc.StacklessFrameworkGcPolicy): + class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): from pypy.rpython.memory.gc import SemiSpaceGC as GCClass GC_PARAMS = {'space_size': 2048} From arigo at codespeak.net Tue Oct 9 15:09:30 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 15:09:30 +0200 (CEST) Subject: [pypy-svn] r47348 - in pypy/dist/pypy/rpython/memory: . test Message-ID: <20071009130930.D6AD881BE@code0.codespeak.net> Author: arigo Date: Tue Oct 9 15:09:28 2007 New Revision: 47348 Modified: pypy/dist/pypy/rpython/memory/gc.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: Make TestSemiSpaceGC runnable. Some of them pass. Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Tue Oct 9 15:09:28 2007 @@ -1152,6 +1152,9 @@ finalizer = self.getfinalizer(hdr.typeid) finalizer(obj) + STATISTICS_NUMBERS = 0 + + class DeferredRefcountingGC(GCBase): _alloc_flavor_ = "raw" Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Tue Oct 9 15:09:28 2007 @@ -89,8 +89,11 @@ gcname = "framework" def heap_usage(self, statistics): - return statistics( - self.gcpolicy.transformerclass.GCClass.STAT_HEAP_USAGE) + if hasattr(self.gcpolicy.transformerclass.GCClass, 'STAT_HEAP_USAGE'): + return statistics( + self.gcpolicy.transformerclass.GCClass.STAT_HEAP_USAGE) + else: + return -1 # xxx def test_llinterp_lists(self): def malloc_a_lot(): From arigo at codespeak.net Tue Oct 9 15:11:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 15:11:11 +0200 (CEST) Subject: [pypy-svn] r47349 - pypy/dist/pypy/translator/c Message-ID: <20071009131111.E411181BF@code0.codespeak.net> Author: arigo Date: Tue Oct 9 15:11:11 2007 New Revision: 47349 Modified: pypy/dist/pypy/translator/c/gc.py Log: Revive gc_reload_possibly_moved in genc too. Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Tue Oct 9 15:11:11 2007 @@ -366,10 +366,7 @@ def OP_GC_RELOAD_POSSIBLY_MOVED(self, funcgen, op): args = [funcgen.expr(v) for v in op.args] - # XXX this more or less assumes mark-and-sweep gc - return '' - # proper return value for moving GCs: - # %s = %s; /* for moving GCs */' % (args[1], args[0]) + return '%s = %s; /* for moving GCs */' % (args[1], args[0]) def common_gcheader_definition(self, defnode): return defnode.db.gctransformer.gc_fields() From fijal at codespeak.net Tue Oct 9 15:48:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Oct 2007 15:48:46 +0200 (CEST) Subject: [pypy-svn] r47350 - in pypy/dist/pypy/rpython/module: . test Message-ID: <20071009134846.4FEF981B4@code0.codespeak.net> Author: fijal Date: Tue Oct 9 15:48:45 2007 New Revision: 47350 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py pypy/dist/pypy/rpython/module/test/test_posix.py Log: whack, whack, until os.fstat works on top of llinterp Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Tue Oct 9 15:48:45 2007 @@ -9,6 +9,7 @@ from pypy.rpython import extregistry from pypy.rpython.extfunc import register_external from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE # XXX on Windows, stat() is flawed; see CPython's posixmodule.c for # an implementation based on the Win32 API @@ -204,6 +205,28 @@ finally: lltype.free(stresult, flavor='raw') + def fakeimpl(arg): + st = getattr(os, name)(arg) + tup = [st[i] for i in range(len(st))] + extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS)) + tup = tup + list(extra_zeroes) + fields = [] + for i in range(len(tup)): + if i in [1, 2, 6]: + fields.append(rffi.LONGLONG) + else: + fields.append(lltype.Signed) + TP = TUPLE_TYPE(fields) + ll_tup = lltype.malloc(TP.TO) + for i in range(len(tup)): + # XXX ARGH! + if i in [1, 6, 2]: + val = rffi.cast(rffi.LONGLONG, tup[i]) + else: + val = tup[i] + setattr(ll_tup, 'item%d' % i, val) + return ll_tup + if arg_is_path: s_arg = str else: @@ -211,10 +234,11 @@ register_external(getattr(os, name), [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), llimpl=func_with_new_name(os_mystat_llimpl, - 'os_%s_llimpl' % (name,))) + 'os_%s_llimpl' % (name,)), + llfakeimpl=func_with_new_name(fakeimpl, + 'os_%s_fake' % (name,))) # ____________________________________________________________ - if 0: XXX - """ disabled for now: Modified: pypy/dist/pypy/rpython/module/test/test_posix.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_posix.py (original) +++ pypy/dist/pypy/rpython/module/test/test_posix.py Tue Oct 9 15:48:45 2007 @@ -24,7 +24,6 @@ assert type(func) == int def test_fstat(self): - import py; py.test.skip("XXX cannot run os.stat() on the llinterp yet") def fo(fi): g = posix.fstat(fi) return g @@ -32,8 +31,7 @@ func = self.interpret(fo,[fi]) stat = os.fstat(fi) for i in range(len(stat)): - stat0 = getattr(func, 'item%d' % i) - assert stat0 == stat[i] + assert getattr(func, 'item%d' % i) == stat[i] def test_times(self): @@ -134,5 +132,6 @@ pass class TestOOtype(BaseTestPosix, OORtypeMixin): - pass + def test_fstat(self): + py.test.skip("not working") From arigo at codespeak.net Tue Oct 9 16:07:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 16:07:42 +0200 (CEST) Subject: [pypy-svn] r47351 - in pypy/dist/pypy: rpython/lltypesystem rpython/lltypesystem/test rpython/memory translator/c translator/c/src Message-ID: <20071009140742.D51D481BD@code0.codespeak.net> Author: arigo Date: Tue Oct 9 16:07:40 2007 New Revision: 47351 Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py pypy/dist/pypy/rpython/memory/lltypelayout.py pypy/dist/pypy/translator/c/primitive.py pypy/dist/pypy/translator/c/src/mem.h Log: Introduce llarena.round_up_for_allocation(size). Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llarena.py Tue Oct 9 16:07:40 2007 @@ -73,6 +73,8 @@ Arena.object_arena_location[addr2.ptr._obj] = self, offset # common case: 'size' starts with a GCHeaderOffset. In this case # we can also remember that the real object starts after the header. + while isinstance(size, RoundedUpForAllocation): + size = size.basesize if (isinstance(size, llmemory.CompositeOffset) and isinstance(size.offsets[0], llmemory.GCHeaderOffset)): objaddr = addr2 + size.offsets[0] @@ -167,6 +169,27 @@ def _cast_to_int(self): return self.arena._getid() + self.offset + +class RoundedUpForAllocation(llmemory.AddressOffset): + """A size that is rounded up in order to preserve alignment of objects + following it. For arenas containing heterogenous objects. + """ + def __init__(self, basesize): + assert isinstance(basesize, llmemory.AddressOffset) + self.basesize = basesize + + def __repr__(self): + return '< RoundedUpForAllocation %r >' % (self.basesize,) + + def ref(self, ptr): + return self.basesize.ref(ptr) + + def _raw_malloc(self, rest, zero): + return self.basesize._raw_malloc(rest, zero=zero) + + def raw_memcopy(self, srcadr, dstadr): + self.basesize.raw_memcopy(srcadr, dstadr) + # ____________________________________________________________ # # Public interface: arena_malloc(), arena_free(), arena_reset() @@ -201,6 +224,11 @@ assert isinstance(addr, fakearenaaddress) addr.arena.allocate_object(addr.offset, size) +def round_up_for_allocation(size): + """Round up the size in order to preserve alignment of objects + following an object. For arenas containing heterogenous objects.""" + return RoundedUpForAllocation(size) + # ____________________________________________________________ # # Translation support: the functions above turn into the code below. @@ -210,12 +238,14 @@ import os, sys from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.extfunc import register_external +from pypy.rlib.objectmodel import debug_assert, CDefinedIntSymbolic if os.name == 'posix': READ_MAX = (sys.maxint//4) + 1 # upper bound on reads to avoid surprises os_read = rffi.llexternal('read', [rffi.INT, llmemory.Address, rffi.SIZE_T], - rffi.SIZE_T) + rffi.SIZE_T, + sandboxsafe=True) def clear_large_memory_chunk(baseaddr, size): # on Linux at least, reading from /dev/zero is the fastest way @@ -277,3 +307,12 @@ llimpl=llimpl_arena_reserve, llfakeimpl=arena_reserve, sandboxsafe=True) + +llimpl_round_up_for_allocation = rffi.llexternal('ROUND_UP_FOR_ALLOCATION', + [rffi.INT], rffi.INT, + sandboxsafe=True) +register_external(round_up_for_allocation, [int], int, + 'll_arena.round_up_for_allocation', + llimpl=llimpl_round_up_for_allocation, + llfakeimpl=round_up_for_allocation, + sandboxsafe=True) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py Tue Oct 9 16:07:40 2007 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr from pypy.rpython.lltypesystem.llarena import arena_malloc, arena_reset from pypy.rpython.lltypesystem.llarena import arena_reserve, arena_free +from pypy.rpython.lltypesystem.llarena import round_up_for_allocation from pypy.rpython.lltypesystem.llarena import ArenaError def test_arena(): @@ -107,19 +108,20 @@ SX = lltype.Struct('S', ('x',lltype.Signed)) +precomputed_size = round_up_for_allocation(llmemory.sizeof(SX)) def test_look_inside_object(): SPTR = lltype.Ptr(SX) myarenasize = 50 a = arena_malloc(myarenasize, False) b = a + 4 - arena_reserve(b, llmemory.sizeof(SX)) + arena_reserve(b, precomputed_size) (b + llmemory.offsetof(SX, 'x')).signed[0] = 123 assert llmemory.cast_adr_to_ptr(b, SPTR).x == 123 llmemory.cast_adr_to_ptr(b, SPTR).x += 1 assert (b + llmemory.offsetof(SX, 'x')).signed[0] == 124 arena_reset(a, myarenasize, True) - arena_reserve(b, llmemory.sizeof(SX)) + arena_reserve(b, round_up_for_allocation(llmemory.sizeof(SX))) assert llmemory.cast_adr_to_ptr(b, SPTR).x == 0 arena_free(a) return 42 Modified: pypy/dist/pypy/rpython/memory/lltypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/lltypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/lltypelayout.py Tue Oct 9 16:07:40 2007 @@ -1,4 +1,4 @@ -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, llarena import struct @@ -106,5 +106,8 @@ return sizeof(offset.gcheaderbuilder.HDR) elif isinstance(offset, llmemory.ArrayLengthOffset): return 0 + elif isinstance(offset, llarena.RoundedUpForAllocation): + basesize = convert_offset_to_int(offset.basesize) + return (basesize + 7) & ~ 7 else: raise Exception("unknown offset type %r"%offset) Modified: pypy/dist/pypy/translator/c/primitive.py ============================================================================== --- pypy/dist/pypy/translator/c/primitive.py (original) +++ pypy/dist/pypy/translator/c/primitive.py Tue Oct 9 16:07:40 2007 @@ -8,6 +8,7 @@ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ CompositeOffset, ArrayLengthOffset, \ GCHeaderOffset +from pypy.rpython.lltypesystem.llarena import RoundedUpForAllocation from pypy.translator.c.support import cdecl, barebonearray # ____________________________________________________________ @@ -48,6 +49,9 @@ return '0' elif type(value) == GCHeaderOffset: return '0' + elif type(value) == RoundedUpForAllocation: + return 'ROUND_UP_FOR_ALLOCATION(%s)' % ( + name_signed(value.basesize, db)) elif isinstance(value, CDefinedIntSymbolic): return str(value.expr) elif isinstance(value, ComputedIntSymbolic): Modified: pypy/dist/pypy/translator/c/src/mem.h ============================================================================== --- pypy/dist/pypy/translator/c/src/mem.h (original) +++ pypy/dist/pypy/translator/c/src/mem.h Tue Oct 9 16:07:40 2007 @@ -2,6 +2,13 @@ /************************************************************/ /*** C header subsection: operations on LowLevelTypes ***/ +/* alignment for arena-based garbage collectors: the following line + enforces an alignment of sizeof(double). */ +#define MEMORY_ALIGNMENT sizeof(double) +#define ROUND_UP_FOR_ALLOCATION(x) \ + (((x) + (MEMORY_ALIGNMENT-1)) & ~(MEMORY_ALIGNMENT-1)) + + #define RAW_MALLOC_ZERO_FILLED 0 #if RAW_MALLOC_ZERO_FILLED From fijal at codespeak.net Tue Oct 9 16:20:07 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Oct 2007 16:20:07 +0200 (CEST) Subject: [pypy-svn] r47352 - in pypy/dist/pypy/rlib: . test Message-ID: <20071009142007.BA2A380BD@code0.codespeak.net> Author: fijal Date: Tue Oct 9 16:20:05 2007 New Revision: 47352 Added: pypy/dist/pypy/rlib/rmmap.py (contents, props changed) pypy/dist/pypy/rlib/test/test_rmmap.py (contents, props changed) Log: Implement rmmap, with slightly non-pythonic interface, but no RPython tricks inside. Added: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/rmmap.py Tue Oct 9 16:20:05 2007 @@ -0,0 +1,664 @@ + +from pypy.rpython.tool import rffi_platform +from pypy.rpython.lltypesystem import rffi, lltype, llmemory + +import sys +import os +import platform +import stat + +_POSIX = os.name == "posix" +_MS_WINDOWS = os.name == "nt" +_LINUX = "linux" in sys.platform +_64BIT = "64bit" in platform.architecture()[0] + + +class CConfig: + _includes_ = ("sys/types.h",'unistd.h') + _header_ = '#define _GNU_SOURCE\n' + size_t = rffi_platform.SimpleType("size_t", rffi.LONG) + off_t = rffi_platform.SimpleType("off_t", rffi.LONG) + +constants = {} +if _POSIX: + CConfig._includes_ += ("sys/mman.h",) + # constants, look in sys/mman.h and platform docs for the meaning + # some constants are linux only so they will be correctly exposed outside + # depending on the OS + constant_names = ['MAP_SHARED', 'MAP_PRIVATE', + 'PROT_READ', 'PROT_WRITE', + 'MS_SYNC'] + opt_constant_names = ['MAP_ANON', 'MAP_ANONYMOUS', + 'PROT_EXEC', + 'MAP_DENYWRITE', 'MAP_EXECUTABLE'] + for name in constant_names: + setattr(CConfig, name, rffi_platform.ConstantInteger(name)) + for name in opt_constant_names: + setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name)) + + CConfig.MREMAP_MAYMOVE = ( + rffi_platform.DefinedConstantInteger("MREMAP_MAYMOVE")) + CConfig.has_mremap = rffi_platform.Has('mremap(NULL, 0, 0, 0)') + # a dirty hack, this is probably a macro + +elif _MS_WINDOWS: + CConfig._includes_ += ("windows.h",) + constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', + 'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY', + 'DUPLICATE_SAME_ACCESS'] + for name in constant_names: + setattr(CConfig, name, rffi_platform.ConstantInteger(name)) + +# export the constants inside and outside. see __init__.py +cConfig = rffi_platform.configure(CConfig) +constants.update(cConfig) + +if _POSIX: + # MAP_ANONYMOUS is not always present but it's always available at CPython level + if constants["MAP_ANONYMOUS"] is None: + constants["MAP_ANONYMOUS"] = constants["MAP_ANON"] + assert constants["MAP_ANONYMOUS"] is not None + constants["MAP_ANON"] = constants["MAP_ANONYMOUS"] + +locals().update(constants) + +_ACCESS_DEFAULT, ACCESS_READ, ACCESS_WRITE, ACCESS_COPY = range(4) + +def external(name, args, result): + return rffi.llexternal(name, args, result, includes=CConfig._includes_) + +PTR = rffi.CCHARP + +has_mremap = cConfig['has_mremap'] + +c_memmove = external('memmove', [PTR, PTR, size_t], lltype.Void) + +if _POSIX: + c_mmap = external('mmap', [PTR, size_t, rffi.INT, rffi.INT, + rffi.INT, off_t], PTR) + c_munmap = external('munmap', [PTR, size_t], rffi.INT) + c_msync = external('msync', [PTR, size_t, rffi.INT], rffi.INT) + if has_mremap: + c_mremap = external('mremap', [PTR, size_t, size_t, rffi.ULONG], PTR) + + _get_page_size = external('getpagesize', [], rffi.INT) + + def _get_error_msg(): + errno = rffi.get_errno() + return os.strerror(errno) +elif _MS_WINDOWS: + XXX + from ctypes import wintypes + + WORD = wintypes.WORD + DWORD = wintypes.DWORD + BOOL = wintypes.BOOL + LONG = wintypes.LONG + LPVOID = PTR + LPCVOID = LPVOID + DWORD_PTR = DWORD + rffi.INT = wintypes.rffi.INT + INVALID_c_int_VALUE = c_int(-1).value + + class SYSINFO_STRUCT(Structure): + _fields_ = [("wProcessorArchitecture", WORD), + ("wReserved", WORD)] + + class SYSINFO_UNION(Union): + _fields_ = [("dwOemId", DWORD), + ("struct", SYSINFO_STRUCT)] + + class SYSTEM_INFO(Structure): + _fields_ = [("union", SYSINFO_UNION), + ("dwPageSize", DWORD), + ("lpMinimumApplicationAddress", LPVOID), + ("lpMaximumApplicationAddress", LPVOID), + ("dwActiveProcessorMask", DWORD_PTR), + ("dwNumberOfProcessors", DWORD), + ("dwProcessorType", DWORD), + ("dwAllocationGranularity", DWORD), + ("wProcessorLevel", WORD), + ("wProcessorRevision", WORD)] + + windll.kernel32.GetSystemInfo.argtypes = [POINTER(SYSTEM_INFO)] + GetFileSize = windll.kernel32.GetFileSize + GetFileSize.argtypes = [rffi.INT, POINTER(rffi.INT)] + GetFileSize.restype = rffi.INT + GetCurrentProcess = windll.kernel32.GetCurrentProcess + GetCurrentProcess.restype = rffi.INT + DuplicateHandle = windll.kernel32.DuplicateHandle + DuplicateHandle.argtypes = [rffi.INT, rffi.INT, rffi.INT, POINTER(rffi.INT), DWORD, + BOOL, DWORD] + DuplicateHandle.restype = BOOL + CreateFileMapping = windll.kernel32.CreateFileMappingA + CreateFileMapping.argtypes = [rffi.INT, PTR, rffi.INT, rffi.INT, rffi.INT, + c_char_p] + CreateFileMapping.restype = rffi.INT + MapViewOfFile = windll.kernel32.MapViewOfFile + MapViewOfFile.argtypes = [rffi.INT, DWORD, DWORD, DWORD, DWORD] + MapViewOfFile.restype = PTR + CloseHandle = windll.kernel32.CloseHandle + CloseHandle.argtypes = [rffi.INT] + CloseHandle.restype = BOOL + UnmapViewOfFile = windll.kernel32.UnmapViewOfFile + UnmapViewOfFile.argtypes = [LPCVOID] + UnmapViewOfFile.restype = BOOL + FlushViewOfFile = windll.kernel32.FlushViewOfFile + FlushViewOfFile.argtypes = [LPCVOID, rffi.INT] + FlushViewOfFile.restype = BOOL + SetFilePointer = windll.kernel32.SetFilePointer + SetFilePointer.argtypes = [rffi.INT, rffi.INT, POINTER(rffi.INT), rffi.INT] + SetEndOfFile = windll.kernel32.SetEndOfFile + SetEndOfFile.argtypes = [rffi.INT] + msvcr71 = cdll.LoadLibrary("msvcr71.dll") + msvcr71._get_osfhandle.argtypes = [rffi.INT] + msvcr71._get_osfhandle.restype = rffi.INT + # libc._lseek.argtypes = [rffi.INT, rffi.INT, rffi.INT] + # libc._lseek.restype = rffi.INT + + + def _get_page_size(): + si = SYSTEM_INFO() + windll.kernel32.GetSystemInfo(byref(si)) + return int(si.dwPageSize) + + def _get_file_size(handle): + # XXX use native Windows types like WORD + high = rffi.INT(0) + low = rffi.INT(windll.kernel32.GetFileSize(rffi.INT(handle.value), byref(high))) + # low might just happen to have the value INVALID_FILE_SIZE + # so we need to check the last error also + INVALID_FILE_SIZE = -1 + NO_ERROR = 0 + dwErr = GetLastError() + if low.value == INVALID_FILE_SIZE and dwErr != NO_ERROR: + raise EnvironmentError(os.strerror(dwErr)) + return low.value, high.value + + def _get_error_msg(): + errno = GetLastError() + return os.strerror(errno) + +PAGESIZE = _get_page_size() +NULL = lltype.nullptr(PTR.TO) +NODATA = lltype.nullptr(PTR.TO) +INVALID_INT_VALUE = -1 + +class MMap(object): + def __init__(self, access): + self.size = 0 + self.pos = 0 + self.access = access + + if _MS_WINDOWS: + self.map_handle = 0 + self.file_handle = 0 + self.tagname = "" + elif _POSIX: + self.fd = -1 + self.closed = False + + def check_valid(self): + if _MS_WINDOWS: + to_close = self.map_handle.value == INVALID_INT_VALUE + elif _POSIX: + to_close = self.closed + + if to_close: + raise ValueError("map closed or invalid") + + def check_writeable(self): + if not (self.access != ACCESS_READ): + raise TypeError("mmap can't modify a readonly memory map.") + + def check_resizeable(self): + if not (self.access == ACCESS_WRITE or self.access == _ACCESS_DEFAULT): + raise TypeError("mmap can't resize a readonly or copy-on-write memory map.") + + def setdata(self, data, size): + """Set the internal data and map size from a PTR.""" + assert size >= 0 + self.data = data + self.size = size + + def close(self): + if _MS_WINDOWS: + if self.size > 0: + self.unmapview() + self.setdata(NODATA, 0) + if self.map_handle.value != INVALID_rffi.INT_VALUE: + CloseHandle(self.map_handle) + self.map_handle.value = INVALID_rffi.INT_VALUE + if self.file_handle.value != INVALID_rffi.INT_VALUE: + CloseHandle(self.file_handle) + self.file_handle.value = INVALID_rffi.INT_VALUE + elif _POSIX: + self.closed = True + if self.fd != -1: + os.close(self.fd) + self.fd = -1 + if self.size > 0: + c_munmap(self.getptr(0), self.size) + self.setdata(NODATA, 0) + + def unmapview(self): + UnmapViewOfFile(self.getptr(0)) + + def read_byte(self): + self.check_valid() + + if self.pos < self.size: + value = self.data[self.pos] + self.pos += 1 + return value + else: + raise ValueError("read byte out of range") + + def readline(self): + self.check_valid() + + data = self.data + for pos in xrange(self.pos, self.size): + if data[pos] == '\n': + eol = pos + 1 # we're interested in the position after new line + break + else: # no '\n' found + eol = self.size + + res = "".join([data[i] for i in range(self.pos, eol)]) + self.pos += len(res) + return res + + def read(self, num=-1): + self.check_valid() + + if num < 0: + # read all + eol = self.size + else: + eol = self.pos + num + # silently adjust out of range requests + if eol > self.size: + eol = self.size + + res = [self.data[i] for i in range(self.pos, eol)] + res = "".join(res) + self.pos += len(res) + return res + + def find(self, tofind, start=0): + self.check_valid() + + # XXX naive! how can we reuse the rstr algorithm? + if start < 0: + start += self.size + if start < 0: + start = 0 + data = self.data + for p in xrange(start, self.size - len(tofind) + 1): + for q in range(len(tofind)): + if data[p+q] != tofind[q]: + break # position 'p' is not a match + else: + # full match + return p + # failure + return -1 + + def seek(self, pos, whence=0): + self.check_valid() + + dist = pos + how = whence + + if how == 0: # relative to start + where = dist + elif how == 1: # relative to current position + where = self.pos + dist + elif how == 2: # relative to the end + where = self.size + dist + else: + raise ValueError("unknown seek type") + + if not (0 <= where <= self.size): + raise ValueError("seek out of range") + + self.pos = where + + def tell(self): + self.check_valid() + return self.pos + + def file_size(self): + self.check_valid() + + size = self.size + if _MS_WINDOWS: + if self.file_handle.value != INVALID_rffi.INT_VALUE: + low, high = _get_file_size(self.file_handle) + if not high and low <= sys.maxint: + return low + size = rffi.INT((high << 32) + low).value + elif _POSIX: + st = os.fstat(self.fd) + size = st[stat.ST_SIZE] + if size > sys.maxint: + size = sys.maxint + else: + size = int(size) + return size + + def write(self, data): + self.check_valid() + self.check_writeable() + + data_len = len(data) + if self.pos + data_len > self.size: + raise ValueError("data out of range") + + internaldata = self.data + start = self.pos + for i in range(data_len): + internaldata[start+i] = data[i] + self.pos = start + data_len + + def write_byte(self, byte): + self.check_valid() + + if len(byte) != 1: + raise TypeError("write_byte() argument must be char") + + self.check_writeable() + self.data[self.pos] = byte[0] + self.pos += 1 + + def getptr(self, offset): + return rffi.ptradd(self.data, offset) + + def flush(self, offset=0, size=0): + self.check_valid() + + if size == 0: + size = self.size + if offset < 0 or size < 0 or offset + size > self.size: + raise ValueError("flush values out of range") + else: + start = self.getptr(offset) + if _MS_WINDOWS: + res = FlushViewOfFile(start, size) + # XXX res == 0 means that an error occurred, but in CPython + # this is not checked + return res + elif _POSIX: +## XXX why is this code here? There is no equivalent in CPython +## if _LINUX: +## # alignment of the address +## value = cast(self.data, c_void_p).value +## aligned_value = value & ~(PAGESIZE - 1) +## # the size should be increased too. otherwise the final +## # part is not "msynced" +## new_size = size + value & (PAGESIZE - 1) + res = c_msync(start, size, MS_SYNC) + if res == -1: + raise EnvironmentError(_get_error_msg()) + + return 0 + + def move(self, dest, src, count): + self.check_valid() + + self.check_writeable() + + # check boundings + if (src < 0 or dest < 0 or count < 0 or + src + count > self.size or dest + count > self.size): + raise ValueError("source or destination out of range") + + datasrc = self.getptr(src) + datadest = self.getptr(dest) + c_memmove(datadest, datasrc, count) + + def resize(self, newsize): + self.check_valid() + + self.check_resizeable() + + if _POSIX: + if not has_mremap: + msg = "mmap: resizing not available -- no mremap()" + raise EnvironmentError(msg) + + # resize the underlying file first + try: + os.ftruncate(self.fd, newsize) + except OSError, e: + raise EnvironmentError(os.strerror(e.errno)) + + # now resize the mmap + newdata = c_mremap(self.getptr(0), self.size, newsize, + MREMAP_MAYMOVE or 0) + self.setdata(newdata, newsize) + elif _MS_WINDOWS: + # disconnect the mapping + self.unmapview() + CloseHandle(self.map_handle) + + # move to the desired EOF position + if _64BIT: + newsize_high = DWORD(newsize >> 32) + newsize_low = DWORD(newsize & 0xFFFFFFFF) + else: + newsize_high = rffi.INT(0) + newsize_low = rffi.INT(newsize) + + FILE_BEGIN = rffi.INT(0) + SetFilePointer(self.file_handle, newsize_low, byref(newsize_high), + FILE_BEGIN) + # resize the file + SetEndOfFile(self.file_handle) + # create another mapping object and remap the file view + res = CreateFileMapping(self.file_handle, NULL, PAGE_READWRITE, + newsize_high, newsize_low, self.tagname) + self.map_handle = rffi.INT(res) + + dwErrCode = DWORD(0) + if self.map_handle: + data = MapViewOfFile(self.map_handle, FILE_MAP_WRITE, + 0, 0, 0) + if data: + self.setdata(data, newsize) + return + else: + dwErrCode = GetLastError() + else: + dwErrCode = GetLastError() + + raise EnvironmentError(os.strerror(dwErrCode)) + + def len(self): + self.check_valid() + + return self.size + + def getitem(self, index): + self.check_valid() + # simplified version, for rpython + if index < 0: + index += self.size + return self.data[index] + + def setitem(self, index, value): + self.check_valid() + self.check_writeable() + + if len(value) != 1: + raise ValueError("mmap assignment must be " + "single-character string") + if index < 0: + index += self.size + self.data[index] = value[0] + +def _check_map_size(size): + if size < 0: + raise TypeError("memory mapped size must be positive") + if rffi.cast(size_t, size) != size: + raise OverflowError("memory mapped size is too large (limited by C int)") + +if _POSIX: + def mmap(fileno, length, flags=MAP_SHARED, + prot=PROT_WRITE | PROT_READ, access=_ACCESS_DEFAULT): + + fd = fileno + + # check size boundaries + _check_map_size(length) + map_size = length + + # check access is not there when flags and prot are there + if access != _ACCESS_DEFAULT and ((flags != MAP_SHARED) or\ + (prot != (PROT_WRITE | PROT_READ))): + raise ValueError("mmap can't specify both access and flags, prot.") + + if access == ACCESS_READ: + flags = MAP_SHARED + prot = PROT_READ + elif access == ACCESS_WRITE: + flags = MAP_SHARED + prot = PROT_READ | PROT_WRITE + elif access == ACCESS_COPY: + flags = MAP_PRIVATE + prot = PROT_READ | PROT_WRITE + elif access == _ACCESS_DEFAULT: + pass + else: + raise ValueError("mmap invalid access parameter.") + + # check file size + try: + st = os.fstat(fd) + except OSError: + pass # ignore errors and trust map_size + else: + mode = st[stat.ST_MODE] + size = st[stat.ST_SIZE] + if size > sys.maxint: + size = sys.maxint + else: + size = int(size) + if stat.S_ISREG(mode): + if map_size == 0: + map_size = size + elif map_size > size: + raise ValueError("mmap length is greater than file size") + + m = MMap(access) + if fd == -1: + # Assume the caller wants to map anonymous memory. + # This is the same behaviour as Windows. mmap.mmap(-1, size) + # on both Windows and Unix map anonymous memory. + m.fd = -1 + + flags |= MAP_ANONYMOUS + + else: + try: + m.fd = os.dup(fd) + except OSError, e: + raise EnvironmentError(os.strerror(e.errno)) + + res = c_mmap(NULL, map_size, prot, flags, fd, 0) + if res == rffi.cast(PTR, -1): + raise EnvironmentError(_get_error_msg()) + + m.setdata(res, map_size) + return m +elif _MS_WINDOWS: + def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT): + # check size boundaries + _check_map_size(length) + map_size = length + + flProtect = 0 + dwDesiredAccess = 0 + fh = 0 + + if access == ACCESS_READ: + flProtect = PAGE_READONLY + dwDesiredAccess = FILE_MAP_READ + elif access == _ACCESS_DEFAULT or access == ACCESS_WRITE: + flProtect = PAGE_READWRITE + dwDesiredAccess = FILE_MAP_WRITE + elif access == ACCESS_COPY: + flProtect = PAGE_WRITECOPY + dwDesiredAccess = FILE_MAP_COPY + else: + raise ValueError("mmap invalid access parameter.") + + # assume -1 and 0 both mean invalid file descriptor + # to 'anonymously' map memory. + if fileno != -1 and fileno != 0: + fh = msvcr71._get_osfhandle(fileno) + if fh == -1: + raise EnvironmentError(_get_error_msg()) + # Win9x appears to need us seeked to zero + # SEEK_SET = 0 + # libc._lseek(fileno, 0, SEEK_SET) + + m = MMap(access) + # XXX the following two attributes should be plain RPython ints + m.file_handle = rffi.INT(INVALID_rffi.INT_VALUE) + m.map_handle = rffi.INT(INVALID_rffi.INT_VALUE) + + if fh: + # it is necessary to duplicate the handle, so the + # Python code can close it on us + res = DuplicateHandle(GetCurrentProcess(), # source process handle + fh, # handle to be duplicated + GetCurrentProcess(), # target process handle + byref(m.file_handle), # result + 0, # access - ignored due to options value + False, # inherited by child procs? + DUPLICATE_SAME_ACCESS) # options + if not res: + raise EnvironmentError(_get_error_msg()) + + if not map_size: + low, high = _get_file_size(rffi.INT(fh)) + if _64BIT: + map_size = rffi.INT((low << 32) + 1).value + else: + if high: + # file is too large to map completely + map_size = -1 + else: + map_size = low + + if tagname: + m.tagname = tagname + + # DWORD is a 4-byte int. If int > 4-byte it must be divided + if _64BIT: + size_hi = DWORD(map_size >> 32) + size_lo = DWORD(map_size & 0xFFFFFFFF) + else: + size_hi = rffi.INT(0) + size_lo = rffi.INT(map_size) + + m.map_handle = rffi.INT(CreateFileMapping(m.file_handle, NULL, flProtect, + size_hi, size_lo, m.tagname)) + + if m.map_handle: + res = MapViewOfFile(m.map_handle, dwDesiredAccess, + 0, 0, 0) + if res: + m.setdata(res, map_size) + return m + else: + dwErr = GetLastError() + else: + dwErr = GetLastError() + + raise EnvironmentError(os.strerror(dwErr)) + + +# register_external here? Added: pypy/dist/pypy/rlib/test/test_rmmap.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/test/test_rmmap.py Tue Oct 9 16:20:05 2007 @@ -0,0 +1,376 @@ +from pypy.tool.udir import udir +import os +from pypy.rpython.test.test_llinterp import interpret +from pypy.rlib import rmmap as mmap +import sys + +class TestMMap: + def setup_class(cls): + cls.tmpname = str(udir.join('mmap-')) + + def test_page_size(self): + def f(): + assert mmap.PAGESIZE > 0 + assert isinstance(mmap.PAGESIZE, int) + + interpret(f, []) + + def test_attributes(self): + def f(): + assert isinstance(mmap.ACCESS_READ, int) + assert isinstance(mmap.ACCESS_WRITE, int) + assert isinstance(mmap.ACCESS_COPY, int) + if os.name == "posix": + assert isinstance(mmap.MAP_ANON, int) + assert isinstance(mmap.MAP_ANONYMOUS, int) + assert isinstance(mmap.MAP_PRIVATE, int) + assert isinstance(mmap.MAP_SHARED, int) + assert isinstance(mmap.PROT_EXEC, int) + assert isinstance(mmap.PROT_READ, int) + assert isinstance(mmap.PROT_WRITE, int) + + interpret(f, []) + + def test_args(self): + from pypy.rlib import rmmap + mmap = rmmap.mmap + if os.name == "posix": + raises(TypeError, mmap, 0, 1, 2, 3, 4, 5) + raises(TypeError, mmap, 0, 1, 2, 3, "foo", 5) + raises(TypeError, mmap, 0, 1, foo="foo") + raises(TypeError, mmap, 0, -1) + raises(OverflowError, mmap, 0, sys.maxint ** 3) + raises(ValueError, mmap, 0, 1, flags=2, access=3) + raises(ValueError, mmap, 0, 1, access=123) + elif os.name == "nt": + raises(TypeError, mmap, 0, 1, 2, 3, 4) + raises(TypeError, mmap, 0, 1, tagname=123) + raises(TypeError, mmap, 0, 1, access="foo") + raises(ValueError, mmap, 0, 1, access=-1) + + def test_file_size(self): + def func(no): + if os.name == "nt": + skip("Only Unix checks file size") + + try: + mmap.mmap(no, 123) + except ValueError: + pass + else: + raise Exception("didn't raise") + + f = open(self.tmpname + "a", "w+") + + f.write("c") + f.flush() + + interpret(func, [f.fileno()]) + f.close() + + def test_create(self): + f = open(self.tmpname + "b", "w+") + + f.write("c") + f.flush() + + def func(no): + m = mmap.mmap(no, 1) + assert m.read(99) == "c" + + interpret(func, [f.fileno()]) + + f.close() + + def test_close(self): + f = open(self.tmpname + "c", "w+") + + f.write("c") + f.flush() + + def func(no): + m = mmap.mmap(no, 1) + m.close() + try: + m.read(1) + except ValueError: + pass + else: + raise Exception("Did not raise") + interpret(func, [f.fileno()]) + + def test_read_byte(self): + f = open(self.tmpname + "d", "w+") + + f.write("c") + f.flush() + + def func(no): + m = mmap.mmap(no, 1) + assert m.read_byte() == "c" + try: + m.read_byte() + except ValueError: + pass + else: + raise Exception("Did not raise") + m.close() + interpret(func, [f.fileno()]) + f.close() + + def test_readline(self): + import os + f = open(self.tmpname + "e", "w+") + + f.write("foo\n") + f.flush() + + def func(no): + m = mmap.mmap(no, 4) + if os.name == "nt": + # windows replaces \n with \r. it's time to change to \n only MS! + assert m.readline() == "foo\r" + elif os.name == "posix": + assert m.readline() == "foo\n" + assert m.readline() == "" + m.close() + + interpret(func, [f.fileno()]) + f.close() + + def test_read(self): + f = open(self.tmpname + "f", "w+") + + f.write("foobar") + f.flush() + + def func(no): + m = mmap.mmap(no, 6) + assert m.read(1) == "f" + assert m.read(6) == "oobar" + assert m.read(1) == "" + m.close() + + interpret(func, [f.fileno()]) + f.close() + + def test_find(self): + f = open(self.tmpname + "g", "w+") + + f.write("foobar\0") + f.flush() + + def func(no): + m = mmap.mmap(no, 7) + assert m.find("b") == 3 + assert m.find("z") == -1 + assert m.find("o", 5) == -1 + assert m.find("ob") == 2 + assert m.find("\0") == 6 + m.close() + + interpret(func, [f.fileno()]) + f.close() + + def test_is_modifiable(self): + f = open(self.tmpname + "h", "w+") + + f.write("foobar") + f.flush() + + def func(no): + m = mmap.mmap(no, 6, access=mmap.ACCESS_READ) + try: + m.write('x') + except TypeError: + pass + else: + assert False + try: + m.resize(7) + except TypeError: + pass + else: + assert False + m.close() + interpret(func, [f.fileno()]) + f.close() + + def test_seek(self): + f = open(self.tmpname + "i", "w+") + + f.write("foobar") + f.flush() + + def func(no): + m = mmap.mmap(no, 6) + m.seek(0) + assert m.tell() == 0 + m.read(1) + m.seek(1, 1) + assert m.tell() == 2 + m.seek(0) + m.seek(-1, 2) + assert m.tell() == 5 + m.close() + interpret(func, [f.fileno()]) + f.close() + + def test_write(self): + f = open(self.tmpname + "j", "w+") + + f.write("foobar") + f.flush() + def func(no): + m = mmap.mmap(no, 6, access=mmap.ACCESS_WRITE) + m.write("ciao\n") + m.seek(0) + assert m.read(6) == "ciao\nr" + m.close() + interpret(func, [f.fileno()]) + f.close() + + def test_write_byte(self): + f = open(self.tmpname + "k", "w+") + + f.write("foobar") + f.flush() + + def func(no): + m = mmap.mmap(no, 6, access=mmap.ACCESS_READ) + m = mmap.mmap(no, 6, access=mmap.ACCESS_WRITE) + m.write_byte("x") + m.seek(0) + assert m.read(6) == "xoobar" + m.close() + interpret(func, [f.fileno()]) + f.close() + + def test_size(self): + f = open(self.tmpname + "l", "w+") + + f.write("foobar") + f.flush() + + def func(no): + m = mmap.mmap(no, 5) + assert m.file_size() == 6 # size of the underline file, not the mmap + m.close() + + interpret(func, [f.fileno()]) + f.close() + + def test_tell(self): + f = open(self.tmpname + "m", "w+") + + f.write("c") + f.flush() + + def func(no): + m = mmap.mmap(no, 1) + assert m.tell() >= 0 + m.close() + + interpret(func, [f.fileno()]) + f.close() + + def test_move(self): + f = open(self.tmpname + "o", "w+") + + f.write("foobar") + f.flush() + + def func(no): + m = mmap.mmap(no, 6, access=mmap.ACCESS_WRITE) + m.move(1, 3, 3) + assert m.read(6) == "fbarar" + m.seek(0) + m.move(1, 3, 2) + a = m.read(6) + assert a == "frarar" + m.close() + + interpret(func, [f.fileno()]) + f.close() + + def test_resize(self): + import sys + if ("darwin" in sys.platform) or ("freebsd" in sys.platform): + skip("resize does not work under OSX or FreeBSD") + + import os + + f = open(self.tmpname + "p", "w+") + f.write("foobar") + f.flush() + + def func(no): + m = mmap.mmap(no, 6, access=mmap.ACCESS_WRITE) + f_size = os.fstat(no).st_size + assert m.file_size() == f_size == 6 + m.resize(10) + f_size = os.fstat(no).st_size + assert m.file_size() == f_size == 10 + m.close() + + interpret(func, [f.fileno()]) + f.close() + + def test_len(self): + + f = open(self.tmpname + "q", "w+") + f.write("foobar") + f.flush() + + def func(no): + m = mmap.mmap(no, 6) + assert m.len() == 6 + m.close() + + interpret(func, [f.fileno()]) + f.close() + + def test_get_item(self): + + f = open(self.tmpname + "r", "w+") + f.write("foobar") + f.flush() + + def func(no): + m = mmap.mmap(no, 6) + assert m.getitem(0) == 'f' + assert m.getitem(-1) == 'r' + # sl = slice(1, 2) + # assert m.get_item(sl) == 'o' + m.close() + + interpret(func, [f.fileno()]) + f.close() + + def test_set_item(self): + f = open(self.tmpname + "s", "w+") + f.write("foobar") + f.flush() + + def func(no): + m = mmap.mmap(no, 6, access=mmap.ACCESS_WRITE) + + # def f(m): m[1:3] = u'xx' + # py.test.raises(IndexError, f, m) + # def f(m): m[1:4] = "zz" + # py.test.raises(IndexError, f, m) + # def f(m): m[1:6] = "z" * 6 + # py.test.raises(IndexError, f, m) + # def f(m): m[:2] = "z" * 5 + # m[1:3] = 'xx' + # assert m.read(6) == "fxxbar" + # m.seek(0) + m.setitem(0, 'x') + assert m.getitem(0) == 'x' + m.setitem(-6, 'y') + data = m.read(6) + assert data == "yoobar" # yxxbar with slice's stuff + m.close() + + interpret(func, [f.fileno()]) + f.close() From arigo at codespeak.net Tue Oct 9 16:40:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 16:40:40 +0200 (CEST) Subject: [pypy-svn] r47353 - in pypy/dist/pypy: rpython/lltypesystem rpython/lltypesystem/test rpython/memory rpython/memory/test translator/c/test Message-ID: <20071009144040.BD7D381C0@code0.codespeak.net> Author: arigo Date: Tue Oct 9 16:40:40 2007 New Revision: 47353 Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py pypy/dist/pypy/rpython/memory/gc.py pypy/dist/pypy/rpython/memory/gctypelayout.py pypy/dist/pypy/rpython/memory/lltypelayout.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py pypy/dist/pypy/translator/c/test/test_boehm.py pypy/dist/pypy/translator/c/test/test_newgc.py Log: Alignment support for the GCs. Fix the SemiSpaceGC to always allocate objects properly aligned. Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llarena.py Tue Oct 9 16:40:40 2007 @@ -216,12 +216,16 @@ assert myarenasize == arena_addr.arena.nbytes arena_addr.arena.reset(zero) -def arena_reserve(addr, size): +def arena_reserve(addr, size, check_alignment=True): """Mark some bytes in an arena as reserved, and returns addr. For debugging this can check that reserved ranges of bytes don't overlap. The size must be symbolic; in non-translated version this is used to know what type of lltype object to allocate.""" + from pypy.rpython.memory.lltypelayout import memory_alignment assert isinstance(addr, fakearenaaddress) + if check_alignment and (addr.offset & (memory_alignment-1)) != 0: + raise ArenaError("object at offset %d would not be correctly aligned" + % (addr.offset,)) addr.arena.allocate_object(addr.offset, size) def round_up_for_allocation(size): Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py Tue Oct 9 16:40:40 2007 @@ -22,7 +22,9 @@ assert s1_ptr2.x == 1 assert s1_ptr1 == s1_ptr2 - arena_reserve(a + ssize + 1, llmemory.sizeof(S)) + py.test.raises(ArenaError, arena_reserve, a + ssize + 1, # misaligned + llmemory.sizeof(S)) + arena_reserve(a + ssize + 1, llmemory.sizeof(S), check_alignment=False) s2_ptr1 = cast_adr_to_ptr(a + ssize + 1, SPTR) py.test.raises(lltype.UninitializedMemoryAccess, 's2_ptr1.x') s2_ptr1.x = 2 @@ -40,10 +42,14 @@ py.test.raises(ArenaError, cast_adr_to_ptr, a+ssize, SPTR) py.test.raises(ArenaError, cast_adr_to_ptr, a+2*ssize, SPTR) py.test.raises(ArenaError, cast_adr_to_ptr, a+2*ssize+1, SPTR) - py.test.raises(ArenaError, arena_reserve, a+1, llmemory.sizeof(S)) - py.test.raises(ArenaError, arena_reserve, a+ssize, llmemory.sizeof(S)) - py.test.raises(ArenaError, arena_reserve, a+2*ssize, llmemory.sizeof(S)) - py.test.raises(ArenaError, arena_reserve, a+2*ssize+1, llmemory.sizeof(S)) + py.test.raises(ArenaError, arena_reserve, a+1, llmemory.sizeof(S), + False) + py.test.raises(ArenaError, arena_reserve, a+ssize, llmemory.sizeof(S), + False) + py.test.raises(ArenaError, arena_reserve, a+2*ssize, llmemory.sizeof(S), + False) + py.test.raises(ArenaError, arena_reserve, a+2*ssize+1, llmemory.sizeof(S), + False) arena_reset(a, myarenasize, True) py.test.raises(ArenaError, cast_adr_to_ptr, a, SPTR) @@ -52,7 +58,7 @@ assert s1_ptr1.x == 0 s1_ptr1.x = 5 - arena_reserve(a + ssize, llmemory.sizeof(S2)) + arena_reserve(a + ssize, llmemory.sizeof(S2), check_alignment=False) s2_ptr1 = cast_adr_to_ptr(a + ssize, S2PTR) assert s2_ptr1.y == '\x00' s2_ptr1.y = 'X' @@ -114,7 +120,7 @@ SPTR = lltype.Ptr(SX) myarenasize = 50 a = arena_malloc(myarenasize, False) - b = a + 4 + b = a + round_up_for_allocation(llmemory.sizeof(lltype.Char)) arena_reserve(b, precomputed_size) (b + llmemory.offsetof(SX, 'x')).signed[0] = 123 assert llmemory.cast_adr_to_ptr(b, SPTR).x == 123 Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Tue Oct 9 16:40:40 2007 @@ -1012,7 +1012,7 @@ llarena.arena_reserve(result, totalsize) self.init_gc_object(result, typeid) (result + size_gc_header + offset_to_length).signed[0] = length - self.free += totalsize + self.free += llarena.round_up_for_allocation(totalsize) if has_finalizer: self.objects_with_finalizers.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) @@ -1129,6 +1129,7 @@ lenaddr = obj + self.varsize_offset_to_length(typeid) length = lenaddr.signed[0] size += length * self.varsize_item_sizes(typeid) + size = llarena.round_up_for_allocation(size) return size def header(self, addr): Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Tue Oct 9 16:40:40 2007 @@ -1,4 +1,4 @@ -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rpython.memory.gctransform.support import find_gc_ptrs_in_type @@ -33,8 +33,13 @@ info["weakptrofs"] = weakpointer_offset(TYPE) if not TYPE._is_varsize(): info["isvarsize"] = False - info["fixedsize"] = llmemory.sizeof(TYPE) + info["fixedsize"] = llarena.round_up_for_allocation( + llmemory.sizeof(TYPE)) info["ofstolength"] = -1 + # note about round_up_for_allocation(): in the 'info' table + # we put a rounded-up size only for fixed-size objects. For + # varsize ones, the GC must anyway compute the size at run-time + # and round up that result. else: info["isvarsize"] = True info["fixedsize"] = llmemory.sizeof(TYPE, 0) Modified: pypy/dist/pypy/rpython/memory/lltypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/lltypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/lltypelayout.py Tue Oct 9 16:40:40 2007 @@ -2,6 +2,8 @@ import struct +memory_alignment = 4 + primitive_to_fmt = {lltype.Signed: "l", lltype.Unsigned: "L", lltype.Char: "c", @@ -108,6 +110,7 @@ return 0 elif isinstance(offset, llarena.RoundedUpForAllocation): basesize = convert_offset_to_int(offset.basesize) - return (basesize + 7) & ~ 7 + mask = memory_alignment - 1 + return (basesize + mask) & ~ mask else: raise Exception("unknown offset type %r"%offset) Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Tue Oct 9 16:40:40 2007 @@ -83,15 +83,17 @@ class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): - from pypy.rpython.memory.gc import MarkSweepGC as GCClass GC_PARAMS = {'start_heap_size': 4096 } root_stack_depth = 200 gcname = "framework" def heap_usage(self, statistics): - if hasattr(self.gcpolicy.transformerclass.GCClass, 'STAT_HEAP_USAGE'): - return statistics( - self.gcpolicy.transformerclass.GCClass.STAT_HEAP_USAGE) + try: + GCClass = self.gcpolicy.transformerclass.GCClass + except AttributeError: + from pypy.rpython.memory.gc import MarkSweepGC as GCClass + if hasattr(GCClass, 'STAT_HEAP_USAGE'): + return statistics(GCClass.STAT_HEAP_USAGE) else: return -1 # xxx Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Tue Oct 9 16:40:40 2007 @@ -27,6 +27,8 @@ config = get_pypy_config(translating=True) config.translation.gc = self.gcpolicy config.translation.stacklessgc = self.stacklessgc + if self.gcpolicy == "framework": + config.translation.frameworkgc = self.frameworkgc config.translation.simplifying = True t = TranslationContext(config=config) self.t = t Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Tue Oct 9 16:40:40 2007 @@ -278,6 +278,7 @@ class TestUsingFramework(AbstractGCTestClass): gcpolicy = "framework" + frameworkgc = "marksweep" def test_empty_collect(self): def f(): @@ -774,6 +775,27 @@ res = fn() assert res == 42 + def test_object_alignment(self): + # all objects returned by the GC should be aligned on a 8-bytes + # boundary, or whatever sizeof(double) is on this platform + mylist = ['a', 'bc', '84139871', 'ajkdh', '876'] + def f(): + result = 0 + buffer = "" + for j in range(100): + for s in mylist: + buffer += s + result |= id(buffer) + return result + + fn = self.getcompiled(f) + res = fn() + import struct + expected_alignment = struct.calcsize("d") + assert (expected_alignment & (expected_alignment-1)) == 0, ( + "sizeof(double) not a power of two") + assert (res & (expected_alignment-1)) == 0 + class TestUsingStacklessFramework(TestUsingFramework): def getcompiled(self, f): @@ -799,3 +821,9 @@ def test_weakref(self): py.test.skip("fails for some reason I couldn't figure out yet :-(") + +class TestSemiSpaceGC(TestUsingFramework): + frameworkgc = "semispace" + + def setup_class(cls): + py.test.skip("in-progress") From arigo at codespeak.net Tue Oct 9 16:55:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 16:55:49 +0200 (CEST) Subject: [pypy-svn] r47354 - pypy/dist/pypy/translator/c/test Message-ID: <20071009145549.67F7E81AF@code0.codespeak.net> Author: arigo Date: Tue Oct 9 16:55:49 2007 New Revision: 47354 Modified: pypy/dist/pypy/translator/c/test/test_newgc.py Log: Fix a couple of test and skip the weakref tests for the SemiSpaceGC. The rest of them passes! Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Tue Oct 9 16:55:49 2007 @@ -3,6 +3,7 @@ import py from py.test import raises +from pypy.objspace.flow.model import summary from pypy.translator.translator import TranslationContext from pypy.translator.backendopt.stat import print_statistics from pypy.translator.c import genc, gc @@ -279,6 +280,7 @@ class TestUsingFramework(AbstractGCTestClass): gcpolicy = "framework" frameworkgc = "marksweep" + should_be_moving = False def test_empty_collect(self): def f(): @@ -305,8 +307,8 @@ fn = self.getcompiled(f) res = fn() assert res == 2 - operations = self.t.graphs[0].startblock.exits[False].target.operations - assert len([op for op in operations if op.opname == "gc_reload_possibly_moved"]) == 0 + insns = summary(self.t.graphs[0]) + assert ('gc_reload_possibly_moved' in insns) == self.should_be_moving def test_framework_safe_pushpop(self): class A(object): @@ -335,10 +337,10 @@ llop.gc__collect(lltype.Void) return global_a.b.a.b.c fn = self.getcompiled(f) - startblock = self.t.graphs[0].startblock res = fn() assert res == 42 - assert len([op for op in startblock.operations if op.opname == "gc_reload_possibly_moved"]) == 0 + insns = summary(self.t.graphs[0]) + assert 'gc_reload_possibly_moved' not in insns def test_framework_protect_getfield(self): class A(object): @@ -824,6 +826,10 @@ class TestSemiSpaceGC(TestUsingFramework): frameworkgc = "semispace" + should_be_moving = True - def setup_class(cls): + def test_weakref(self): + py.test.skip("in-progress") + + def test_prebuilt_weakref(self): py.test.skip("in-progress") From fijal at codespeak.net Tue Oct 9 17:08:11 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Oct 2007 17:08:11 +0200 (CEST) Subject: [pypy-svn] r47355 - in pypy/dist/pypy/rlib: . test Message-ID: <20071009150811.F285581B0@code0.codespeak.net> Author: fijal Date: Tue Oct 9 17:08:10 2007 New Revision: 47355 Modified: pypy/dist/pypy/rlib/rmmap.py pypy/dist/pypy/rlib/test/test_rmmap.py Log: Use our exception instead of default ones Modified: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/rmmap.py (original) +++ pypy/dist/pypy/rlib/rmmap.py Tue Oct 9 17:08:10 2007 @@ -12,6 +12,17 @@ _LINUX = "linux" in sys.platform _64BIT = "64bit" in platform.architecture()[0] +class RValueError(Exception): + def __init__(self, message): + self.message = message + +class REnvironmentError(Exception): + def __init__(self, message): + self.message = message + +class RTypeError(Exception): + def __init__(self, message): + self.message = message class CConfig: _includes_ = ("sys/types.h",'unistd.h') @@ -172,7 +183,7 @@ NO_ERROR = 0 dwErr = GetLastError() if low.value == INVALID_FILE_SIZE and dwErr != NO_ERROR: - raise EnvironmentError(os.strerror(dwErr)) + raise REnvironmentError(os.strerror(dwErr)) return low.value, high.value def _get_error_msg(): @@ -205,15 +216,15 @@ to_close = self.closed if to_close: - raise ValueError("map closed or invalid") + raise RValueError("map closed or invalid") def check_writeable(self): if not (self.access != ACCESS_READ): - raise TypeError("mmap can't modify a readonly memory map.") + raise RTypeError("mmap can't modify a readonly memory map.") def check_resizeable(self): if not (self.access == ACCESS_WRITE or self.access == _ACCESS_DEFAULT): - raise TypeError("mmap can't resize a readonly or copy-on-write memory map.") + raise RTypeError("mmap can't resize a readonly or copy-on-write memory map.") def setdata(self, data, size): """Set the internal data and map size from a PTR.""" @@ -252,7 +263,7 @@ self.pos += 1 return value else: - raise ValueError("read byte out of range") + raise RValueError("read byte out of range") def readline(self): self.check_valid() @@ -318,10 +329,10 @@ elif how == 2: # relative to the end where = self.size + dist else: - raise ValueError("unknown seek type") + raise RValueError("unknown seek type") if not (0 <= where <= self.size): - raise ValueError("seek out of range") + raise RValueError("seek out of range") self.pos = where @@ -354,7 +365,7 @@ data_len = len(data) if self.pos + data_len > self.size: - raise ValueError("data out of range") + raise RValueError("data out of range") internaldata = self.data start = self.pos @@ -366,7 +377,7 @@ self.check_valid() if len(byte) != 1: - raise TypeError("write_byte() argument must be char") + raise RTypeError("write_byte() argument must be char") self.check_writeable() self.data[self.pos] = byte[0] @@ -381,7 +392,7 @@ if size == 0: size = self.size if offset < 0 or size < 0 or offset + size > self.size: - raise ValueError("flush values out of range") + raise RValueError("flush values out of range") else: start = self.getptr(offset) if _MS_WINDOWS: @@ -400,7 +411,7 @@ ## new_size = size + value & (PAGESIZE - 1) res = c_msync(start, size, MS_SYNC) if res == -1: - raise EnvironmentError(_get_error_msg()) + raise REnvironmentError(_get_error_msg()) return 0 @@ -412,7 +423,7 @@ # check boundings if (src < 0 or dest < 0 or count < 0 or src + count > self.size or dest + count > self.size): - raise ValueError("source or destination out of range") + raise RValueError("source or destination out of range") datasrc = self.getptr(src) datadest = self.getptr(dest) @@ -426,13 +437,13 @@ if _POSIX: if not has_mremap: msg = "mmap: resizing not available -- no mremap()" - raise EnvironmentError(msg) + raise REnvironmentError(msg) # resize the underlying file first try: os.ftruncate(self.fd, newsize) except OSError, e: - raise EnvironmentError(os.strerror(e.errno)) + raise REnvironmentError(os.strerror(e.errno)) # now resize the mmap newdata = c_mremap(self.getptr(0), self.size, newsize, @@ -473,7 +484,7 @@ else: dwErrCode = GetLastError() - raise EnvironmentError(os.strerror(dwErrCode)) + raise REnvironmentError(os.strerror(dwErrCode)) def len(self): self.check_valid() @@ -492,7 +503,7 @@ self.check_writeable() if len(value) != 1: - raise ValueError("mmap assignment must be " + raise RValueError("mmap assignment must be " "single-character string") if index < 0: index += self.size @@ -500,7 +511,7 @@ def _check_map_size(size): if size < 0: - raise TypeError("memory mapped size must be positive") + raise RTypeError("memory mapped size must be positive") if rffi.cast(size_t, size) != size: raise OverflowError("memory mapped size is too large (limited by C int)") @@ -517,7 +528,7 @@ # check access is not there when flags and prot are there if access != _ACCESS_DEFAULT and ((flags != MAP_SHARED) or\ (prot != (PROT_WRITE | PROT_READ))): - raise ValueError("mmap can't specify both access and flags, prot.") + raise RValueError("mmap can't specify both access and flags, prot.") if access == ACCESS_READ: flags = MAP_SHARED @@ -531,7 +542,7 @@ elif access == _ACCESS_DEFAULT: pass else: - raise ValueError("mmap invalid access parameter.") + raise RValueError("mmap invalid access parameter.") # check file size try: @@ -549,7 +560,7 @@ if map_size == 0: map_size = size elif map_size > size: - raise ValueError("mmap length is greater than file size") + raise RValueError("mmap length is greater than file size") m = MMap(access) if fd == -1: @@ -564,11 +575,11 @@ try: m.fd = os.dup(fd) except OSError, e: - raise EnvironmentError(os.strerror(e.errno)) + raise REnvironmentError(os.strerror(e.errno)) res = c_mmap(NULL, map_size, prot, flags, fd, 0) if res == rffi.cast(PTR, -1): - raise EnvironmentError(_get_error_msg()) + raise REnvironmentError(_get_error_msg()) m.setdata(res, map_size) return m @@ -592,14 +603,14 @@ flProtect = PAGE_WRITECOPY dwDesiredAccess = FILE_MAP_COPY else: - raise ValueError("mmap invalid access parameter.") + raise RValueError("mmap invalid access parameter.") # assume -1 and 0 both mean invalid file descriptor # to 'anonymously' map memory. if fileno != -1 and fileno != 0: fh = msvcr71._get_osfhandle(fileno) if fh == -1: - raise EnvironmentError(_get_error_msg()) + raise REnvironmentError(_get_error_msg()) # Win9x appears to need us seeked to zero # SEEK_SET = 0 # libc._lseek(fileno, 0, SEEK_SET) @@ -620,7 +631,7 @@ False, # inherited by child procs? DUPLICATE_SAME_ACCESS) # options if not res: - raise EnvironmentError(_get_error_msg()) + raise REnvironmentError(_get_error_msg()) if not map_size: low, high = _get_file_size(rffi.INT(fh)) @@ -658,7 +669,7 @@ else: dwErr = GetLastError() - raise EnvironmentError(os.strerror(dwErr)) + raise REnvironmentError(os.strerror(dwErr)) # register_external here? Modified: pypy/dist/pypy/rlib/test/test_rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rmmap.py (original) +++ pypy/dist/pypy/rlib/test/test_rmmap.py Tue Oct 9 17:08:10 2007 @@ -2,6 +2,7 @@ import os from pypy.rpython.test.test_llinterp import interpret from pypy.rlib import rmmap as mmap +from pypy.rlib.rmmap import RTypeError, RValueError, REnvironmentError import sys class TestMMap: @@ -30,23 +31,6 @@ assert isinstance(mmap.PROT_WRITE, int) interpret(f, []) - - def test_args(self): - from pypy.rlib import rmmap - mmap = rmmap.mmap - if os.name == "posix": - raises(TypeError, mmap, 0, 1, 2, 3, 4, 5) - raises(TypeError, mmap, 0, 1, 2, 3, "foo", 5) - raises(TypeError, mmap, 0, 1, foo="foo") - raises(TypeError, mmap, 0, -1) - raises(OverflowError, mmap, 0, sys.maxint ** 3) - raises(ValueError, mmap, 0, 1, flags=2, access=3) - raises(ValueError, mmap, 0, 1, access=123) - elif os.name == "nt": - raises(TypeError, mmap, 0, 1, 2, 3, 4) - raises(TypeError, mmap, 0, 1, tagname=123) - raises(TypeError, mmap, 0, 1, access="foo") - raises(ValueError, mmap, 0, 1, access=-1) def test_file_size(self): def func(no): @@ -55,7 +39,7 @@ try: mmap.mmap(no, 123) - except ValueError: + except RValueError: pass else: raise Exception("didn't raise") @@ -93,7 +77,7 @@ m.close() try: m.read(1) - except ValueError: + except RValueError: pass else: raise Exception("Did not raise") @@ -110,7 +94,7 @@ assert m.read_byte() == "c" try: m.read_byte() - except ValueError: + except RValueError: pass else: raise Exception("Did not raise") @@ -182,13 +166,13 @@ m = mmap.mmap(no, 6, access=mmap.ACCESS_READ) try: m.write('x') - except TypeError: + except RTypeError: pass else: assert False try: m.resize(7) - except TypeError: + except RTypeError: pass else: assert False From fijal at codespeak.net Tue Oct 9 17:10:21 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Oct 2007 17:10:21 +0200 (CEST) Subject: [pypy-svn] r47356 - in pypy/dist/pypy/module/mmap: . test Message-ID: <20071009151021.35D1D80C9@code0.codespeak.net> Author: fijal Date: Tue Oct 9 17:10:20 2007 New Revision: 47356 Modified: pypy/dist/pypy/module/mmap/__init__.py pypy/dist/pypy/module/mmap/interp_mmap.py pypy/dist/pypy/module/mmap/test/test_mmap.py Log: Move mmap module to use new interface. Windows version is broken because of that (should be relatively trivial to fix) Modified: pypy/dist/pypy/module/mmap/__init__.py ============================================================================== --- pypy/dist/pypy/module/mmap/__init__.py (original) +++ pypy/dist/pypy/module/mmap/__init__.py Tue Oct 9 17:10:20 2007 @@ -1,4 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule +from pypy.rlib import rmmap class Module(MixedModule): interpleveldefs = { @@ -15,7 +16,7 @@ def buildloaders(cls): from pypy.module.mmap import interp_mmap - for constant, value in interp_mmap.constants.iteritems(): + for constant, value in rmmap.constants.iteritems(): if isinstance(value, int): Module.interpleveldefs[constant] = "space.wrap(%r)" % value Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Tue Oct 9 17:10:20 2007 @@ -4,529 +4,142 @@ from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app +from pypy.rlib import rmmap +from pypy.rlib.rmmap import RValueError, RTypeError, REnvironmentError import sys import os import platform import stat -_POSIX = os.name == "posix" -_MS_WINDOWS = os.name == "nt" -_LINUX = "linux" in sys.platform -_64BIT = "64bit" in platform.architecture()[0] - -class CConfig: - _includes_ = ("sys/types.h",'unistd.h') - _header_ = '#define _GNU_SOURCE\n' - size_t = rffi_platform.SimpleType("size_t", rffi.LONG) - off_t = rffi_platform.SimpleType("off_t", rffi.LONG) - -constants = {} -if _POSIX: - CConfig._includes_ += ("sys/mman.h",) - # constants, look in sys/mman.h and platform docs for the meaning - # some constants are linux only so they will be correctly exposed outside - # depending on the OS - constant_names = ['MAP_SHARED', 'MAP_PRIVATE', - 'PROT_READ', 'PROT_WRITE', - 'MS_SYNC'] - opt_constant_names = ['MAP_ANON', 'MAP_ANONYMOUS', - 'PROT_EXEC', - 'MAP_DENYWRITE', 'MAP_EXECUTABLE'] - for name in constant_names: - setattr(CConfig, name, rffi_platform.ConstantInteger(name)) - for name in opt_constant_names: - setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name)) - - CConfig.MREMAP_MAYMOVE = ( - rffi_platform.DefinedConstantInteger("MREMAP_MAYMOVE")) - CConfig.has_mremap = rffi_platform.Has('mremap(NULL, 0, 0, 0)') - # a dirty hack, this is probably a macro - -elif _MS_WINDOWS: - CConfig._includes_ += ("windows.h",) - constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', - 'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY', - 'DUPLICATE_SAME_ACCESS'] - for name in constant_names: - setattr(CConfig, name, rffi_platform.ConstantInteger(name)) - -# export the constants inside and outside. see __init__.py -cConfig = rffi_platform.configure(CConfig) -constants.update(cConfig) - -if _POSIX: - # MAP_ANONYMOUS is not always present but it's always available at CPython level - if constants["MAP_ANONYMOUS"] is None: - constants["MAP_ANONYMOUS"] = constants["MAP_ANON"] - assert constants["MAP_ANONYMOUS"] is not None - constants["MAP_ANON"] = constants["MAP_ANONYMOUS"] - -locals().update(constants) - -_ACCESS_DEFAULT, ACCESS_READ, ACCESS_WRITE, ACCESS_COPY = range(4) - -def external(name, args, result): - return rffi.llexternal(name, args, result, includes=CConfig._includes_) - -PTR = rffi.CCHARP - -has_mremap = cConfig['has_mremap'] - -c_memmove = external('memmove', [PTR, PTR, size_t], lltype.Void) - -if _POSIX: - c_mmap = external('mmap', [PTR, size_t, rffi.INT, rffi.INT, - rffi.INT, off_t], PTR) - c_munmap = external('munmap', [PTR, size_t], rffi.INT) - c_msync = external('msync', [PTR, size_t, rffi.INT], rffi.INT) - if has_mremap: - c_mremap = external('mremap', [PTR, size_t, size_t, rffi.ULONG], PTR) - - _get_page_size = external('getpagesize', [], rffi.INT) - - def _get_error_msg(): - errno = rffi.get_errno() - return os.strerror(errno) -elif _MS_WINDOWS: - XXX - from ctypes import wintypes - - WORD = wintypes.WORD - DWORD = wintypes.DWORD - BOOL = wintypes.BOOL - LONG = wintypes.LONG - LPVOID = PTR - LPCVOID = LPVOID - DWORD_PTR = DWORD - rffi.INT = wintypes.rffi.INT - INVALID_c_int_VALUE = c_int(-1).value - - class SYSINFO_STRUCT(Structure): - _fields_ = [("wProcessorArchitecture", WORD), - ("wReserved", WORD)] - - class SYSINFO_UNION(Union): - _fields_ = [("dwOemId", DWORD), - ("struct", SYSINFO_STRUCT)] - - class SYSTEM_INFO(Structure): - _fields_ = [("union", SYSINFO_UNION), - ("dwPageSize", DWORD), - ("lpMinimumApplicationAddress", LPVOID), - ("lpMaximumApplicationAddress", LPVOID), - ("dwActiveProcessorMask", DWORD_PTR), - ("dwNumberOfProcessors", DWORD), - ("dwProcessorType", DWORD), - ("dwAllocationGranularity", DWORD), - ("wProcessorLevel", WORD), - ("wProcessorRevision", WORD)] - - windll.kernel32.GetSystemInfo.argtypes = [POINTER(SYSTEM_INFO)] - GetFileSize = windll.kernel32.GetFileSize - GetFileSize.argtypes = [rffi.INT, POINTER(rffi.INT)] - GetFileSize.restype = rffi.INT - GetCurrentProcess = windll.kernel32.GetCurrentProcess - GetCurrentProcess.restype = rffi.INT - DuplicateHandle = windll.kernel32.DuplicateHandle - DuplicateHandle.argtypes = [rffi.INT, rffi.INT, rffi.INT, POINTER(rffi.INT), DWORD, - BOOL, DWORD] - DuplicateHandle.restype = BOOL - CreateFileMapping = windll.kernel32.CreateFileMappingA - CreateFileMapping.argtypes = [rffi.INT, PTR, rffi.INT, rffi.INT, rffi.INT, - c_char_p] - CreateFileMapping.restype = rffi.INT - MapViewOfFile = windll.kernel32.MapViewOfFile - MapViewOfFile.argtypes = [rffi.INT, DWORD, DWORD, DWORD, DWORD] - MapViewOfFile.restype = PTR - CloseHandle = windll.kernel32.CloseHandle - CloseHandle.argtypes = [rffi.INT] - CloseHandle.restype = BOOL - UnmapViewOfFile = windll.kernel32.UnmapViewOfFile - UnmapViewOfFile.argtypes = [LPCVOID] - UnmapViewOfFile.restype = BOOL - FlushViewOfFile = windll.kernel32.FlushViewOfFile - FlushViewOfFile.argtypes = [LPCVOID, rffi.INT] - FlushViewOfFile.restype = BOOL - SetFilePointer = windll.kernel32.SetFilePointer - SetFilePointer.argtypes = [rffi.INT, rffi.INT, POINTER(rffi.INT), rffi.INT] - SetEndOfFile = windll.kernel32.SetEndOfFile - SetEndOfFile.argtypes = [rffi.INT] - msvcr71 = cdll.LoadLibrary("msvcr71.dll") - msvcr71._get_osfhandle.argtypes = [rffi.INT] - msvcr71._get_osfhandle.restype = rffi.INT - # libc._lseek.argtypes = [rffi.INT, rffi.INT, rffi.INT] - # libc._lseek.restype = rffi.INT - - - def _get_page_size(): - si = SYSTEM_INFO() - windll.kernel32.GetSystemInfo(byref(si)) - return int(si.dwPageSize) - - def _get_file_size(space, handle): - # XXX use native Windows types like WORD - high = rffi.INT(0) - low = rffi.INT(windll.kernel32.GetFileSize(rffi.INT(handle.value), byref(high))) - # low might just happen to have the value INVALID_FILE_SIZE - # so we need to check the last error also - INVALID_FILE_SIZE = -1 - NO_ERROR = 0 - dwErr = GetLastError() - if low.value == INVALID_FILE_SIZE and dwErr != NO_ERROR: - raise OperationError(space.w_EnvironmentError, - space.wrap(os.strerror(dwErr))) - return low.value, high.value - - def _get_error_msg(): - errno = GetLastError() - return os.strerror(errno) - -PAGESIZE = _get_page_size() -NULL = lltype.nullptr(PTR.TO) -NODATA = lltype.nullptr(PTR.TO) -INVALID_INT_VALUE = -1 - -# ____________________________________________________________ - -# XXX the methods should take unsigned int arguments instead of int - class W_MMap(Wrappable): - def __init__(self, space, access): + def __init__(self, space, mmap_obj): self.space = space - self.size = 0 - self.pos = 0 - self.access = access - - if _MS_WINDOWS: - self.map_handle = 0 - self.file_handle = 0 - self.tagname = "" - elif _POSIX: - self.fd = -1 - self.closed = False - - def check_valid(self): - if _MS_WINDOWS: - to_close = self.map_handle.value == INVALID_INT_VALUE - elif _POSIX: - to_close = self.closed - - if to_close: - raise OperationError(self.space.w_ValueError, - self.space.wrap("map closed or invalid")) - - def check_writeable(self): - if not (self.access != ACCESS_READ): - raise OperationError(self.space.w_TypeError, - self.space.wrap("mmap can't modify a readonly memory map.")) - - def check_resizeable(self): - if not (self.access == ACCESS_WRITE or self.access == _ACCESS_DEFAULT): - raise OperationError(self.space.w_TypeError, - self.space.wrap( - "mmap can't resize a readonly or copy-on-write memory map.")) - - def setdata(self, data, size): - """Set the internal data and map size from a PTR.""" - assert size >= 0 - self.data = data - self.size = size - + self.mmap = mmap_obj + def close(self): - if _MS_WINDOWS: - if self.size > 0: - self.unmapview() - self.setdata(NODATA, 0) - if self.map_handle.value != INVALID_rffi.INT_VALUE: - CloseHandle(self.map_handle) - self.map_handle.value = INVALID_rffi.INT_VALUE - if self.file_handle.value != INVALID_rffi.INT_VALUE: - CloseHandle(self.file_handle) - self.file_handle.value = INVALID_rffi.INT_VALUE - elif _POSIX: - self.closed = True - if self.fd != -1: - os.close(self.fd) - self.fd = -1 - if self.size > 0: - c_munmap(self.getptr(0), self.size) - self.setdata(NODATA, 0) + self.mmap.close() close.unwrap_spec = ['self'] - - def unmapview(self): - UnmapViewOfFile(self.getptr(0)) - - def read_byte(self): - self.check_valid() - if self.pos < self.size: - value = self.data[self.pos] - self.pos += 1 - return self.space.wrap(value) - else: + def read_byte(self): + try: + return self.space.wrap(self.mmap.read_byte()) + except RValueError, v: raise OperationError(self.space.w_ValueError, - self.space.wrap("read byte out of range")) + self.space.wrap(v.message)) read_byte.unwrap_spec = ['self'] def readline(self): - self.check_valid() - - data = self.data - for pos in xrange(self.pos, self.size): - if data[pos] == '\n': - eol = pos + 1 # we're interested in the position after new line - break - else: # no '\n' found - eol = self.size - - res = "".join([data[i] for i in range(self.pos, eol)]) - self.pos += len(res) - return self.space.wrap(res) + return self.space.wrap(self.mmap.readline()) readline.unwrap_spec = ['self'] def read(self, num=-1): self.check_valid() - - if num < 0: - # read all - eol = self.size - else: - eol = self.pos + num - # silently adjust out of range requests - if eol > self.size: - eol = self.size - - res = [self.data[i] for i in range(self.pos, eol)] - res = "".join(res) - self.pos += len(res) - return self.space.wrap(res) + return self.space.wrap(self.mmap.read(num)) read.unwrap_spec = ['self', int] def find(self, tofind, start=0): - self.check_valid() - - # XXX naive! how can we reuse the rstr algorithm? - if start < 0: - start += self.size - if start < 0: - start = 0 - data = self.data - for p in xrange(start, self.size - len(tofind) + 1): - for q in range(len(tofind)): - if data[p+q] != tofind[q]: - break # position 'p' is not a match - else: - # full match - return self.space.wrap(p) - # failure - return self.space.wrap(-1) + return self.space.wrap(self.mmap.find(tofind, start)) find.unwrap_spec = ['self', str, int] def seek(self, pos, whence=0): - self.check_valid() - - dist = pos - how = whence - - if how == 0: # relative to start - where = dist - elif how == 1: # relative to current position - where = self.pos + dist - elif how == 2: # relative to the end - where = self.size + dist - else: - raise OperationError(self.space.w_ValueError, - self.space.wrap("unknown seek type")) - - if not (0 <= where <= self.size): + try: + self.mmap.seek(pos, whence) + except RValueError, v: raise OperationError(self.space.w_ValueError, - self.space.wrap("seek out of range")) - - self.pos = where + self.space.wrap(v.message)) seek.unwrap_spec = ['self', 'index', int] def tell(self): - self.check_valid() - - return self.space.wrap(self.pos) + return self.space.wrap(self.mmap.tell()) tell.unwrap_spec = ['self'] def descr_size(self): - self.check_valid() - - size = self.size - if _MS_WINDOWS: - if self.file_handle.value != INVALID_rffi.INT_VALUE: - low, high = _get_file_size(self.space, self.file_handle) - if not high and low <= sys.maxint: - return self.space.wrap(low) - size = rffi.INT((high << 32) + low).value - elif _POSIX: - st = os.fstat(self.fd) - size = st[stat.ST_SIZE] - if size > sys.maxint: - size = sys.maxint - else: - size = int(size) - return self.space.wrap(size) + return self.space.wrap(self.mmap.file_size()) descr_size.unwrap_spec = ['self'] def write(self, data): - self.check_valid() self.check_writeable() - - data_len = len(data) - if self.pos + data_len > self.size: + try: + self.mmap.write(data) + except RValueError, v: raise OperationError(self.space.w_ValueError, - self.space.wrap("data out of range")) - - internaldata = self.data - start = self.pos - for i in range(data_len): - internaldata[start+i] = data[i] - self.pos = start + data_len + self.space.wrap(v.message)) write.unwrap_spec = ['self', str] def write_byte(self, byte): - self.check_valid() - - if len(byte) != 1: + try: + self.mmap.write_byte(byte) + except RValueError, v: + raise OperationError(self.space.w_ValueError, + self.space.wrap(v.message)) + except RTypeError, v: raise OperationError(self.space.w_TypeError, - self.space.wrap("write_byte() argument must be char")) - - self.check_writeable() - self.data[self.pos] = byte[0] - self.pos += 1 + self.space.wrap(v.message)) write_byte.unwrap_spec = ['self', str] - def getptr(self, offset): - return rffi.ptradd(self.data, offset) - def flush(self, offset=0, size=0): - self.check_valid() - - if size == 0: - size = self.size - if offset < 0 or size < 0 or offset + size > self.size: + try: + return self.space.wrap(self.mmap.flush(offset, size)) + except RValueError, v: raise OperationError(self.space.w_ValueError, - self.space.wrap("flush values out of range")) - else: - start = self.getptr(offset) - if _MS_WINDOWS: - res = FlushViewOfFile(start, size) - # XXX res == 0 means that an error occurred, but in CPython - # this is not checked - return self.space.wrap(res) - elif _POSIX: -## XXX why is this code here? There is no equivalent in CPython -## if _LINUX: -## # alignment of the address -## value = cast(self.data, c_void_p).value -## aligned_value = value & ~(PAGESIZE - 1) -## # the size should be increased too. otherwise the final -## # part is not "msynced" -## new_size = size + value & (PAGESIZE - 1) - res = c_msync(start, size, MS_SYNC) - if res == -1: - raise OperationError(self.space.w_EnvironmentError, - self.space.wrap(_get_error_msg())) - - return self.space.wrap(0) + self.space.wrap(v.message)) flush.unwrap_spec = ['self', int, int] def move(self, dest, src, count): - self.check_valid() - - self.check_writeable() - - # check boundings - if (src < 0 or dest < 0 or count < 0 or - src + count > self.size or dest + count > self.size): + try: + self.mmap.move(dest, src, count) + except RValueError, v: raise OperationError(self.space.w_ValueError, - self.space.wrap("source or destination out of range")) - - datasrc = self.getptr(src) - datadest = self.getptr(dest) - c_memmove(datadest, datasrc, count) + self.space.wrap(v.message)) move.unwrap_spec = ['self', int, int, int] def resize(self, newsize): self.check_valid() - self.check_resizeable() - - if _POSIX: - if not has_mremap: - msg = "mmap: resizing not available -- no mremap()" - raise OperationError(self.space.w_EnvironmentError, - self.space.wrap(msg)) - - # resize the underlying file first - try: - os.ftruncate(self.fd, newsize) - except OSError, e: - raise OperationError(self.space.w_EnvironmentError, - self.space.wrap(os.strerror(e.errno))) - - # now resize the mmap - newdata = c_mremap(self.getptr(0), self.size, newsize, - MREMAP_MAYMOVE or 0) - self.setdata(newdata, newsize) - elif _MS_WINDOWS: - # disconnect the mapping - self.unmapview() - CloseHandle(self.map_handle) - - # move to the desired EOF position - if _64BIT: - newsize_high = DWORD(newsize >> 32) - newsize_low = DWORD(newsize & 0xFFFFFFFF) - else: - newsize_high = rffi.INT(0) - newsize_low = rffi.INT(newsize) - - FILE_BEGIN = rffi.INT(0) - SetFilePointer(self.file_handle, newsize_low, byref(newsize_high), - FILE_BEGIN) - # resize the file - SetEndOfFile(self.file_handle) - # create another mapping object and remap the file view - res = CreateFileMapping(self.file_handle, NULL, PAGE_READWRITE, - newsize_high, newsize_low, self.tagname) - self.map_handle = rffi.INT(res) - - dwErrCode = DWORD(0) - if self.map_handle: - data = MapViewOfFile(self.map_handle, FILE_MAP_WRITE, - 0, 0, 0) - if data: - self.setdata(data, newsize) - return - else: - dwErrCode = GetLastError() - else: - dwErrCode = GetLastError() - - raise OperationError(self.space.w_EnvironmentError, - self.space.wrap(os.strerror(dwErrCode))) + self.mmap.resize(newsize) resize.unwrap_spec = ['self', int] def __len__(self): - self.check_valid() - - return self.space.wrap(self.size) + return self.space.wrap(self.mmap.size) __len__.unwrap_spec = ['self'] - + + def check_valid(self): + try: + self.mmap.check_valid() + except RValueError, v: + raise OperationError(self.space.w_ValueError, + self.space.wrap(v.message)) + + def check_writeable(self): + try: + self.mmap.check_writeable() + except RValueError, v: + raise OperationError(self.space.w_ValueError, + self.space.wrap(v.message)) + except RTypeError, v: + raise OperationError(self.space.w_TypeError, + self.space.wrap(v.message)) + + def check_resizeable(self): + try: + self.mmap.check_resizeable() + except RValueError, v: + raise OperationError(self.space.w_ValueError, + self.space.wrap(v.message)) + except RTypeError, v: + raise OperationError(self.space.w_TypeError, + self.space.wrap(v.message)) + def descr_getitem(self, w_index): self.check_valid() space = self.space - start, stop, step = space.decode_index(w_index, self.size) + start, stop, step = space.decode_index(w_index, self.mmap.size) if step == 0: # index only - return space.wrap(self.data[start]) + return space.wrap(self.mmap.getitem(start)) elif step == 1: if 0 <= start <= stop: - res = "".join([self.data[i] for i in range(start, stop)]) + res = "".join([self.mmap.getitem(i) for i in range(start, stop)]) else: res = '' return space.wrap(res) @@ -541,13 +154,13 @@ self.check_writeable() space = self.space - start, stop, step = space.decode_index(w_index, self.size) + start, stop, step = space.decode_index(w_index, self.mmap.size) if step == 0: # index only if len(value) != 1: raise OperationError(space.w_ValueError, space.wrap("mmap assignment must be " "single-character string")) - self.data[start] = value[0] + self.mmap.setitem(start, value) elif step == 1: length = stop - start if start < 0 or length < 0: @@ -556,13 +169,12 @@ raise OperationError(space.w_ValueError, space.wrap("mmap slice assignment is wrong size")) for i in range(length): - self.data[start + i] = value[i] + self.mmap.setitem(start + i, value[i]) else: raise OperationError(space.w_ValueError, space.wrap("mmap object does not support slicing with a step")) descr_setitem.unwrap_spec = ['self', W_Root, str] - W_MMap.typedef = TypeDef("mmap", close = interp2app(W_MMap.close), read_byte = interp2app(W_MMap.read_byte), @@ -592,170 +204,24 @@ raise OperationError(space.w_OverflowError, space.wrap("memory mapped size is too large (limited by C int)")) -if _POSIX: - def mmap(space, fileno, length, flags=MAP_SHARED, - prot=PROT_WRITE | PROT_READ, access=_ACCESS_DEFAULT): - - fd = fileno - - # check size boundaries - _check_map_size(space, length) - map_size = length - - # check access is not there when flags and prot are there - if access != _ACCESS_DEFAULT and ((flags != MAP_SHARED) or\ - (prot != (PROT_WRITE | PROT_READ))): - raise OperationError(space.w_ValueError, - space.wrap("mmap can't specify both access and flags, prot.")) - - if access == ACCESS_READ: - flags = MAP_SHARED - prot = PROT_READ - elif access == ACCESS_WRITE: - flags = MAP_SHARED - prot = PROT_READ | PROT_WRITE - elif access == ACCESS_COPY: - flags = MAP_PRIVATE - prot = PROT_READ | PROT_WRITE - elif access == _ACCESS_DEFAULT: - pass - else: - raise OperationError(space.w_ValueError, - space.wrap("mmap invalid access parameter.")) +if rmmap._POSIX: + def mmap(space, fileno, length, flags=rmmap.MAP_SHARED, + prot=rmmap.PROT_WRITE | rmmap.PROT_READ, access=rmmap._ACCESS_DEFAULT): - # check file size try: - st = os.fstat(fd) - except OSError: - pass # ignore errors and trust map_size - else: - mode = st[stat.ST_MODE] - size = st[stat.ST_SIZE] - if size > sys.maxint: - size = sys.maxint - else: - size = int(size) - if stat.S_ISREG(mode): - if map_size == 0: - map_size = size - elif map_size > size: - raise OperationError(space.w_ValueError, - space.wrap("mmap length is greater than file size")) - - m = W_MMap(space, access) - if fd == -1: - # Assume the caller wants to map anonymous memory. - # This is the same behaviour as Windows. mmap.mmap(-1, size) - # on both Windows and Unix map anonymous memory. - m.fd = -1 - - flags |= MAP_ANONYMOUS - - else: - try: - m.fd = os.dup(fd) - except OSError, e: - raise OperationError(space.w_EnvironmentError, - space.wrap(os.strerror(e.errno))) - - res = c_mmap(NULL, map_size, prot, flags, fd, 0) - if res == rffi.cast(PTR, -1): - raise OperationError(space.w_EnvironmentError, - space.wrap(_get_error_msg())) - - m.setdata(res, map_size) - - return space.wrap(m) + return space.wrap(W_MMap(space, rmmap.mmap(fileno, length, + flags, prot, access))) + except REnvironmentError, e: + raise OperationError(space.w_EnvironmentError, space.wrap(e.message)) + except RValueError, e: + raise OperationError(space.w_ValueError, space.wrap(e.message)) + except RTypeError, e: + raise OperationError(space.w_TypeError, space.wrap(e.message)) mmap.unwrap_spec = [ObjSpace, int, 'index', int, int, int] -elif _MS_WINDOWS: +elif rmmap._MS_WINDOWS: def mmap(space, fileno, length, tagname="", access=_ACCESS_DEFAULT): - # check size boundaries - _check_map_size(space, length) - map_size = length - - flProtect = 0 - dwDesiredAccess = 0 - fh = 0 - - if access == ACCESS_READ: - flProtect = PAGE_READONLY - dwDesiredAccess = FILE_MAP_READ - elif access == _ACCESS_DEFAULT or access == ACCESS_WRITE: - flProtect = PAGE_READWRITE - dwDesiredAccess = FILE_MAP_WRITE - elif access == ACCESS_COPY: - flProtect = PAGE_WRITECOPY - dwDesiredAccess = FILE_MAP_COPY - else: - raise OperationError(space.w_ValueError, - space.wrap("mmap invalid access parameter.")) - - # assume -1 and 0 both mean invalid file descriptor - # to 'anonymously' map memory. - if fileno != -1 and fileno != 0: - fh = msvcr71._get_osfhandle(fileno) - if fh == -1: - raise OperationError(space.w_EnvironmentError, - space.wrap(_get_error_msg())) - # Win9x appears to need us seeked to zero - # SEEK_SET = 0 - # libc._lseek(fileno, 0, SEEK_SET) - - m = W_MMap(space, access) - # XXX the following two attributes should be plain RPython ints - m.file_handle = rffi.INT(INVALID_rffi.INT_VALUE) - m.map_handle = rffi.INT(INVALID_rffi.INT_VALUE) - - if fh: - # it is necessary to duplicate the handle, so the - # Python code can close it on us - res = DuplicateHandle(GetCurrentProcess(), # source process handle - fh, # handle to be duplicated - GetCurrentProcess(), # target process handle - byref(m.file_handle), # result - 0, # access - ignored due to options value - False, # inherited by child procs? - DUPLICATE_SAME_ACCESS) # options - if not res: - raise OperationError(space.w_EnvironmentError, - space.wrap(_get_error_msg())) - - if not map_size: - low, high = _get_file_size(space, rffi.INT(fh)) - if _64BIT: - map_size = rffi.INT((low << 32) + 1).value - else: - if high: - # file is too large to map completely - map_size = -1 - else: - map_size = low - - if tagname: - m.tagname = tagname - - # DWORD is a 4-byte int. If int > 4-byte it must be divided - if _64BIT: - size_hi = DWORD(map_size >> 32) - size_lo = DWORD(map_size & 0xFFFFFFFF) - else: - size_hi = rffi.INT(0) - size_lo = rffi.INT(map_size) - - m.map_handle = rffi.INT(CreateFileMapping(m.file_handle, NULL, flProtect, - size_hi, size_lo, m.tagname)) - - if m.map_handle: - res = MapViewOfFile(m.map_handle, dwDesiredAccess, - 0, 0, 0) - if res: - m.setdata(res, map_size) - return space.wrap(m) - else: - dwErr = GetLastError() - else: - dwErr = GetLastError() - - raise OperationError(space.w_EnvironmentError, - space.wrap(os.strerror(dwErr))) + XXX mmap.unwrap_spec = [ObjSpace, int, 'index', str, int] + +constants = rmmap.constants +PAGESIZE = rmmap.PAGESIZE Modified: pypy/dist/pypy/module/mmap/test/test_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/test/test_mmap.py (original) +++ pypy/dist/pypy/module/mmap/test/test_mmap.py Tue Oct 9 17:10:20 2007 @@ -611,3 +611,4 @@ assert m.read(10) == "ABCDEABCDE" m.close() f.close() + From cfbolz at codespeak.net Tue Oct 9 18:31:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 9 Oct 2007 18:31:53 +0200 (CEST) Subject: [pypy-svn] r47357 - in pypy/dist/pypy/rpython: . lltypesystem memory memory/test Message-ID: <20071009163153.E809781C2@code0.codespeak.net> Author: cfbolz Date: Tue Oct 9 18:31:52 2007 New Revision: 47357 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/llheap.py pypy/dist/pypy/rpython/memory/gc.py pypy/dist/pypy/rpython/memory/gcwrapper.py pypy/dist/pypy/rpython/memory/test/test_gc.py Log: attempt to implement weakrefs with the semispace GC. does not work completely yet, but some tests are passing. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Tue Oct 9 18:31:52 2007 @@ -723,10 +723,10 @@ op_weakref_deref.need_result_type = True def op_cast_ptr_to_weakrefptr(self, obj): - return self.heap.cast_ptr_to_weakrefptr(obj) + return llmemory.cast_ptr_to_weakrefptr(obj) def op_cast_weakrefptr_to_ptr(self, PTRTYPE, obj): - return self.heap.cast_weakrefptr_to_ptr(PTRTYPE, obj) + return llmemory.cast_weakrefptr_to_ptr(PTRTYPE, obj) op_cast_weakrefptr_to_ptr.need_result_type = True def op_gc__collect(self): @@ -1236,12 +1236,26 @@ if index != 0: raise IndexError("address of local vars only support [0] indexing") p = self.frame.getval(self.v) - return llmemory.cast_ptr_to_adr(p) + result = llmemory.cast_ptr_to_adr(p) + # the GC should never see instances of _gctransformed_wref + result = self.unwrap_possible_weakref(result) + return result def __setitem__(self, index, newvalue): if index != 0: raise IndexError("address of local vars only support [0] indexing") - p = llmemory.cast_adr_to_ptr(newvalue, self.v.concretetype) + if self.v.concretetype == llmemory.WeakRefPtr: + # fish some more + assert isinstance(newvalue, llmemory.fakeaddress) + p = llmemory.cast_ptr_to_weakrefptr(newvalue.ptr) + else: + p = llmemory.cast_adr_to_ptr(newvalue, self.v.concretetype) self.frame.setvar(self.v, p) + def unwrap_possible_weakref(self, addr): + # fish fish fish + if isinstance(addr.ptr._obj, llmemory._gctransformed_wref): + return llmemory.fakeaddress(addr.ptr._obj._ptr) + return addr + # by default we route all logging messages to nothingness # e.g. tests can then switch on logging to get more help Modified: pypy/dist/pypy/rpython/lltypesystem/llheap.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llheap.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llheap.py Tue Oct 9 18:31:52 2007 @@ -4,8 +4,7 @@ from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free from pypy.rpython.lltypesystem.llmemory import raw_memclear, raw_memcopy from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage, \ - weakref_create, weakref_deref, cast_weakrefptr_to_ptr, \ - cast_ptr_to_weakrefptr + weakref_create, weakref_deref setfield = setattr from operator import setitem as setarrayitem Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Tue Oct 9 18:31:52 2007 @@ -56,7 +56,14 @@ """ size = self.fixed_size(typeid) needs_finalizer = bool(self.getfinalizer(typeid)) - contains_weakptr = self.weakpointer_offset(typeid) != -1 + weakptr_offset = self.weakpointer_offset(typeid) + #XXX cannot compare weakptr_offset with -1 + #contains_weakptr = weakpointer_offset. != -1 + if isinstance(weakptr_offset, int): + assert weakptr_offset == -1 + contains_weakptr = False + else: + contains_weakptr = True assert not (needs_finalizer and contains_weakptr) if self.is_varsize(typeid): assert not contains_weakptr @@ -966,11 +973,10 @@ self.free = self.tospace self.objects_with_finalizers = self.AddressLinkedList() self.run_finalizers = self.AddressLinkedList() + self.objects_with_weakrefs = self.AddressLinkedList() def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): - if contains_weakptr: - raise NotImplementedError("weakptr in SemiSpaceGC") size_gc_header = self.gcheaderbuilder.size_gc_header totalsize = size_gc_header + size if raw_malloc_usage(totalsize) > self.top_of_space - self.free: @@ -987,12 +993,12 @@ self.free += totalsize if has_finalizer: self.objects_with_finalizers.append(result + size_gc_header) + if contains_weakptr: + self.objects_with_weakrefs.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, can_collect, has_finalizer=False): - if has_finalizer: - raise NotImplementedError("finalizers in SemiSpaceGC") size_gc_header = self.gcheaderbuilder.size_gc_header nonvarsize = size_gc_header + size try: @@ -1037,6 +1043,25 @@ root.address[0] = self.copy(root.address[0]) free_non_gc_object(roots) scan = self.scan_copied(scan) + # walk over list of objects that contain weakrefs + # if the object it references survives and invalidate it otherwise + new_with_weakref = self.AddressLinkedList() + while self.objects_with_weakrefs.non_empty(): + obj = self.objects_with_weakrefs.pop() + if not self.is_forwarded(obj): + continue # weakref itself dies + obj = self.get_forwarding_address(obj) + offset = self.weakpointer_offset(self.header(obj).typeid) + pointing_to = (obj + offset).address[0] + if pointing_to: + if self.is_forwarded(pointing_to): + (obj + offset).address[0] = self.get_forwarding_address( + pointing_to) + else: + (obj + offset).address[0] = NULL + new_with_weakref.append(obj) + self.objects_with_weakrefs.delete() + self.objects_with_weakrefs = new_with_weakref # walk over list of objects with finalizers # if it is not copied, add it to the list of to-be-called finalizers # and copy it, to me make the finalizer runnable Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Tue Oct 9 18:31:52 2007 @@ -67,6 +67,17 @@ def collect(self): self.gc.collect() + def weakref_create(self, obj): + type_id = self.get_type_id(gctypelayout.WEAKREF) + addr = self.gc.malloc(type_id, None, zero=False) + result = llmemory.cast_adr_to_ptr(addr, gctypelayout.WEAKREFPTR) + result.weakptr = llmemory.cast_ptr_to_adr(obj) + return llmemory.cast_ptr_to_weakrefptr(result) + + def weakref_deref(self, PTRTYPE, obj): + addr = gctypelayout.ll_weakref_deref(obj) + return llmemory.cast_adr_to_ptr(addr, PTRTYPE) + # ____________________________________________________________ @@ -89,11 +100,15 @@ assert not type_contains_pyobjs(TYPE), "not implemented" def ll_finalizer(addr): + old_active_frame = self.llinterp.active_frame try: v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) self.llinterp.eval_graph(destrgraph, [v]) except llinterp.LLException: - print "a destructor raised an exception, ignoring it" + raise RuntimeError( + "a finalizer raised an exception, shouldn't happen") + finally: + self.llinterp.active_frame = old_active_frame return ll_finalizer Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Tue Oct 9 18:31:52 2007 @@ -183,6 +183,46 @@ res = self.interpret(f, [5]) assert 160 <= res <= 165 + def test_weakref(self): + import weakref, gc + class A(object): + pass + def g(): + a = A() + return weakref.ref(a) + def f(): + a = A() + ref = weakref.ref(a) + result = ref() is a + ref = g() + llop.gc__collect(lltype.Void) + result = result and (ref() is None) + # check that a further collection is fine + llop.gc__collect(lltype.Void) + result = result and (ref() is None) + return result + res = self.interpret(f, []) + assert res + + def test_weakref_to_object_with_finalizer(self): + import weakref, gc + class A(object): + count = 0 + a = A() + class B(object): + def __del__(self): + a.count += 1 + def g(): + b = B() + return weakref.ref(b) + def f(): + ref = g() + llop.gc__collect(lltype.Void) + llop.gc__collect(lltype.Void) + result = a.count == 1 and (ref() is None) + return result + res = self.interpret(f, []) + assert res class TestMarkSweepGC(GCTest): from pypy.rpython.memory.gc import MarkSweepGC as GCClass From fijal at codespeak.net Tue Oct 9 18:50:00 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Oct 2007 18:50:00 +0200 (CEST) Subject: [pypy-svn] r47358 - in pypy/dist/pypy/rlib: . test Message-ID: <20071009165000.5B76681C3@code0.codespeak.net> Author: fijal Date: Tue Oct 9 18:50:00 2007 New Revision: 47358 Modified: pypy/dist/pypy/rlib/libffi.py pypy/dist/pypy/rlib/test/test_libffi.py Log: Intermediate checkin, test is failing for mysterious reason Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Tue Oct 9 18:50:00 2007 @@ -73,7 +73,6 @@ rffi.ULONG : ffi_type_ulong, rffi.LONG : ffi_type_slong, lltype.Void : ffi_type_void, - # some shortcuts } def external(name, args, result): @@ -129,60 +128,77 @@ # XXX rffi.cast here... return res -def new_funcptr(argtypes, restype): - argnum = len(argtypes) +def cast_type_to_ffitype(tp): + """ This function returns ffi representation of rpython type tp + """ + return TYPE_MAP[tp] +cast_type_to_ffitype._annspecialcase_ = 'specialize:memo' - argtypes_iterable = unrolling_iterable(enumerate(argtypes)) +def push_arg_as_ffiptr(ffitp, TP, arg, ll_buf): + # this is for primitive types. For structures and arrays + # would be something different (more dynamic) + TP_P = rffi.CArray(TP) + rffi.cast(TP_P, ll_buf)[0] = arg +push_arg_as_ffiptr._annspecialcase_ = 'specialize:argtype(1)' + +class FuncPtr(object): + def __init__(self, name, argtypes, restype, funcsym): + self.name = name + self.argtypes = argtypes + self.restype = restype + self.funcsym = funcsym + argnum = len(argtypes) + self.ready_args = [0] * argnum + TP = rffi.CArray(rffi.VOIDP) + self.ll_args = lltype.malloc(TP, argnum, flavor='raw') + self.ll_cif = lltype.malloc(FFI_CIFP.TO, flavor='raw') + self.ll_argtypes = lltype.malloc(FFI_TYPE_PP.TO, argnum, flavor='raw') + for i in range(argnum): + self.ll_argtypes[i] = argtypes[i] + res = c_ffi_prep_cif(self.ll_cif, FFI_DEFAULT_ABI, + rffi.cast(rffi.UINT, argnum), restype, + self.ll_argtypes) + if not res == FFI_OK: + raise OSError(-1, "Wrong typedef") + for i in range(argnum): + # space for each argument + self.ll_args[i] = lltype.malloc(rffi.VOIDP.TO, argtypes[i].c_size, + flavor='raw') + self.ll_result = lltype.malloc(rffi.VOIDP.TO, restype.c_size, + flavor='raw') + + def push_arg(self, num, TP, value): + push_arg_as_ffiptr(self.argtypes[i], TP, value, self.ll_args[i]) + self.ready_args[num] = 1 + + def _check_args(self): + for num in range(len(self.ready_args)): + if not self.ready_args[num]: + raise TypeError("Did not specify arg nr %d" % num) + + def _clean_args(self): + for num in range(len(self.ready_args)): + self.ready_args[num] = 0 + + def call(self, RES_TP): + self._check_args() + c_ffi_call(self.ll_cif, self.func_sym, + rffi.cast(rffi.VOIDP, self.restype), + rffi.cast(VOIDPP, self.ll_args)) + if self.restype != ffi_type_void: + res = rffi.cast(lltype.Ptr(CArray(RES_TP)), self.ll_result)[0] + self._clean_args() + return res + call._annspecialcase_ = 'specialize:argtype(1)' - class FuncPtr: - def __init__(self, func_sym): - TP = rffi.CFixedArray(FFI_TYPE_P, argnum) - self.ll_argtypes = lltype.malloc(TP, flavor='raw') - self.argtypes = argtypes - for i, argtype in argtypes_iterable: - self.ll_argtypes[i] = TYPE_MAP[argtype] - TP = rffi.CFixedArray(rffi.VOIDP, argnum) - self.ll_args = lltype.malloc(TP, flavor='raw') - for i, argtype in argtypes_iterable: - # XXX - TP = rffi.CFixedArray(argtypes[i], 1) - self.ll_args[i] = rffi.cast(rffi.VOIDP, - lltype.malloc(TP, flavor='raw')) - self.restype = restype - if restype is not None: - TP = rffi.CFixedArray(restype, 1) - self.ll_res = lltype.malloc(TP, flavor='raw') - if not func_sym: - raise OSError(-1, "NULL func_sym") - self.func_sym = func_sym - self.ll_cif = lltype.malloc(FFI_CIFP.TO, flavor='raw') - res = c_ffi_prep_cif(self.ll_cif, FFI_DEFAULT_ABI, - rffi.cast(rffi.UINT, argnum), - TYPE_MAP[restype], - rffi.cast(FFI_TYPE_PP, self.ll_argtypes)) - if not res == FFI_OK: - raise OSError(-1, "Wrong typedef") - - def call(self, args): - # allocated result should be padded and stuff - PTR_T = lltype.Ptr(rffi.CFixedArray(rffi.INT, 1)) - for i, argtype in argtypes_iterable: - TP = lltype.Ptr(rffi.CFixedArray(argtype, 1)) - addr = rffi.cast(TP, self.ll_args[i]) - addr[0] = args[i] - c_ffi_call(self.ll_cif, self.func_sym, - rffi.cast(rffi.VOIDP, self.ll_res), - rffi.cast(VOIDPP, self.ll_args)) - return self.ll_res[0] - - def __del__(self): - lltype.free(self.ll_argtypes, flavor='raw') - lltype.free(self.ll_args, flavor='raw') - lltype.free(self.ll_cif, flavor='raw') - if self.restype is not None: - lltype.free(self.ll_res, flavor='raw') - return FuncPtr -new_funcptr._annspecialcase_ = 'specialize:memo' + def __del__(self): + argnum = len(self.argtypes) + for i in range(argnum): + lltype.free(self.ll_args[i], flavor='raw') + lltype.free(self.ll_args, flavor='raw') + lltype.free(self.ll_result, flavor='raw') + lltype.free(self.ll_cif, flavor='raw') + lltype.free(self.ll_argtypes, flavor='raw') class CDLL: def __init__(self, libname): @@ -192,6 +208,7 @@ c_dlclose(self.lib) def getpointer(self, name, argtypes, restype): - funcptr = new_funcptr(argtypes, restype) - return funcptr(dlsym(self.lib, name)) - getpointer._annspecialcase_ = 'specialize:arg(2, 3)' + # these arguments are already casted to proper ffi + # structures! + return FuncPtr(name, argtypes, restype, dlsym(self.lib, name)) + Modified: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_libffi.py (original) +++ pypy/dist/pypy/rlib/test/test_libffi.py Tue Oct 9 18:50:00 2007 @@ -3,7 +3,7 @@ """ from pypy.rpython.test.test_llinterp import interpret -from pypy.rlib.libffi import CDLL, dlopen +from pypy.rlib.libffi import * from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED from pypy.rpython.lltypesystem import rffi, lltype import os, sys @@ -34,13 +34,13 @@ def test_library_get_func(self): lib = self.get_libc() - ptr = lib.getpointer('time', [], lltype.Void) - py.test.raises(KeyError, lib.getpointer, 'xxxxxxxxxxxxxxx', [], lltype.Void) + ptr = lib.getpointer('time', [], ffi_type_void) + py.test.raises(KeyError, lib.getpointer, 'xxxxxxxxxxxxxxx', [], ffi_type_void) del lib def test_library_func_call(self): lib = self.get_libc() - ptr = lib.getpointer('rand', [], rffi.INT) + ptr = lib.getpointer('rand', [], ffi_type_sint) zeroes = 0 for i in range(100): res = ptr.call([]) @@ -51,15 +51,20 @@ def test_call_args(self): libm = CDLL('libm.so') - pow = libm.getpointer('pow', [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE) - assert pow.call((2.0, 2.0)) == 4.0 - assert pow.call((3.0, 3.0)) == 27.0 + pow = libm.getpointer('pow', [ffi_type_double, ffi_type_double], + ffi_type_double) + pow.push_arg(0, rffi.DOUBLE, 2.0) + pow.push_arg(1, rffi.DOUBLE, 2.0) + assert pow.call() == 4.0 + pow.push_arg(0, rffi.DOUBLE, 3.0) + pow.push_arg(1, rffi.DOUBLE, 3.0) + assert pow.call() == 27.0 def test_compile(self): py.test.skip("in-progress") def f(x, y): libm = CDLL('libm.so') - c_pow = libm.getpointer('pow', (rffi.DOUBLE, rffi.DOUBLE), rffi.DOUBLE) + c_pow = libm.getpointer('pow', (ffi_type_double, ffi_type_double), ffi_type_double) return c_pow.call((x, y)) interpret(f, [2.0, 4.0]) From cfbolz at codespeak.net Tue Oct 9 19:14:14 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 9 Oct 2007 19:14:14 +0200 (CEST) Subject: [pypy-svn] r47359 - in pypy/dist/pypy/rpython: lltypesystem memory memory/test Message-ID: <20071009171414.812E481B9@code0.codespeak.net> Author: cfbolz Date: Tue Oct 9 19:14:12 2007 New Revision: 47359 Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/lltypesystem/opimpl.py pypy/dist/pypy/rpython/memory/gc.py pypy/dist/pypy/rpython/memory/test/test_gc.py Log: make the semispace GC safe against recursive collects Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Tue Oct 9 19:14:12 2007 @@ -436,7 +436,7 @@ # __________ debugging __________ 'debug_view': LLOp(), - 'debug_print': LLOp(), + 'debug_print': LLOp(canrun=True), 'debug_pdb': LLOp(), 'debug_assert': LLOp(tryfold=True), 'debug_fatalerror': LLOp(), Modified: pypy/dist/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/opimpl.py Tue Oct 9 19:14:12 2007 @@ -386,6 +386,10 @@ raise TypeError("cannot fold getfield on mutable array") return p[index] +def op_debug_print(*args): + for arg in args: + print arg + # ____________________________________________________________ def get_op_impl(opname): Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Tue Oct 9 19:14:12 2007 @@ -1044,7 +1044,8 @@ free_non_gc_object(roots) scan = self.scan_copied(scan) # walk over list of objects that contain weakrefs - # if the object it references survives and invalidate it otherwise + # if the object it references survives then update the weakref + # otherwise invalidate the weakref new_with_weakref = self.AddressLinkedList() while self.objects_with_weakrefs.non_empty(): obj = self.objects_with_weakrefs.pop() @@ -1059,9 +1060,20 @@ pointing_to) else: (obj + offset).address[0] = NULL + # XXX the next line can be indented? only do after weakref + # tests pass new_with_weakref.append(obj) self.objects_with_weakrefs.delete() self.objects_with_weakrefs = new_with_weakref + if self.run_finalizers.non_empty(): + # we are in an inner collection, caused by a finalizer + # the run_finalizers objects need to be copied + new_run_finalizer = self.AddressLinkedList() + while self.run_finalizers.non_empty(): + obj = self.run_finalizers.pop() + new_run_finalizer.append(self.copy(obj)) + self.run_finalizers.delete() + self.run_finalizers = new_run_finalizer # walk over list of objects with finalizers # if it is not copied, add it to the list of to-be-called finalizers # and copy it, to me make the finalizer runnable Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Tue Oct 9 19:14:12 2007 @@ -153,6 +153,31 @@ res = self.interpret(f, [5]) assert res == 12 + def test_finalizer_calls_collect(self): + class B(object): + pass + b = B() + b.nextid = 0 + b.num_deleted = 0 + class A(object): + def __init__(self): + self.id = b.nextid + b.nextid += 1 + def __del__(self): + b.num_deleted += 1 + llop.gc__collect(lltype.Void) + def f(x): + a = A() + i = 0 + while i < x: + i += 1 + a = A() + llop.gc__collect(lltype.Void) + llop.gc__collect(lltype.Void) + return b.num_deleted + res = self.interpret(f, [5]) + assert res == 6 + def test_finalizer_resurrects(self): class B(object): pass @@ -224,6 +249,7 @@ res = self.interpret(f, []) assert res + class TestMarkSweepGC(GCTest): from pypy.rpython.memory.gc import MarkSweepGC as GCClass From arigo at codespeak.net Tue Oct 9 19:19:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 19:19:29 +0200 (CEST) Subject: [pypy-svn] r47360 - in pypy/dist/pypy/rpython: lltypesystem memory memory/test Message-ID: <20071009171929.CD64781B9@code0.codespeak.net> Author: arigo Date: Tue Oct 9 19:19:29 2007 New Revision: 47360 Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py pypy/dist/pypy/rpython/memory/gc.py pypy/dist/pypy/rpython/memory/gcwrapper.py pypy/dist/pypy/rpython/memory/test/test_gc.py Log: Expand space_size when memory runs out. This allows us to start with a smaller space_size, currently 8MB, and grow as needed. Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llarena.py Tue Oct 9 19:19:29 2007 @@ -279,7 +279,7 @@ def llimpl_arena_malloc(nbytes, zero): addr = llmemory.raw_malloc(nbytes) - if zero: + if zero and bool(addr): clear_large_memory_chunk(addr, nbytes) return addr register_external(arena_malloc, [int, bool], llmemory.Address, Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Tue Oct 9 19:19:29 2007 @@ -957,14 +957,15 @@ ('typeid', lltype.Signed)) def __init__(self, AddressLinkedList, space_size=4096, + max_space_size=sys.maxint//2+1, get_roots=None): self.space_size = space_size + self.max_space_size = max_space_size self.get_roots = get_roots self.gcheaderbuilder = GCHeaderBuilder(self.HDR) self.AddressLinkedList = AddressLinkedList def setup(self): - self.bytes_malloced = 0 self.tospace = llarena.arena_malloc(self.space_size, True) debug_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size @@ -973,6 +974,7 @@ self.free = self.tospace self.objects_with_finalizers = self.AddressLinkedList() self.run_finalizers = self.AddressLinkedList() + self.executing_finalizers = False self.objects_with_weakrefs = self.AddressLinkedList() def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, @@ -980,12 +982,7 @@ size_gc_header = self.gcheaderbuilder.size_gc_header totalsize = size_gc_header + size if raw_malloc_usage(totalsize) > self.top_of_space - self.free: - if not can_collect: - raise memoryError - self.collect() - #XXX need to increase the space size if the object is too big - #for bonus points do big objects differently - if raw_malloc_usage(totalsize) > self.top_of_space - self.free: + if not can_collect or not self.obtain_free_space(totalsize): raise memoryError result = self.free llarena.arena_reserve(result, totalsize) @@ -1007,12 +1004,7 @@ except OverflowError: raise memoryError if raw_malloc_usage(totalsize) > self.top_of_space - self.free: - if not can_collect: - raise memoryError - self.collect() - #XXX need to increase the space size if the object is too big - #for bonus points do big objects differently - if raw_malloc_usage(totalsize) > self.top_of_space - self.free: + if not can_collect or not self.obtain_free_space(totalsize): raise memoryError result = self.free llarena.arena_reserve(result, totalsize) @@ -1027,7 +1019,68 @@ malloc_fixedsize_clear = malloc_fixedsize malloc_varsize_clear = malloc_varsize - def collect(self): + def obtain_free_space(self, needed): + # XXX for bonus points do big objects differently + needed = raw_malloc_usage(needed) + self.collect() + missing = needed - (self.top_of_space - self.free) + if missing <= 0: + return True # success + else: + # first check if the object could possibly fit + proposed_size = self.space_size + while missing > 0: + if proposed_size >= self.max_space_size: + return False # no way + missing -= proposed_size + proposed_size *= 2 + # For address space fragmentation reasons, we double the space + # size possibly several times, moving the objects at each step, + # instead of going directly for the final size. We assume that + # it's a rare case anyway. + while self.space_size < proposed_size: + if not self.double_space_size(): + return False + debug_assert(needed <= self.top_of_space - self.free, + "double_space_size() failed to do its job") + return True + + def double_space_size(self): + old_fromspace = self.fromspace + newsize = self.space_size * 2 + newspace = llarena.arena_malloc(newsize, True) + if not newspace: + return False # out of memory + llarena.arena_free(old_fromspace) + self.fromspace = newspace + # now self.tospace contains the existing objects and + # self.fromspace is the freshly allocated bigger space + + self.collect(size_changing=True) + self.top_of_space = self.tospace + newsize + # now self.tospace is the freshly allocated bigger space, + # and self.fromspace is the old smaller space, now empty + llarena.arena_free(self.fromspace) + + newspace = llarena.arena_malloc(newsize, True) + if not newspace: + # Complex failure case: we have in self.tospace a big chunk + # of memory, and the two smaller original spaces are already gone. + # Unsure if it's worth these efforts, but we can artificially + # split self.tospace in two again... + self.max_space_size = self.space_size # don't try to grow again, + # because doing arena_free(self.fromspace) would crash + self.fromspace = self.tospace + self.space_size + self.top_of_space = self.fromspace + debug_assert(self.free <= self.top_of_space, + "unexpected growth of GC space usage during collect") + return False # out of memory + + self.fromspace = newspace + self.space_size = newsize + return True # success + + def collect(self, size_changing=False): ## print "collecting" tospace = self.fromspace fromspace = self.tospace @@ -1087,8 +1140,9 @@ scan = self.scan_copied(scan) self.objects_with_finalizers.delete() self.objects_with_finalizers = new_with_finalizer - llarena.arena_reset(fromspace, self.space_size, True) - self.execute_finalizers() + if not size_changing: + llarena.arena_reset(fromspace, self.space_size, True) + self.execute_finalizers() def scan_copied(self, scan): while scan < self.free: @@ -1184,11 +1238,17 @@ hdr.typeid = typeid def execute_finalizers(self): - while self.run_finalizers.non_empty(): - obj = self.run_finalizers.pop() - hdr = self.header(obj) - finalizer = self.getfinalizer(hdr.typeid) - finalizer(obj) + if self.executing_finalizers: + return # the outer invocation of execute_finalizers() will do it + self.executing_finalizers = True + try: + while self.run_finalizers.non_empty(): + obj = self.run_finalizers.pop() + hdr = self.header(obj) + finalizer = self.getfinalizer(hdr.typeid) + finalizer(obj) + finally: + self.executing_finalizers = False STATISTICS_NUMBERS = 0 @@ -1331,7 +1391,7 @@ GC_PARAMS = {'start_heap_size': 8*1024*1024} # XXX adjust return MarkSweepGC, GC_PARAMS elif config.translation.frameworkgc == "semispace": - GC_PARAMS = {'space_size': 32*1024*1024} # XXX fixed at 32MB + GC_PARAMS = {'space_size': 8*1024*1024} # XXX adjust return SemiSpaceGC, GC_PARAMS else: raise ValueError("unknown value for frameworkgc: %r" % ( Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Tue Oct 9 19:19:29 2007 @@ -7,9 +7,9 @@ class GCManagedHeap(object): - def __init__(self, llinterp, flowgraphs, gc_class): + def __init__(self, llinterp, flowgraphs, gc_class, GC_PARAMS={}): self.AddressLinkedList = get_address_linked_list(10, hackishpop=True) - self.gc = gc_class(self.AddressLinkedList) + self.gc = gc_class(self.AddressLinkedList, **GC_PARAMS) self.gc.get_roots = self.get_roots_from_llinterp self.llinterp = llinterp self.constantroots = [] @@ -102,11 +102,12 @@ def ll_finalizer(addr): old_active_frame = self.llinterp.active_frame try: - v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) - self.llinterp.eval_graph(destrgraph, [v]) - except llinterp.LLException: - raise RuntimeError( - "a finalizer raised an exception, shouldn't happen") + try: + v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) + self.llinterp.eval_graph(destrgraph, [v]) + except llinterp.LLException: + raise RuntimeError( + "a finalizer raised an exception, shouldn't happen") finally: self.llinterp.active_frame = old_active_frame return ll_finalizer @@ -145,6 +146,6 @@ if parent is not None: reccollect(constants, parent._as_ptr()) -def prepare_graphs_and_create_gc(llinterp, GCClass): +def prepare_graphs_and_create_gc(llinterp, GCClass, GC_PARAMS={}): flowgraphs = llinterp.typer.annotator.translator.graphs[:] - llinterp.heap = GCManagedHeap(llinterp, flowgraphs, GCClass) + llinterp.heap = GCManagedHeap(llinterp, flowgraphs, GCClass, GC_PARAMS) Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Tue Oct 9 19:19:29 2007 @@ -17,6 +17,7 @@ class GCTest(object): + GC_PARAMS = {} def setup_class(cls): cls._saved_logstate = py.log._getstate() @@ -29,7 +30,8 @@ def interpret(self, func, values, **kwds): interp, graph = get_interpreter(func, values, **kwds) - gcwrapper.prepare_graphs_and_create_gc(interp, self.GCClass) + gcwrapper.prepare_graphs_and_create_gc(interp, self.GCClass, + self.GC_PARAMS) return interp.eval_graph(graph, values) def test_llinterp_lists(self): @@ -256,6 +258,9 @@ class TestSemiSpaceGC(GCTest): from pypy.rpython.memory.gc import SemiSpaceGC as GCClass +class TestGrowingSemiSpaceGC(TestSemiSpaceGC): + GC_PARAMS = {'space_size': 64} + class TestDeferredRefcountingGC(GCTest): from pypy.rpython.memory.gc import DeferredRefcountingGC as GCClass def setup_class(cls): From cfbolz at codespeak.net Tue Oct 9 19:20:42 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 9 Oct 2007 19:20:42 +0200 (CEST) Subject: [pypy-svn] r47361 - pypy/dist/pypy/rpython/memory/test Message-ID: <20071009172042.0FD2E81BA@code0.codespeak.net> Author: cfbolz Date: Tue Oct 9 19:20:41 2007 New Revision: 47361 Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: reshuffle these tests so that the cloning tests are only run on top of mark and sweep. Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Tue Oct 9 19:20:41 2007 @@ -79,13 +79,7 @@ else: return run -class TestMarkSweepGC(GCTest): - - class gcpolicy(gc.FrameworkGcPolicy): - class transformerclass(framework.FrameworkGCTransformer): - GC_PARAMS = {'start_heap_size': 4096 } - root_stack_depth = 200 - gcname = "framework" +class GenericGCTests(GCTest): def heap_usage(self, statistics): try: @@ -323,6 +317,72 @@ res = run([4, 42]) #XXX pure lazyness here too assert res == 12 + def test_interior_ptrs(self): + from pypy.rpython.lltypesystem.lltype import Struct, GcStruct, GcArray + from pypy.rpython.lltypesystem.lltype import Array, Signed, malloc + + S1 = Struct("S1", ('x', Signed)) + T1 = GcStruct("T1", ('s', S1)) + def f1(): + t = malloc(T1) + t.s.x = 1 + return t.s.x + + S2 = Struct("S2", ('x', Signed)) + T2 = GcArray(S2) + def f2(): + t = malloc(T2, 1) + t[0].x = 1 + return t[0].x + + S3 = Struct("S3", ('x', Signed)) + T3 = GcStruct("T3", ('items', Array(S3))) + def f3(): + t = malloc(T3, 1) + t.items[0].x = 1 + return t.items[0].x + + S4 = Struct("S4", ('x', Signed)) + T4 = Struct("T4", ('s', S4)) + U4 = GcArray(T4) + def f4(): + u = malloc(U4, 1) + u[0].s.x = 1 + return u[0].s.x + + S5 = Struct("S5", ('x', Signed)) + T5 = GcStruct("T5", ('items', Array(S5))) + def f5(): + t = malloc(T5, 1) + return len(t.items) + + T6 = GcStruct("T6", ('s', Array(Signed))) + def f6(): + t = malloc(T6, 1) + t.s[0] = 1 + return t.s[0] + + def func(): + return (f1() * 100000 + + f2() * 10000 + + f3() * 1000 + + f4() * 100 + + f5() * 10 + + f6()) + + assert func() == 111111 + run = self.runner(func) + res = run([]) + assert res == 111111 + +class TestMarkSweepGC(GenericGCTests): + class gcpolicy(gc.FrameworkGcPolicy): + class transformerclass(framework.FrameworkGCTransformer): + GC_PARAMS = {'start_heap_size': 4096 } + root_stack_depth = 200 + gcname = "framework" + + def test_cloning(self): B = lltype.GcStruct('B', ('x', lltype.Signed)) A = lltype.GcStruct('A', ('b', lltype.Ptr(B)), @@ -537,63 +597,6 @@ res = run([3, 0]) assert res == 1 - def test_interior_ptrs(self): - from pypy.rpython.lltypesystem.lltype import Struct, GcStruct, GcArray - from pypy.rpython.lltypesystem.lltype import Array, Signed, malloc - - S1 = Struct("S1", ('x', Signed)) - T1 = GcStruct("T1", ('s', S1)) - def f1(): - t = malloc(T1) - t.s.x = 1 - return t.s.x - - S2 = Struct("S2", ('x', Signed)) - T2 = GcArray(S2) - def f2(): - t = malloc(T2, 1) - t[0].x = 1 - return t[0].x - - S3 = Struct("S3", ('x', Signed)) - T3 = GcStruct("T3", ('items', Array(S3))) - def f3(): - t = malloc(T3, 1) - t.items[0].x = 1 - return t.items[0].x - - S4 = Struct("S4", ('x', Signed)) - T4 = Struct("T4", ('s', S4)) - U4 = GcArray(T4) - def f4(): - u = malloc(U4, 1) - u[0].s.x = 1 - return u[0].s.x - - S5 = Struct("S5", ('x', Signed)) - T5 = GcStruct("T5", ('items', Array(S5))) - def f5(): - t = malloc(T5, 1) - return len(t.items) - - T6 = GcStruct("T6", ('s', Array(Signed))) - def f6(): - t = malloc(T6, 1) - t.s[0] = 1 - return t.s[0] - - def func(): - return (f1() * 100000 + - f2() * 10000 + - f3() * 1000 + - f4() * 100 + - f5() * 10 + - f6()) - - assert func() == 111111 - run = self.runner(func) - res = run([]) - assert res == 111111 class TestStacklessMarkSweepGC(TestMarkSweepGC): @@ -625,7 +628,7 @@ assert res == 20 -class TestSemiSpaceGC(TestMarkSweepGC): +class TestSemiSpaceGC(GenericGCTests): def setup_class(cls): py.test.skip("in-progress") From arigo at codespeak.net Tue Oct 9 20:06:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 20:06:11 +0200 (CEST) Subject: [pypy-svn] r47362 - pypy/dist/pypy/rpython/memory Message-ID: <20071009180611.BE22B81B9@code0.codespeak.net> Author: arigo Date: Tue Oct 9 20:06:11 2007 New Revision: 47362 Modified: pypy/dist/pypy/rpython/memory/gc.py Log: This fixes test_newgc. Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Tue Oct 9 20:06:11 2007 @@ -1022,7 +1022,7 @@ def obtain_free_space(self, needed): # XXX for bonus points do big objects differently needed = raw_malloc_usage(needed) - self.collect() + self.semispace_collect() missing = needed - (self.top_of_space - self.free) if missing <= 0: return True # success @@ -1056,7 +1056,7 @@ # now self.tospace contains the existing objects and # self.fromspace is the freshly allocated bigger space - self.collect(size_changing=True) + self.semispace_collect(size_changing=True) self.top_of_space = self.tospace + newsize # now self.tospace is the freshly allocated bigger space, # and self.fromspace is the old smaller space, now empty @@ -1080,7 +1080,12 @@ self.space_size = newsize return True # success - def collect(self, size_changing=False): + def collect(self): + self.semispace_collect() + # the indirection is required by the fact that collect() is referred + # to by the gc transformer, and the default argument would crash + + def semispace_collect(self, size_changing=False): ## print "collecting" tospace = self.fromspace fromspace = self.tospace From arigo at codespeak.net Tue Oct 9 21:12:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 9 Oct 2007 21:12:59 +0200 (CEST) Subject: [pypy-svn] r47363 - pypy/dist/pypy/rpython/memory/test Message-ID: <20071009191259.8BF0D81BC@code0.codespeak.net> Author: arigo Date: Tue Oct 9 21:12:55 2007 New Revision: 47363 Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: Make the (skipped) SemiSpaceG tests run again. Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Tue Oct 9 21:12:55 2007 @@ -80,6 +80,7 @@ return run class GenericGCTests(GCTest): + gcname = "framework" def heap_usage(self, statistics): try: @@ -380,7 +381,6 @@ class transformerclass(framework.FrameworkGCTransformer): GC_PARAMS = {'start_heap_size': 4096 } root_stack_depth = 200 - gcname = "framework" def test_cloning(self): From cfbolz at codespeak.net Wed Oct 10 00:39:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 10 Oct 2007 00:39:27 +0200 (CEST) Subject: [pypy-svn] r47365 - in pypy/dist/pypy/rpython/memory: gctransform test Message-ID: <20071009223927.1B343819B@code0.codespeak.net> Author: cfbolz Date: Wed Oct 10 00:39:25 2007 New Revision: 47365 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: make all the semi space tests in test_transformed_gc pass, by inserting necessary casts (which don't do anything after translation, but are needed on top of the llinterp to get the layers of translation done correctly). Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Wed Oct 10 00:39:25 2007 @@ -536,6 +536,12 @@ c_type = rmodel.inputconst(lltype.Void, llmemory.Address) for k,var in enumerate(livevars): c_k = rmodel.inputconst(lltype.Signed, k) + # this cast doesn't actually do anything after translation to C + # however, on top of the llinterp it is needed + # see comment in llmemory before cast_ptr_to_weakrefptr + if var.concretetype == llmemory.WeakRefPtr: + var = hop.llops.genop("cast_weakrefptr_to_ptr", [var], + resulttype=WEAKREFPTR) v_adr = gen_cast(hop.llops, llmemory.Address, var) hop.genop("raw_store", [base_addr, c_type, c_k, v_adr]) @@ -555,6 +561,18 @@ c_k = rmodel.inputconst(lltype.Signed, k) v_newaddr = hop.genop("raw_load", [base_addr, c_type, c_k], resulttype=llmemory.Address) + if var.concretetype == llmemory.WeakRefPtr: + # these casts doesn't actually do anything after + # translation to C however, on top of the llinterp it is + # needed see comment in llmemory before + # cast_ptr_to_weakrefptr + # XXX not nice + nvar = hop.llops.genop("cast_adr_to_ptr", [v_newaddr], + resulttype=WEAKREFPTR) + nvar = hop.llops.genop("cast_ptr_to_weakrefptr", [nvar], + resulttype=llmemory.WeakRefPtr) + v_newaddr = hop.llops.genop("cast_ptr_to_adr", [nvar], + resulttype=llmemory.Address) hop.genop("gc_reload_possibly_moved", [v_newaddr, var]) def compute_borrowed_vars(self, graph): Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Wed Oct 10 00:39:25 2007 @@ -630,9 +630,6 @@ class TestSemiSpaceGC(GenericGCTests): - def setup_class(cls): - py.test.skip("in-progress") - class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): from pypy.rpython.memory.gc import SemiSpaceGC as GCClass From fijal at codespeak.net Wed Oct 10 09:19:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Oct 2007 09:19:51 +0200 (CEST) Subject: [pypy-svn] r47368 - in pypy/dist/pypy/rlib: . test Message-ID: <20071010071951.2FE2D81DE@code0.codespeak.net> Author: fijal Date: Wed Oct 10 09:19:50 2007 New Revision: 47368 Modified: pypy/dist/pypy/rlib/libffi.py pypy/dist/pypy/rlib/test/test_libffi.py Log: Another approach - more rpythonic, still complaining about bad cast from <* Array of Char> to <* Array of Float> Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Wed Oct 10 09:19:50 2007 @@ -5,6 +5,7 @@ from pypy.rpython.tool import rffi_platform from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.rarithmetic import intmask includes = ['dlfcn.h', 'ffi.h'] @@ -137,7 +138,7 @@ def push_arg_as_ffiptr(ffitp, TP, arg, ll_buf): # this is for primitive types. For structures and arrays # would be something different (more dynamic) - TP_P = rffi.CArray(TP) + TP_P = rffi.CArrayPtr(TP) rffi.cast(TP_P, ll_buf)[0] = arg push_arg_as_ffiptr._annspecialcase_ = 'specialize:argtype(1)' @@ -155,20 +156,24 @@ self.ll_argtypes = lltype.malloc(FFI_TYPE_PP.TO, argnum, flavor='raw') for i in range(argnum): self.ll_argtypes[i] = argtypes[i] + # XXX why cast to FFI_TYPE_PP is needed? ll2ctypes bug? res = c_ffi_prep_cif(self.ll_cif, FFI_DEFAULT_ABI, rffi.cast(rffi.UINT, argnum), restype, - self.ll_argtypes) + rffi.cast(FFI_TYPE_PP, self.ll_argtypes)) if not res == FFI_OK: raise OSError(-1, "Wrong typedef") for i in range(argnum): # space for each argument - self.ll_args[i] = lltype.malloc(rffi.VOIDP.TO, argtypes[i].c_size, + self.ll_args[i] = lltype.malloc(rffi.VOIDP.TO, + intmask(argtypes[i].c_size), flavor='raw') - self.ll_result = lltype.malloc(rffi.VOIDP.TO, restype.c_size, + self.ll_result = lltype.malloc(rffi.VOIDP.TO, intmask(restype.c_size), flavor='raw') - def push_arg(self, num, TP, value): - push_arg_as_ffiptr(self.argtypes[i], TP, value, self.ll_args[i]) + # XXX some rpython trick to get rid of TP here? + def push_arg(self, num, value): + TP = lltype.typeOf(value) + push_arg_as_ffiptr(self.argtypes[num], TP, value, self.ll_args[num]) self.ready_args[num] = 1 def _check_args(self): @@ -182,11 +187,14 @@ def call(self, RES_TP): self._check_args() - c_ffi_call(self.ll_cif, self.func_sym, - rffi.cast(rffi.VOIDP, self.restype), + c_ffi_call(self.ll_cif, self.funcsym, + rffi.cast(rffi.VOIDP, self.ll_result), rffi.cast(VOIDPP, self.ll_args)) if self.restype != ffi_type_void: - res = rffi.cast(lltype.Ptr(CArray(RES_TP)), self.ll_result)[0] + TP = rffi.CArrayPtr(RES_TP) + res = rffi.cast(TP, self.ll_result)[0] + else: + res = None self._clean_args() return res call._annspecialcase_ = 'specialize:argtype(1)' Modified: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_libffi.py (original) +++ pypy/dist/pypy/rlib/test/test_libffi.py Wed Oct 10 09:19:50 2007 @@ -43,7 +43,8 @@ ptr = lib.getpointer('rand', [], ffi_type_sint) zeroes = 0 for i in range(100): - res = ptr.call([]) + res = ptr.call(rffi.INT) + print res if not res: zeroes += 1 assert not zeroes @@ -53,18 +54,22 @@ libm = CDLL('libm.so') pow = libm.getpointer('pow', [ffi_type_double, ffi_type_double], ffi_type_double) - pow.push_arg(0, rffi.DOUBLE, 2.0) - pow.push_arg(1, rffi.DOUBLE, 2.0) - assert pow.call() == 4.0 - pow.push_arg(0, rffi.DOUBLE, 3.0) - pow.push_arg(1, rffi.DOUBLE, 3.0) - assert pow.call() == 27.0 + pow.push_arg(0, 2.0) + pow.push_arg(1, 2.0) + res = pow.call(rffi.DOUBLE) + assert res == 4.0 + pow.push_arg(0, 3.0) + pow.push_arg(1, 3.0) + res = pow.call(rffi.DOUBLE) + assert res == 27.0 def test_compile(self): - py.test.skip("in-progress") def f(x, y): libm = CDLL('libm.so') - c_pow = libm.getpointer('pow', (ffi_type_double, ffi_type_double), ffi_type_double) - return c_pow.call((x, y)) + c_pow = libm.getpointer('pow', [ffi_type_double, ffi_type_double], ffi_type_double) + c_pow.push_arg(0, x) + c_pow.push_arg(1, y) + return c_pow.call(rffi.DOUBLE) interpret(f, [2.0, 4.0]) + From fijal at codespeak.net Wed Oct 10 09:20:35 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Oct 2007 09:20:35 +0200 (CEST) Subject: [pypy-svn] r47369 - pypy/dist/pypy/rlib/test Message-ID: <20071010072035.7F9A781DE@code0.codespeak.net> Author: fijal Date: Wed Oct 10 09:20:35 2007 New Revision: 47369 Modified: pypy/dist/pypy/rlib/test/test_libffi.py Log: Skip this test Modified: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_libffi.py (original) +++ pypy/dist/pypy/rlib/test/test_libffi.py Wed Oct 10 09:20:35 2007 @@ -64,6 +64,7 @@ assert res == 27.0 def test_compile(self): + py.test.skip("Broken") def f(x, y): libm = CDLL('libm.so') c_pow = libm.getpointer('pow', [ffi_type_double, ffi_type_double], ffi_type_double) From fijal at codespeak.net Wed Oct 10 09:24:01 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Oct 2007 09:24:01 +0200 (CEST) Subject: [pypy-svn] r47370 - in pypy/dist/pypy/rlib: . test Message-ID: <20071010072401.4123A81DE@code0.codespeak.net> Author: fijal Date: Wed Oct 10 09:24:00 2007 New Revision: 47370 Modified: pypy/dist/pypy/rlib/libffi.py pypy/dist/pypy/rlib/test/test_libffi.py Log: Change interface slightly Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Wed Oct 10 09:24:00 2007 @@ -149,7 +149,8 @@ self.restype = restype self.funcsym = funcsym argnum = len(argtypes) - self.ready_args = [0] * argnum + self.argnum = argnum + self.pushed_args = 0 TP = rffi.CArray(rffi.VOIDP) self.ll_args = lltype.malloc(TP, argnum, flavor='raw') self.ll_cif = lltype.malloc(FFI_CIFP.TO, flavor='raw') @@ -171,19 +172,21 @@ flavor='raw') # XXX some rpython trick to get rid of TP here? - def push_arg(self, num, value): + def push_arg(self, value): + if self.pushed_args == self.argnum: + raise TypeError("Too much arguments, eats %d, pushed %d" % + (self.argnum, self.argnum + 1)) TP = lltype.typeOf(value) - push_arg_as_ffiptr(self.argtypes[num], TP, value, self.ll_args[num]) - self.ready_args[num] = 1 + push_arg_as_ffiptr(self.argtypes[self.pushed_args], TP, value, + self.ll_args[self.pushed_args]) + self.pushed_args += 1 def _check_args(self): - for num in range(len(self.ready_args)): - if not self.ready_args[num]: - raise TypeError("Did not specify arg nr %d" % num) + if self.pushed_args < self.argnum: + raise TypeError("Did not specify arg nr %d" % (self.pushed_args + 1)) def _clean_args(self): - for num in range(len(self.ready_args)): - self.ready_args[num] = 0 + self.pushed_args = 0 def call(self, RES_TP): self._check_args() Modified: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_libffi.py (original) +++ pypy/dist/pypy/rlib/test/test_libffi.py Wed Oct 10 09:24:00 2007 @@ -54,12 +54,12 @@ libm = CDLL('libm.so') pow = libm.getpointer('pow', [ffi_type_double, ffi_type_double], ffi_type_double) - pow.push_arg(0, 2.0) - pow.push_arg(1, 2.0) + pow.push_arg(2.0) + pow.push_arg(2.0) res = pow.call(rffi.DOUBLE) assert res == 4.0 - pow.push_arg(0, 3.0) - pow.push_arg(1, 3.0) + pow.push_arg(3.0) + pow.push_arg(3.0) res = pow.call(rffi.DOUBLE) assert res == 27.0 @@ -68,8 +68,8 @@ def f(x, y): libm = CDLL('libm.so') c_pow = libm.getpointer('pow', [ffi_type_double, ffi_type_double], ffi_type_double) - c_pow.push_arg(0, x) - c_pow.push_arg(1, y) + c_pow.push_arg(x) + c_pow.push_arg(y) return c_pow.call(rffi.DOUBLE) interpret(f, [2.0, 4.0]) From fijal at codespeak.net Wed Oct 10 10:40:06 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Oct 2007 10:40:06 +0200 (CEST) Subject: [pypy-svn] r47371 - pypy/dist/pypy/rlib Message-ID: <20071010084006.6D113815F@code0.codespeak.net> Author: fijal Date: Wed Oct 10 10:40:05 2007 New Revision: 47371 Modified: pypy/dist/pypy/rlib/libffi.py Log: Strange hack with trying different includes Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Wed Oct 10 10:40:05 2007 @@ -54,7 +54,11 @@ class cConfig: pass -cConfig.__dict__.update(rffi_platform.configure(CConfig)) +try: + cConfig.__dict__.update(rffi_platform.configure(CConfig)) +except: + includes[1] = ['ffi/ffi.h'] + cConfig.__dict__.update(rffi_platform.configure(CConfig)) FFI_TYPE_P.TO.become(cConfig.ffi_type) size_t = cConfig.size_t From fijal at codespeak.net Wed Oct 10 10:41:26 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Oct 2007 10:41:26 +0200 (CEST) Subject: [pypy-svn] r47372 - pypy/dist/pypy/rlib Message-ID: <20071010084126.AEAD581E6@code0.codespeak.net> Author: fijal Date: Wed Oct 10 10:41:26 2007 New Revision: 47372 Modified: pypy/dist/pypy/rlib/libffi.py Log: Another strange hack Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Wed Oct 10 10:41:26 2007 @@ -57,7 +57,7 @@ try: cConfig.__dict__.update(rffi_platform.configure(CConfig)) except: - includes[1] = ['ffi/ffi.h'] + includes[1] = 'ffi/ffi.h' cConfig.__dict__.update(rffi_platform.configure(CConfig)) FFI_TYPE_P.TO.become(cConfig.ffi_type) From fijal at codespeak.net Wed Oct 10 10:46:19 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Oct 2007 10:46:19 +0200 (CEST) Subject: [pypy-svn] r47373 - pypy/dist/pypy/rlib Message-ID: <20071010084619.A308681E7@code0.codespeak.net> Author: fijal Date: Wed Oct 10 10:46:19 2007 New Revision: 47373 Modified: pypy/dist/pypy/rlib/libffi.py Log: Another try Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Wed Oct 10 10:46:19 2007 @@ -8,6 +8,7 @@ from pypy.rlib.rarithmetic import intmask includes = ['dlfcn.h', 'ffi.h'] +include_dirs = ['/usr/include/libffi'] FFI_TYPE_P = lltype.Ptr(lltype.ForwardReference()) FFI_TYPE_PP = rffi.CArrayPtr(FFI_TYPE_P) @@ -15,6 +16,7 @@ class CConfig: _includes_ = includes _libraries_ = ['ffi'] + _include_dirs_ = include_dirs RTLD_LOCAL = rffi_platform.DefinedConstantInteger('RTLD_LOCAL') RTLD_NOW = rffi_platform.DefinedConstantInteger('RTLD_NOW') @@ -54,11 +56,7 @@ class cConfig: pass -try: - cConfig.__dict__.update(rffi_platform.configure(CConfig)) -except: - includes[1] = 'ffi/ffi.h' - cConfig.__dict__.update(rffi_platform.configure(CConfig)) +cConfig.__dict__.update(rffi_platform.configure(CConfig)) FFI_TYPE_P.TO.become(cConfig.ffi_type) size_t = cConfig.size_t From fijal at codespeak.net Wed Oct 10 10:47:44 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Oct 2007 10:47:44 +0200 (CEST) Subject: [pypy-svn] r47374 - pypy/dist/pypy/rpython/tool Message-ID: <20071010084744.1A5EE81E7@code0.codespeak.net> Author: fijal Date: Wed Oct 10 10:47:43 2007 New Revision: 47374 Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py Log: Forward libraries as well Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Wed Oct 10 10:47:43 2007 @@ -113,7 +113,9 @@ self.f.write(question + "\n") self.close() include_dirs = getattr(self.config, '_include_dirs_', []) - build_executable([self.path], include_dirs=include_dirs) + libraries = getattr(self.config, '_libraries_', []) + build_executable([self.path], include_dirs=include_dirs, + libraries=libraries) def configure(CConfig): """Examine the local system by running the C compiler. @@ -140,7 +142,9 @@ writer.close() include_dirs = getattr(CConfig, '_include_dirs_', []) - infolist = list(run_example_code(writer.path, include_dirs)) + libraries = getattr(CConfig, '_libraries_', []) + infolist = list(run_example_code(writer.path, include_dirs, + libraries)) assert len(infolist) == len(entries) resultinfo = {} @@ -347,7 +351,6 @@ return info['value'] return None - class DefinedConstantString(CConfigEntry): """ """ @@ -478,8 +481,9 @@ } """ -def run_example_code(filepath, include_dirs=[]): - executable = build_executable([filepath], include_dirs=include_dirs) +def run_example_code(filepath, include_dirs=[], libraries=[]): + executable = build_executable([filepath], include_dirs=include_dirs, + libraries=libraries) output = py.process.cmdexec(executable) section = None for line in output.splitlines(): From arigo at codespeak.net Wed Oct 10 12:21:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 10 Oct 2007 12:21:29 +0200 (CEST) Subject: [pypy-svn] r47377 - in pypy/dist/pypy/rpython/memory: . gctransform Message-ID: <20071010102129.B922C81EC@code0.codespeak.net> Author: arigo Date: Wed Oct 10 12:21:28 2007 New Revision: 47377 Modified: pypy/dist/pypy/rpython/memory/gc.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctransform/stacklessframework.py pypy/dist/pypy/rpython/memory/gctypelayout.py pypy/dist/pypy/rpython/memory/gcwrapper.py Log: Remove the 'static_roots' list. Now there is only the list of addresses of gcptrs that live inside prebuilt structures, independently of whether these prebuilt structures are GcStruct or plain Struct. This simplifies both mark&sweep and the semispace GC. In the latter it actually avoids a deep recursion that was overflowing the stack in pypy-c. (Tests still running, sorry, I have to check in in a bit of a hurry...) Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Wed Oct 10 12:21:28 2007 @@ -308,11 +308,6 @@ break # roots is a list of addresses to addresses: objects.append(curr.address[0]) - # the last sweep did not clear the mark bit of static roots, - # since they are not in the malloced_objects list - gc_info = curr.address[0] - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - hdr.typeid = hdr.typeid & (~1) free_non_gc_object(roots) # from this point onwards, no more mallocs should be possible old_malloced = self.bytes_malloced @@ -575,7 +570,11 @@ def init_gc_object(self, addr, typeid): hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) hdr.typeid = typeid << 1 - init_gc_object_immortal = init_gc_object + + def init_gc_object_immortal(self, addr, typeid): + # prebuilt gc structures always have the mark bit set + hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) + hdr.typeid = (typeid << 1) | 1 # experimental support for thread cloning def x_swap_pool(self, newpool): @@ -1086,7 +1085,7 @@ # to by the gc transformer, and the default argument would crash def semispace_collect(self, size_changing=False): -## print "collecting" + #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) tospace = self.fromspace fromspace = self.tospace self.fromspace = fromspace @@ -1153,12 +1152,13 @@ while scan < self.free: curr = scan + self.size_gc_header() self.trace_and_copy(curr) - scan += self.get_size(curr) + self.size_gc_header() + scan += self.size_gc_header() + self.get_size(curr) return scan def copy(self, obj): - if not self.fromspace <= obj < self.fromspace + self.space_size: - return self.copy_non_managed_obj(obj) + # Objects not living the GC heap have all been initialized by + # setting their 'forw' address so that it points to themselves. + # The logic below will thus simply return 'obj' if 'obj' is prebuilt. ## print "copying regularly", obj, if self.is_forwarded(obj): ## print "already copied to", self.get_forwarding_address(obj) @@ -1174,12 +1174,6 @@ self.set_forwarding_address(obj, newobj) return newobj - def copy_non_managed_obj(self, obj): #umph, PBCs, not really copy -## print "copying nonmanaged", obj - #we have to do the tracing here because PBCs are not moved to tospace - self.trace_and_copy(obj) - return obj - def trace_and_copy(self, obj): gc_info = self.header(obj) typeid = gc_info.typeid @@ -1238,8 +1232,9 @@ hdr.typeid = typeid def init_gc_object_immortal(self, addr, typeid): + # immortal objects always have forward to themselves hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.forw = NULL + hdr.forw = addr + self.gcheaderbuilder.size_gc_header hdr.typeid = typeid def execute_finalizers(self): Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Wed Oct 10 12:21:28 2007 @@ -28,7 +28,6 @@ class FrameworkGCTransformer(GCTransformer): use_stackless = False - extra_static_slots = 0 root_stack_depth = 163840 def __init__(self, translator): @@ -105,8 +104,6 @@ # set up dummy a table, to be overwritten with the real one in finish() gcdata.type_info_table = lltype.malloc(GCData.TYPE_INFO_TABLE, 0, immortal=True) - gcdata.static_roots = lltype.malloc(lltype.Array(llmemory.Address), 0, - immortal=True) # initialize the following two fields with a random non-NULL address, # to make the annotator happy. The fields are patched in finish() # to point to a real array (not 'static_roots', another one). @@ -271,10 +268,6 @@ llmemory.raw_memclear(stackbase, rootstacksize) gcdata.root_stack_top = stackbase gcdata.root_stack_base = stackbase - i = 0 - while i < len(gcdata.static_roots): - StackRootIterator.push_root(gcdata.static_roots[i]) - i += 1 setup_root_stack = staticmethod(setup_root_stack) need_root_stack = True @@ -359,29 +352,17 @@ ll_instance.inst_type_info_table = table #self.gcdata.type_info_table = table - static_gc_roots = self.layoutbuilder.static_gc_roots - ll_static_roots = lltype.malloc(lltype.Array(llmemory.Address), - len(static_gc_roots) + - self.extra_static_slots, - immortal=True) - for i in range(len(static_gc_roots)): - adr = static_gc_roots[i] - ll_static_roots[i] = adr - ll_instance.inst_static_roots = ll_static_roots - - addresses_of_static_ptrs_in_nongc = \ - self.layoutbuilder.addresses_of_static_ptrs_in_nongc + addresses_of_static_ptrs = self.layoutbuilder.addresses_of_static_ptrs ll_static_roots_inside = lltype.malloc(lltype.Array(llmemory.Address), - len(addresses_of_static_ptrs_in_nongc), + len(addresses_of_static_ptrs), immortal=True) - for i in range(len(addresses_of_static_ptrs_in_nongc)): - ll_static_roots_inside[i] = addresses_of_static_ptrs_in_nongc[i] + for i in range(len(addresses_of_static_ptrs)): + ll_static_roots_inside[i] = addresses_of_static_ptrs[i] ll_instance.inst_static_root_start = llmemory.cast_ptr_to_adr(ll_static_roots_inside) + llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address)) ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(ll_static_roots_inside) newgcdependencies = [] newgcdependencies.append(table) - newgcdependencies.append(ll_static_roots) newgcdependencies.append(ll_static_roots_inside) return newgcdependencies Modified: pypy/dist/pypy/rpython/memory/gctransform/stacklessframework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/stacklessframework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/stacklessframework.py Wed Oct 10 12:21:28 2007 @@ -5,7 +5,6 @@ class StacklessFrameworkGCTransformer(FrameworkGCTransformer): use_stackless = True - extra_static_slots = 1 # for the stack_capture()'d frame def __init__(self, translator): FrameworkGCTransformer.__init__(self, translator) @@ -41,6 +40,8 @@ from pypy.rlib.rstack import stack_capture sizeofaddr = llmemory.sizeof(llmemory.Address) gcdata = self.gcdata + captured_frame_holder = llmemory.raw_malloc(sizeofaddr) + captured_frame_holder.address[0] = llmemory.NULL class StackRootIterator: _alloc_flavor_ = 'raw' @@ -52,11 +53,12 @@ need_root_stack = False def __init__(self): + # XXX what should be done with the stack_capture()d frames + # when we are finished? what about moving GCs? frame = llmemory.cast_ptr_to_adr(stack_capture()) self.static_current = gcdata.static_root_start - index = len(gcdata.static_roots) - self.static_roots_index = index - gcdata.static_roots[index-1] = frame + captured_frame_holder.address[0] = frame + self.finished = False def pop(self): while self.static_current != gcdata.static_root_end: @@ -64,13 +66,9 @@ self.static_current += sizeofaddr if result.address[0].address[0] != llmemory.NULL: return result.address[0] - i = self.static_roots_index - if i > 0: - i -= 1 - self.static_roots_index = i - p = lltype.direct_arrayitems(gcdata.static_roots) - p = lltype.direct_ptradd(p, i) - return llmemory.cast_ptr_to_adr(p) + if not self.finished: + self.finished = True + return captured_frame_holder return llmemory.NULL return StackRootIterator Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Wed Oct 10 12:21:28 2007 @@ -1,5 +1,4 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena -from pypy.rpython.memory.gctransform.support import find_gc_ptrs_in_type class TypeLayoutBuilder(object): @@ -11,8 +10,12 @@ self.type_info_list = [dummy] # don't use typeid 0, helps debugging self.id_of_type = {} # {LLTYPE: type_id} self.seen_roots = {} - self.static_gc_roots = [] - self.addresses_of_static_ptrs_in_nongc = [] + # the following is a list of addresses of gc pointers living inside + # the prebuilt structures (independently on whether the prebuilt + # structures are themselves GcStruct or plain Struct). It should + # list all the locations that could possibly point to a GC heap + # object. + self.addresses_of_static_ptrs = [] self.finalizer_funcptrs = {} def get_type_id(self, TYPE): @@ -143,13 +146,11 @@ adr = llmemory.cast_ptr_to_adr(hdr) gc.init_gc_object_immortal(adr, typeid) - if find_gc_ptrs_in_type(TYPE): - adr = llmemory.cast_ptr_to_adr(value._as_ptr()) - if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): - self.static_gc_roots.append(adr) - else: - for a in gc_pointers_inside(value, adr): - self.addresses_of_static_ptrs_in_nongc.append(a) + # XXX should skip the gc pointers inside immutable structures, because + # they cannot be dynamically modified to point to GC heap objects + adr = llmemory.cast_ptr_to_adr(value._as_ptr()) + for a in gc_pointers_inside(value, adr): + self.addresses_of_static_ptrs.append(a) # ____________________________________________________________ # @@ -197,7 +198,7 @@ for a in gc_pointers_inside(getattr(v, n), adr + llmemory.offsetof(t, n)): yield a elif isinstance(t, lltype.Array): - if isinstance(t.OF, lltype.Ptr) and t2._needsgc(): + if isinstance(t.OF, lltype.Ptr) and t.OF.TO._gckind == 'gc': for i in range(len(v.items)): yield adr + llmemory.itemoffsetof(t, i) elif isinstance(t.OF, lltype.Struct): Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Wed Oct 10 12:21:28 2007 @@ -8,11 +8,11 @@ class GCManagedHeap(object): def __init__(self, llinterp, flowgraphs, gc_class, GC_PARAMS={}): - self.AddressLinkedList = get_address_linked_list(10, hackishpop=True) + self.RootLinkedList = get_address_linked_list(10, hackishpop=True) + self.AddressLinkedList = get_address_linked_list(10) self.gc = gc_class(self.AddressLinkedList, **GC_PARAMS) self.gc.get_roots = self.get_roots_from_llinterp self.llinterp = llinterp - self.constantroots = [] self.prepare_graphs(flowgraphs) self.gc.setup() @@ -26,20 +26,17 @@ TYPE = lltype.typeOf(obj) layoutbuilder.consider_constant(TYPE, obj, self.gc) - sizeofaddr = llmemory.sizeof(llmemory.Address) - for addr in layoutbuilder.static_gc_roots: - addrofaddr = llmemory.raw_malloc(sizeofaddr) - addrofaddr.address[0] = addr - self.constantroots.append(addrofaddr) - self.constantroots += layoutbuilder.addresses_of_static_ptrs_in_nongc + self.constantroots = layoutbuilder.addresses_of_static_ptrs def get_roots_from_llinterp(self): sizeofaddr = llmemory.sizeof(llmemory.Address) - ll = self.AddressLinkedList() + ll = self.RootLinkedList() for addrofaddr in self.constantroots: - ll.append(addrofaddr) + if addrofaddr.address[0]: + ll.append(addrofaddr) for addrofaddr in self.llinterp.find_roots(): - ll.append(addrofaddr) + if addrofaddr.address[0]: + ll.append(addrofaddr) return ll # ____________________________________________________________ From fijal at codespeak.net Wed Oct 10 13:44:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Oct 2007 13:44:46 +0200 (CEST) Subject: [pypy-svn] r47379 - in pypy/dist/pypy/rlib: . test Message-ID: <20071010114446.2989A81EC@code0.codespeak.net> Author: fijal Date: Wed Oct 10 13:44:45 2007 New Revision: 47379 Modified: pypy/dist/pypy/rlib/libffi.py pypy/dist/pypy/rlib/test/test_libffi.py Log: libffi wrapper works and compiles for basic types. Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Wed Oct 10 13:44:45 2007 @@ -78,11 +78,12 @@ lltype.Void : ffi_type_void, } -def external(name, args, result): +def external(name, args, result, **kwds): return rffi.llexternal(name, args, result, includes=includes, - libraries=['dl', 'ffi']) + libraries=['dl', 'ffi'], **kwds) -c_dlopen = external('dlopen', [rffi.CCHARP, rffi.INT], rffi.VOIDP) +c_dlopen = external('dlopen', [rffi.CCHARP, rffi.INT], rffi.VOIDP, + _nowrapper=True) c_dlclose = external('dlclose', [rffi.VOIDP], rffi.INT) c_dlerror = external('dlerror', [], rffi.CCHARP) c_dlsym = external('dlsym', [rffi.VOIDP, rffi.CCHARP], rffi.VOIDP) @@ -141,7 +142,8 @@ # this is for primitive types. For structures and arrays # would be something different (more dynamic) TP_P = rffi.CArrayPtr(TP) - rffi.cast(TP_P, ll_buf)[0] = arg + buf = rffi.cast(TP_P, ll_buf) + buf[0] = arg push_arg_as_ffiptr._annspecialcase_ = 'specialize:argtype(1)' class FuncPtr(object): @@ -215,10 +217,12 @@ class CDLL: def __init__(self, libname): - self.lib = dlopen(libname) + self.ll_libname = rffi.str2charp(libname) + self.lib = dlopen(self.ll_libname) def __del__(self): c_dlclose(self.lib) + lltype.free(self.ll_libname, flavor='raw') def getpointer(self, name, argtypes, restype): # these arguments are already casted to proper ffi Modified: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_libffi.py (original) +++ pypy/dist/pypy/rlib/test/test_libffi.py Wed Oct 10 13:44:45 2007 @@ -3,6 +3,7 @@ """ from pypy.rpython.test.test_llinterp import interpret +from pypy.translator.c.test.test_genc import compile from pypy.rlib.libffi import * from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED from pypy.rpython.lltypesystem import rffi, lltype @@ -22,8 +23,8 @@ #assert not ALLOCATED, not yet def test_dlopen(self): - py.test.raises(OSError, "dlopen('xxxxxxxxxxxx')") - assert dlopen('/lib/libc.so.6') + py.test.raises(OSError, "dlopen(rffi.str2charp('xxxxxxxxxxxx'))") + assert dlopen(rffi.str2charp('/lib/libc.so.6')) def get_libc(self): return CDLL('/lib/libc.so.6') @@ -64,7 +65,9 @@ assert res == 27.0 def test_compile(self): - py.test.skip("Broken") + # XXX cannot run it on top of llinterp, some problems + # with pointer casts + def f(x, y): libm = CDLL('libm.so') c_pow = libm.getpointer('pow', [ffi_type_double, ffi_type_double], ffi_type_double) @@ -72,5 +75,8 @@ c_pow.push_arg(y) return c_pow.call(rffi.DOUBLE) - interpret(f, [2.0, 4.0]) + fn = compile(f, [float, float]) + res = fn(2.0, 4.0) + assert res == 16.0 + From simonb at codespeak.net Wed Oct 10 18:30:03 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Wed, 10 Oct 2007 18:30:03 +0200 (CEST) Subject: [pypy-svn] r47380 - pypy/dist/pypy/translator/c/test Message-ID: <20071010163003.4182381F8@code0.codespeak.net> Author: simonb Date: Wed Oct 10 18:30:01 2007 New Revision: 47380 Modified: pypy/dist/pypy/translator/c/test/test_typed.py Log: this test breaks the c backend in a curious way Modified: pypy/dist/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_typed.py (original) +++ pypy/dist/pypy/translator/c/test/test_typed.py Wed Oct 10 18:30:01 2007 @@ -771,3 +771,17 @@ res = fn(i) assert res == expected + def test_swap(self): + py.test.skip("Failing test. ") + def func_swap(): + a = [] + b = range(10) + while b: + item = b.pop() + a.extend(b) + tmp = a + a = b + b = tmp + del a[:] + + f = self.getcompiled(func_swap, []) From tismer at codespeak.net Wed Oct 10 21:36:33 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Wed, 10 Oct 2007 21:36:33 +0200 (CEST) Subject: [pypy-svn] r47381 - pypy/dist/pypy/rlib Message-ID: <20071010193633.EC0848214@code0.codespeak.net> Author: tismer Date: Wed Oct 10 21:36:33 2007 New Revision: 47381 Modified: pypy/dist/pypy/rlib/rmmap.py Log: hacked quite a bit to make test_rmmap work for windows. But the mixture of rffi and ctypes seems to be not readily thought out yet. Want to talk to Maciei before hacking on. Modified: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/rmmap.py (original) +++ pypy/dist/pypy/rlib/rmmap.py Wed Oct 10 21:36:33 2007 @@ -25,7 +25,9 @@ self.message = message class CConfig: - _includes_ = ("sys/types.h",'unistd.h') + _includes_ = ["sys/types.h"] + if _POSIX: + _includes_.append('unistd.h') _header_ = '#define _GNU_SOURCE\n' size_t = rffi_platform.SimpleType("size_t", rffi.LONG) off_t = rffi_platform.SimpleType("off_t", rffi.LONG) @@ -80,11 +82,10 @@ PTR = rffi.CCHARP -has_mremap = cConfig['has_mremap'] - c_memmove = external('memmove', [PTR, PTR, size_t], lltype.Void) if _POSIX: + has_mremap = cConfig['has_mremap'] c_mmap = external('mmap', [PTR, size_t, rffi.INT, rffi.INT, rffi.INT, off_t], PTR) c_munmap = external('munmap', [PTR, size_t], rffi.INT) @@ -98,18 +99,20 @@ errno = rffi.get_errno() return os.strerror(errno) elif _MS_WINDOWS: - XXX - from ctypes import wintypes + from ctypes import wintypes, Union, Structure WORD = wintypes.WORD DWORD = wintypes.DWORD BOOL = wintypes.BOOL LONG = wintypes.LONG - LPVOID = PTR + # LPVOID = PTR does not work with ctypes + LPVOID = wintypes.c_void_p LPCVOID = LPVOID - DWORD_PTR = DWORD - rffi.INT = wintypes.rffi.INT - INVALID_c_int_VALUE = c_int(-1).value + DWORD_PTR = DWORD # ??? + INT = wintypes.c_int # there is no wintypes.INT + POINTER = wintypes.POINTER + INVALID_c_int_VALUE = wintypes.c_int(-1).value + windll = wintypes.windll class SYSINFO_STRUCT(Structure): _fields_ = [("wProcessorArchitecture", WORD), From fijal at codespeak.net Thu Oct 11 07:28:07 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 11 Oct 2007 07:28:07 +0200 (CEST) Subject: [pypy-svn] r47383 - pypy/dist/pypy/rlib Message-ID: <20071011052807.44559824D@code0.codespeak.net> Author: fijal Date: Thu Oct 11 07:28:05 2007 New Revision: 47383 Modified: pypy/dist/pypy/rlib/libffi.py Log: * expose RTLD_GLOBAL flag * use include_dirs also in llexternal Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Thu Oct 11 07:28:05 2007 @@ -19,6 +19,7 @@ _include_dirs_ = include_dirs RTLD_LOCAL = rffi_platform.DefinedConstantInteger('RTLD_LOCAL') + RTLD_GLOBAL = rffi_platform.DefinedConstantInteger('RTLD_GLOBAL') RTLD_NOW = rffi_platform.DefinedConstantInteger('RTLD_NOW') FFI_OK = rffi_platform.ConstantInteger('FFI_OK') @@ -80,6 +81,7 @@ def external(name, args, result, **kwds): return rffi.llexternal(name, args, result, includes=includes, + include_dirs=include_dirs, libraries=['dl', 'ffi'], **kwds) c_dlopen = external('dlopen', [rffi.CCHARP, rffi.INT], rffi.VOIDP, @@ -89,6 +91,7 @@ c_dlsym = external('dlsym', [rffi.VOIDP, rffi.CCHARP], rffi.VOIDP) RTLD_LOCAL = cConfig.RTLD_LOCAL +RTLD_GLOBAL = cConfig.RTLD_GLOBAL RTLD_NOW = cConfig.RTLD_NOW FFI_OK = cConfig.FFI_OK FFI_BAD_TYPEDEF = cConfig.FFI_BAD_TYPEDEF From fijal at codespeak.net Thu Oct 11 08:22:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 11 Oct 2007 08:22:41 +0200 (CEST) Subject: [pypy-svn] r47384 - in pypy/dist/pypy/module/ctypes: . test Message-ID: <20071011062241.7E3B8824B@code0.codespeak.net> Author: fijal Date: Thu Oct 11 08:22:39 2007 New Revision: 47384 Added: pypy/dist/pypy/module/ctypes/ (props changed) pypy/dist/pypy/module/ctypes/__init__.py (contents, props changed) pypy/dist/pypy/module/ctypes/interp_ctypes.py (contents, props changed) pypy/dist/pypy/module/ctypes/test/ (props changed) pypy/dist/pypy/module/ctypes/test/__init__.py (contents, props changed) pypy/dist/pypy/module/ctypes/test/test_ctypes.py (contents, props changed) Log: Add app-level version of _ctypes module. Very rudimentary by now. (but libc.rand works) Added: pypy/dist/pypy/module/ctypes/__init__.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/ctypes/__init__.py Thu Oct 11 08:22:39 2007 @@ -0,0 +1,19 @@ + +""" App-level ctypes module for pypy +""" + +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + applevel_name = '_ctypes' + + interpleveldefs = { + '_CDLL' : 'interp_ctypes.W_CDLL', + 'RTLD_LOCAL' : 'space.wrap(interp_ctypes.RTLD_LOCAL)', + 'RTLD_GLOBAL' : 'space.wrap(interp_ctypes.RTLD_GLOBAL)', + 'dlopen' : 'interp_ctypes.dlopen', + '_SimpleCData' : 'interp_ctypes.W_SimpleCData', + 'CFuncPtr' : 'interp_ctypes.W_CFuncPtr', + } + + appleveldefs = {} Added: pypy/dist/pypy/module/ctypes/interp_ctypes.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/ctypes/interp_ctypes.py Thu Oct 11 08:22:39 2007 @@ -0,0 +1,65 @@ + +from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable +from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.rlib.libffi import CDLL, RTLD_LOCAL, RTLD_GLOBAL,\ + ffi_type_sint +from pypy.rpython.lltypesystem import lltype, rffi + +DEFAULT_MODE = RTLD_LOCAL + +class W_CDLL(Wrappable): + def __init__(self, name, mode): + # XXX ignore mode for now + self.cdll = CDLL(name) + +class W_SimpleCData(Wrappable): + pass + +W_SimpleCData.typedef = TypeDef("_SimpleCData") + +class W_CFuncPtr(Wrappable): + def init(self, space, w_args): + w_name_or_ordinal = space.getitem(w_args, space.wrap(0)) + w_cdll = space.getitem(w_args, space.wrap(1)) + handle = space.getattr(w_cdll, space.wrap('_handle')) + self.space = space + # XXX + self.name = space.str_w(w_name_or_ordinal) + self.handle = handle.cdll.getpointer(self.name, [], ffi_type_sint) + init.unwrap_spec = ['self', ObjSpace, W_Root] + + def call(self, space): + # NO ARGS + return space.wrap(self.handle.call(rffi.INT)) + call.unwrap_spec = ['self', ObjSpace] + +def descr_new_cfuncptr(space, w_subtype, w_args): + return space.allocate_instance(W_CFuncPtr, w_subtype) +descr_new_cfuncptr.unwrap_spec = [ObjSpace, W_Root, W_Root] + +def descr_name(space, w_obj): + return space.wrap(space.type(w_obj).name) + +def descr_set_name(space, w_obj, w_to): + space.type(w_obj).name = space.str_w(w_to) + +W_CFuncPtr.typedef = TypeDef("CFuncPtr", + __new__ = interp2app(descr_new_cfuncptr), + __init__ = interp2app(W_CFuncPtr.init), + __name__ = GetSetProperty(descr_name, descr_set_name), + __call__ = interp2app(W_CFuncPtr.call), +) + +def dlopen(space, name, mode=DEFAULT_MODE): + try: + return space.wrap(W_CDLL(name, mode)) + except OSError, e: + raise wrap_oserror(space, e) +dlopen.unwrap_spec = [ObjSpace, str, int] + +# no public fields here +W_CDLL.typedef = TypeDef("_CDLL") + + Added: pypy/dist/pypy/module/ctypes/test/__init__.py ============================================================================== Added: pypy/dist/pypy/module/ctypes/test/test_ctypes.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/ctypes/test/test_ctypes.py Thu Oct 11 08:22:39 2007 @@ -0,0 +1,34 @@ + +from pypy.conftest import gettestobjspace + +import os, sys, py + +def setup_module(mod): + if sys.platform != 'linux2': + py.test.skip("Linux only tests by now") + +class AppTestCTypes: + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('_ctypes',)) + + def test_libload(self): + import ctypes + ctypes.CDLL('libc.so.6') + + def test_getattr(self): + import ctypes + libc = ctypes.CDLL('libc.so.6') + func = libc.rand + assert func.__class__ is libc.__class__._FuncPtr + assert isinstance(func, ctypes._CFuncPtr) + + def test_rand(self): + import ctypes + libc = ctypes.CDLL('libc.so.6') + first = libc.rand() + count = 0 + for i in range(100): + res = libc.rand() + if res == first: + count += 1 + assert count != 100 From fijal at codespeak.net Thu Oct 11 08:23:05 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 11 Oct 2007 08:23:05 +0200 (CEST) Subject: [pypy-svn] r47385 - pypy/dist/pypy/lib Message-ID: <20071011062305.7572C824B@code0.codespeak.net> Author: fijal Date: Thu Oct 11 08:23:05 2007 New Revision: 47385 Added: pypy/dist/pypy/lib/ctypes.py (contents, props changed) Log: Add ctypes lib (pure python part) Added: pypy/dist/pypy/lib/ctypes.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/ctypes.py Thu Oct 11 08:23:05 2007 @@ -0,0 +1,56 @@ + +""" App-level part of ctypes module +""" + +from _ctypes import dlopen, RTLD_LOCAL, RTLD_GLOBAL +from _ctypes import CFuncPtr as _CFuncPtr +from _ctypes import _SimpleCData + +# XXX check size, etc +class c_int(_SimpleCData): + _type_ = 'i' + +DEFAULT_MODE = RTLD_LOCAL + +class CDLL(object): + """An instance of this class represents a loaded dll/shared + library, exporting functions using the standard C calling + convention (named 'cdecl' on Windows). + + The exported functions can be accessed as attributes, or by + indexing with the function name. Examples: + + .qsort -> callable object + ['qsort'] -> callable object + + Calling the functions releases the Python GIL during the call and + reaquires it afterwards. + """ + class _FuncPtr(_CFuncPtr): + #_flags_ = _FUNCFLAG_CDECL + _restype_ = c_int # default, can be overridden in instances + + def __init__(self, name, mode=DEFAULT_MODE, handle=None): + self._name = name + if handle is None: + self._handle = dlopen(name, mode) + else: + self._handle = handle + + def __repr__(self): + return "<%s '%s'>" % (self.__class__.__name__, self._name) + + def __getattr__(self, name): + if name.startswith('__') and name.endswith('__'): + raise AttributeError, name + func = self.__getitem__(name) + setattr(self, name, func) + return func + + def __getitem__(self, name_or_ordinal): + # right now only name + assert isinstance(name_or_ordinal, (str, unicode)) + func = self._FuncPtr((name_or_ordinal, self)) + if not isinstance(name_or_ordinal, (int, long)): + func.__name__ = name_or_ordinal + return func From fijal at codespeak.net Thu Oct 11 09:28:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 11 Oct 2007 09:28:51 +0200 (CEST) Subject: [pypy-svn] r47386 - pypy/dist/pypy/interpreter Message-ID: <20071011072851.50ACB824C@code0.codespeak.net> Author: fijal Date: Thu Oct 11 09:28:50 2007 New Revision: 47386 Modified: pypy/dist/pypy/interpreter/gateway.py Log: More elaborate assertion Modified: pypy/dist/pypy/interpreter/gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/gateway.py (original) +++ pypy/dist/pypy/interpreter/gateway.py Thu Oct 11 09:28:50 2007 @@ -131,7 +131,7 @@ self.orig_arg() def visit__W_Root(self, el, app_sig): - assert el is W_Root, "oops" + assert el is W_Root, "%s is not W_Root (forgotten to put .im_func in interp2app argument?)" % (el,) argname = self.orig_arg() assert argname.startswith('w_'), ( "argument %s of built-in function %r should " From cfbolz at codespeak.net Thu Oct 11 10:53:03 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 11 Oct 2007 10:53:03 +0200 (CEST) Subject: [pypy-svn] r47388 - pypy/dist/pypy/module/ctypes Message-ID: <20071011085303.44846822C@code0.codespeak.net> Author: cfbolz Date: Thu Oct 11 10:53:03 2007 New Revision: 47388 Removed: pypy/dist/pypy/module/ctypes/ Log: (fijal): kill this attempt at a ctypes module, will start with a simple wrapper around libffi first. From cfbolz at codespeak.net Thu Oct 11 11:26:36 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 11 Oct 2007 11:26:36 +0200 (CEST) Subject: [pypy-svn] r47389 - in pypy/dist/pypy/translator/c: . test Message-ID: <20071011092636.3388F8240@code0.codespeak.net> Author: cfbolz Date: Thu Oct 11 11:26:34 2007 New Revision: 47389 Modified: pypy/dist/pypy/translator/c/database.py pypy/dist/pypy/translator/c/node.py pypy/dist/pypy/translator/c/test/test_newgc.py Log: hack differently: make sure that the two incarnations of a weakref object are both in the caches, but point to the same node. Modified: pypy/dist/pypy/translator/c/database.py ============================================================================== --- pypy/dist/pypy/translator/c/database.py (original) +++ pypy/dist/pypy/translator/c/database.py Thu Oct 11 11:26:34 2007 @@ -149,7 +149,7 @@ else: raise Exception("don't know about type %r" % (T,)) - def getcontainernode(self, container, _cached=True, **buildkwds): + def getcontainernode(self, container, _dont_write_c_code=True, **buildkwds): try: node = self.containernodes[container] except KeyError: @@ -159,13 +159,14 @@ self.gctransformer.consider_constant(T, container) nodefactory = ContainerNodeFactory[T.__class__] node = nodefactory(self, T, container, **buildkwds) - # _cached should only be False for a hack in weakrefnode_factory() - if not _cached: - return node self.containernodes[container] = node - self.containerlist.append(node) + # _dont_write_c_code should only be False for a hack in + # weakrefnode_factory() + if not _dont_write_c_code: + return node kind = getattr(node, 'nodekind', '?') self.containerstats[kind] = self.containerstats.get(kind, 0) + 1 + self.containerlist.append(node) if self.completed: assert not node.globalcontainer # non-global containers are found very late, e.g. _subarrays Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Thu Oct 11 11:26:34 2007 @@ -938,7 +938,7 @@ ptarget = obj._dereference() wrapper = db.gcpolicy.convert_weakref_to(ptarget) container = wrapper._obj - return db.getcontainernode(container, _cached=False) + return db.getcontainernode(container, _dont_write_c_code=False) ContainerNodeFactory = { Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Thu Oct 11 11:26:34 2007 @@ -827,9 +827,3 @@ class TestSemiSpaceGC(TestUsingFramework): frameworkgc = "semispace" should_be_moving = True - - def test_weakref(self): - py.test.skip("in-progress") - - def test_prebuilt_weakref(self): - py.test.skip("in-progress") From cfbolz at codespeak.net Thu Oct 11 11:33:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 11 Oct 2007 11:33:23 +0200 (CEST) Subject: [pypy-svn] r47390 - pypy/dist/pypy/rpython/memory Message-ID: <20071011093323.3E1F58254@code0.codespeak.net> Author: cfbolz Date: Thu Oct 11 11:33:22 2007 New Revision: 47390 Modified: pypy/dist/pypy/rpython/memory/gc.py Log: small optimization about weakrefs: don't add the weakref object to the list of weakrefs if it is invalidated Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Thu Oct 11 11:33:22 2007 @@ -1115,11 +1115,9 @@ if self.is_forwarded(pointing_to): (obj + offset).address[0] = self.get_forwarding_address( pointing_to) + new_with_weakref.append(obj) else: (obj + offset).address[0] = NULL - # XXX the next line can be indented? only do after weakref - # tests pass - new_with_weakref.append(obj) self.objects_with_weakrefs.delete() self.objects_with_weakrefs = new_with_weakref if self.run_finalizers.non_empty(): @@ -1177,7 +1175,6 @@ def trace_and_copy(self, obj): gc_info = self.header(obj) typeid = gc_info.typeid -## print "scanning", obj, typeid offsets = self.offsets_to_gc_pointers(typeid) i = 0 while i < len(offsets): From cfbolz at codespeak.net Thu Oct 11 11:48:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 11 Oct 2007 11:48:00 +0200 (CEST) Subject: [pypy-svn] r47391 - in pypy/dist/pypy/rpython/memory: . test Message-ID: <20071011094800.EBCCE8230@code0.codespeak.net> Author: cfbolz Date: Thu Oct 11 11:48:00 2007 New Revision: 47391 Modified: pypy/dist/pypy/rpython/memory/gc.py pypy/dist/pypy/rpython/memory/test/test_gc.py Log: kill deferred refcounting: it was broken and unmaintained. Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Thu Oct 11 11:48:00 2007 @@ -1250,134 +1250,6 @@ STATISTICS_NUMBERS = 0 -class DeferredRefcountingGC(GCBase): - _alloc_flavor_ = "raw" - - def __init__(self, AddressLinkedList, max_refcount_zero=50, get_roots=None): - self.zero_ref_counts = None - self.AddressLinkedList = AddressLinkedList - self.length_zero_ref_counts = 0 - self.max_refcount_zero = max_refcount_zero - #self.set_query_functions(None, None, None, None, None, None, None) - self.get_roots = get_roots - self.collecting = False - - def setup(self): - self.zero_ref_counts = self.AddressLinkedList() - - - def malloc(self, typeid, length=0): - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - size += length * self.varsize_item_sizes(typeid) - size_gc_header = self.size_gc_header() - result = raw_malloc(size + size_gc_header) -## print "mallocing %s, size %s at %s" % (typeid, size, result) - if not result: - raise memoryError - result.signed[0] = 0 # refcount - result.signed[1] = typeid - return result + size_gc_header - - def collect(self): - if self.collecting: - return - else: - self.collecting = True - roots = self.get_roots() - roots_copy = self.AddressLinkedList() - curr = roots.pop() - while curr != NULL: -## print "root", root, root.address[0] -## assert self.refcount(root.address[0]) >= 0, "refcount negative" - self.incref(curr.address[0]) - roots_copy.append(curr) - curr = roots.pop() - roots = roots_copy - dealloc_list = self.AddressLinkedList() - self.length_zero_ref_counts = 0 - while self.zero_ref_counts.non_empty(): - candidate = self.zero_ref_counts.pop() - refcount = self.refcount(candidate) - typeid = (candidate - self.size_gc_header()).signed[1] - if (refcount == 0 and typeid >= 0): - (candidate - self.size_gc_header()).signed[1] = -typeid - 1 - dealloc_list.append(candidate) - while dealloc_list.non_empty(): - deallocate = dealloc_list.pop() - typeid = (deallocate - self.size_gc_header()).signed[1] - (deallocate - self.size_gc_header()).signed[1] = -typeid - 1 - self.deallocate(deallocate) - dealloc_list.delete() - while roots.non_empty(): - root = roots.pop() - self.decref(root.address[0]) - roots.delete() - self.collecting = False - - def write_barrier(self, addr, addr_to, addr_struct): - self.decref(addr_to.address[0]) - addr_to.address[0] = addr - self.incref(addr) - - def deallocate(self, obj): - gc_info = obj - self.size_gc_header() - typeid = gc_info.signed[1] -## print "deallocating", obj, typeid - offsets = self.offsets_to_gc_pointers(typeid) - i = 0 - while i < len(offsets): - pointer = obj + offsets[i] - self.decref(pointer.address[0]) - i += 1 - if self.is_varsize(typeid): - offset = self.varsize_offset_to_variable_part( - typeid) - length = (obj + self.varsize_offset_to_length(typeid)).signed[0] - offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) - itemlength = self.varsize_item_sizes(typeid) - i = 0 - while i < length: - item = obj + offset + itemlength * i - j = 0 - while j < len(offsets): - pointer = item + offsets[j] - self.decref(pointer.address[0]) - j += 1 - i += 1 - raw_free(gc_info) - - def incref(self, addr): - if addr == NULL: - return - (addr - self.size_gc_header()).signed[0] += 1 - - def decref(self, addr): - if addr == NULL: - return - refcount = (addr - self.size_gc_header()).signed[0] -## assert refcount > 0, "neg refcount" - if refcount == 1: - self.zero_ref_counts.append(addr) - self.length_zero_ref_counts += 1 - if self.length_zero_ref_counts > self.max_refcount_zero: - self.collect() - (addr - self.size_gc_header()).signed[0] = refcount - 1 - - def refcount(self, addr): - return (addr - self.size_gc_header()).signed[0] - - def init_gc_object(self, addr, typeid): - addr.signed[0] = 0 # refcount - addr.signed[1] = typeid - - def init_gc_object_immortal(self, addr, typeid): - addr.signed[0] = sys.maxint // 2 # refcount - addr.signed[1] = typeid - - def size_gc_header(self, typeid=0): - XXX - # ____________________________________________________________ def choose_gc_from_config(config): Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Thu Oct 11 11:48:00 2007 @@ -260,8 +260,3 @@ class TestGrowingSemiSpaceGC(TestSemiSpaceGC): GC_PARAMS = {'space_size': 64} - -class TestDeferredRefcountingGC(GCTest): - from pypy.rpython.memory.gc import DeferredRefcountingGC as GCClass - def setup_class(cls): - py.test.skip("DeferredRefcounting is unmaintained") From cfbolz at codespeak.net Thu Oct 11 12:50:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 11 Oct 2007 12:50:27 +0200 (CEST) Subject: [pypy-svn] r47392 - pypy/dist/pypy/doc Message-ID: <20071011105027.2BFA48223@code0.codespeak.net> Author: cfbolz Date: Thu Oct 11 12:50:25 2007 New Revision: 47392 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: maintain the cleanup list a bit. while keepalives still haven't died, they are rather harmless now, so remove this task from the list. Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Thu Oct 11 12:50:25 2007 @@ -11,7 +11,6 @@ - low level backends should share more code - all backends should have more consistent interfaces - geninterp is a hack - - keepalives need to die, finally - delegate finding type stuff like vtables etc to GC, cleaner interface for rtti, simplify translator/c/gc.py - clean up the tangle of including headers in the C backend @@ -23,7 +22,6 @@ timeshifter) - unicode strings in RPython - finish rctypes removal - - move genrffi to not use rctypes stuff any more - think about approaches to id, especially concerning boehm, where the id will keep the object alive and concerning a moving GC From cfbolz at codespeak.net Thu Oct 11 13:45:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 11 Oct 2007 13:45:28 +0200 (CEST) Subject: [pypy-svn] r47393 - pypy/dist/pypy/rpython/memory Message-ID: <20071011114528.AEBB7823C@code0.codespeak.net> Author: cfbolz Date: Thu Oct 11 13:45:27 2007 New Revision: 47393 Modified: pypy/dist/pypy/rpython/memory/gc.py Log: refactor SemiSpace.collectinto a bunch of small methods. makes it easier to understand (and easier to profile). Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Thu Oct 11 13:45:27 2007 @@ -1091,57 +1091,16 @@ self.fromspace = fromspace self.tospace = tospace self.top_of_space = tospace + self.space_size - roots = self.get_roots() scan = self.free = tospace - while 1: - root = roots.pop() - if root == NULL: - break - root.address[0] = self.copy(root.address[0]) - free_non_gc_object(roots) + self.collect_roots() scan = self.scan_copied(scan) - # walk over list of objects that contain weakrefs - # if the object it references survives then update the weakref - # otherwise invalidate the weakref - new_with_weakref = self.AddressLinkedList() - while self.objects_with_weakrefs.non_empty(): - obj = self.objects_with_weakrefs.pop() - if not self.is_forwarded(obj): - continue # weakref itself dies - obj = self.get_forwarding_address(obj) - offset = self.weakpointer_offset(self.header(obj).typeid) - pointing_to = (obj + offset).address[0] - if pointing_to: - if self.is_forwarded(pointing_to): - (obj + offset).address[0] = self.get_forwarding_address( - pointing_to) - new_with_weakref.append(obj) - else: - (obj + offset).address[0] = NULL - self.objects_with_weakrefs.delete() - self.objects_with_weakrefs = new_with_weakref + if self.objects_with_weakrefs.non_empty(): + self.invalidate_weakrefs() if self.run_finalizers.non_empty(): - # we are in an inner collection, caused by a finalizer - # the run_finalizers objects need to be copied - new_run_finalizer = self.AddressLinkedList() - while self.run_finalizers.non_empty(): - obj = self.run_finalizers.pop() - new_run_finalizer.append(self.copy(obj)) - self.run_finalizers.delete() - self.run_finalizers = new_run_finalizer - # walk over list of objects with finalizers - # if it is not copied, add it to the list of to-be-called finalizers - # and copy it, to me make the finalizer runnable - new_with_finalizer = self.AddressLinkedList() - while self.objects_with_finalizers.non_empty(): - obj = self.objects_with_finalizers.pop() - if self.is_forwarded(obj): - new_with_finalizer.append(self.get_forwarding_address(obj)) - else: - self.run_finalizers.append(self.copy(obj)) + self.update_run_finalizers() + if self.objects_with_finalizers.non_empty(): + self.deal_with_objects_with_finalizers() scan = self.scan_copied(scan) - self.objects_with_finalizers.delete() - self.objects_with_finalizers = new_with_finalizer if not size_changing: llarena.arena_reset(fromspace, self.space_size, True) self.execute_finalizers() @@ -1153,6 +1112,15 @@ scan += self.size_gc_header() + self.get_size(curr) return scan + def collect_roots(self): + roots = self.get_roots() + while 1: + root = roots.pop() + if root == NULL: + break + root.address[0] = self.copy(root.address[0]) + free_non_gc_object(roots) + def copy(self, obj): # Objects not living the GC heap have all been initialized by # setting their 'forw' address so that it points to themselves. @@ -1234,6 +1202,52 @@ hdr.forw = addr + self.gcheaderbuilder.size_gc_header hdr.typeid = typeid + def deal_with_objects_with_finalizers(self): + # walk over list of objects with finalizers + # if it is not copied, add it to the list of to-be-called finalizers + # and copy it, to me make the finalizer runnable + new_with_finalizer = self.AddressLinkedList() + while self.objects_with_finalizers.non_empty(): + obj = self.objects_with_finalizers.pop() + if self.is_forwarded(obj): + new_with_finalizer.append(self.get_forwarding_address(obj)) + else: + self.run_finalizers.append(self.copy(obj)) + self.objects_with_finalizers.delete() + self.objects_with_finalizers = new_with_finalizer + + def invalidate_weakrefs(self): + # walk over list of objects that contain weakrefs + # if the object it references survives then update the weakref + # otherwise invalidate the weakref + new_with_weakref = self.AddressLinkedList() + while self.objects_with_weakrefs.non_empty(): + obj = self.objects_with_weakrefs.pop() + if not self.is_forwarded(obj): + continue # weakref itself dies + obj = self.get_forwarding_address(obj) + offset = self.weakpointer_offset(self.header(obj).typeid) + pointing_to = (obj + offset).address[0] + if pointing_to: + if self.is_forwarded(pointing_to): + (obj + offset).address[0] = self.get_forwarding_address( + pointing_to) + new_with_weakref.append(obj) + else: + (obj + offset).address[0] = NULL + self.objects_with_weakrefs.delete() + self.objects_with_weakrefs = new_with_weakref + + def update_run_finalizers(self): + # we are in an inner collection, caused by a finalizer + # the run_finalizers objects need to be copied + new_run_finalizer = self.AddressLinkedList() + while self.run_finalizers.non_empty(): + obj = self.run_finalizers.pop() + new_run_finalizer.append(self.copy(obj)) + self.run_finalizers.delete() + self.run_finalizers = new_run_finalizer + def execute_finalizers(self): if self.executing_finalizers: return # the outer invocation of execute_finalizers() will do it From fijal at codespeak.net Thu Oct 11 14:34:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 11 Oct 2007 14:34:04 +0200 (CEST) Subject: [pypy-svn] r47394 - in pypy/dist/pypy/module/_ffi: . test Message-ID: <20071011123404.D0D7A8250@code0.codespeak.net> Author: fijal Date: Thu Oct 11 14:34:03 2007 New Revision: 47394 Added: pypy/dist/pypy/module/_ffi/ (props changed) pypy/dist/pypy/module/_ffi/__init__.py (contents, props changed) pypy/dist/pypy/module/_ffi/app_ffi.py (contents, props changed) pypy/dist/pypy/module/_ffi/interp_ffi.py (contents, props changed) pypy/dist/pypy/module/_ffi/test/ (props changed) pypy/dist/pypy/module/_ffi/test/__init__.py (contents, props changed) pypy/dist/pypy/module/_ffi/test/test__ffi.py (contents, props changed) Log: Expose libffi to applevel with a very basic interface Added: pypy/dist/pypy/module/_ffi/__init__.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/_ffi/__init__.py Thu Oct 11 14:34:03 2007 @@ -0,0 +1,18 @@ + +""" Low-level interface to libffi +""" + +from pypy.interpreter.mixedmodule import MixedModule +from pypy.module._ffi.interp_ffi import W_CDLL +from pypy.rpython.lltypesystem import lltype, rffi + +class Module(MixedModule): + applevelname = '_ffi' + + interpleveldefs = { + 'CDLL' : 'interp_ffi.W_CDLL', + 'FuncPtr' : 'interp_ffi.W_FuncPtr', + } + + appleveldefs = { + } Added: pypy/dist/pypy/module/_ffi/app_ffi.py ============================================================================== Added: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Thu Oct 11 14:34:03 2007 @@ -0,0 +1,107 @@ + +from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable, \ + Arguments +from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.typedef import TypeDef, GetSetProperty + +from pypy.rlib.libffi import CDLL, RTLD_LOCAL, RTLD_GLOBAL,\ + ffi_type_sint, ffi_type_double, ffi_type_slong +from pypy.rpython.lltypesystem import lltype, rffi + +typemap = { + 'i' : ffi_type_sint, + 'l' : ffi_type_slong, + 'd' : ffi_type_double +} + +class W_CDLL(Wrappable): + def __init__(self, space, name): + self.cdll = CDLL(name) + self.name = name + self.w_cache = space.newdict() + self.space = space + + def get_type(self, key): + space = self.space + try: + return typemap[key] + except KeyError: + raise OperationError(space.w_ValueError, space.wrap( + "Uknown type letter %s" % key)) + + def ptr(self, space, name, w_argtypes, restype): + """ Get a pointer for function name with provided argtypes + and restype + """ + w = space.wrap + w_argtypes = space.newtuple(space.unpackiterable(w_argtypes)) + w_key = space.newtuple([w(name), w_argtypes, w(restype)]) + try: + return space.getitem(self.w_cache, w_key) + except OperationError, e: + if e.match(space, space.w_KeyError): + pass + else: + raise + argtypes_w = space.unpackiterable(w_argtypes) + argtypes = [space.str_w(w_arg) for w_arg in argtypes_w] + ffi_argtypes = [self.get_type(arg) for arg in argtypes] + ffi_restype = self.get_type(restype) + try: + ptr = self.cdll.getpointer(name, ffi_argtypes, ffi_restype) + w_funcptr = W_FuncPtr(ptr, argtypes, restype) + space.setitem(self.w_cache, w_key, w_funcptr) + return w_funcptr + except KeyError: + raise OperationError(space.w_AttributeError, space.wrap( + "No symbol %s found in library %s" % (name, self.name))) + ptr.unwrap_spec = ['self', ObjSpace, str, W_Root, str] + +def descr_new_cdll(space, w_type, name): + try: + return space.wrap(W_CDLL(space, name)) + except OSError, e: + raise wrap_oserror(space, e) +descr_new_cdll.unwrap_spec = [ObjSpace, W_Root, str] + +W_CDLL.typedef = TypeDef( + 'CDLL', + __new__ = interp2app(descr_new_cdll), + ptr = interp2app(W_CDLL.ptr) +) + +def push_arg(space, ptr, argtype, w_arg): + if argtype == "i": + ptr.push_arg(space.int_w(w_arg)) + elif argtype == "d": + ptr.push_arg(space.float_w(w_arg)) + else: + raise TypeError("Stuff changed in between?") + +class W_FuncPtr(Wrappable): + def __init__(self, ptr, argtypes, restype): + self.ptr = ptr + self.restype = restype + self.argtypes = argtypes + + def call(self, space, arguments): + args_w, kwds_w = arguments.unpack() + # C has no keyword arguments + if kwds_w: + raise OperationError(space.w_TypeError, space.wrap( + "Provided keyword arguments for C function call")) + for argtype, w_arg in zip(self.argtypes, args_w): + push_arg(space, self.ptr, argtype, w_arg) + # a bit of specialcasing, rpython trick instead? + if self.restype == "i": + return space.wrap(self.ptr.call(rffi.INT)) + elif self.restype == "d": + return space.wrap(self.ptr.call(rffi.DOUBLE)) + raise TypeError("Stuff changed in between?") + call.unwrap_spec = ['self', ObjSpace, Arguments] + +W_FuncPtr.typedef = TypeDef( + 'FuncPtr', + __call__ = interp2app(W_FuncPtr.call) +) Added: pypy/dist/pypy/module/_ffi/test/__init__.py ============================================================================== Added: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Thu Oct 11 14:34:03 2007 @@ -0,0 +1,49 @@ + + +from pypy.conftest import gettestobjspace + +import os, sys, py + +def setup_module(mod): + if sys.platform != 'linux2': + py.test.skip("Linux only tests by now") + +class AppTestCTypes: + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('_ffi',)) + + def test_libload(self): + import _ffi + _ffi.CDLL('libc.so.6') + + def test_getattr(self): + import _ffi + libc = _ffi.CDLL('libc.so.6') + func = libc.ptr('rand', [], 'i') + assert libc.ptr('rand', [], 'i') is func # caching + assert libc.ptr('rand', [], 'l') is not func + assert isinstance(func, _ffi.FuncPtr) + raises(AttributeError, "libc.xxxxxxxxxxxxxx") + + def test_rand(self): + import _ffi + libc = _ffi.CDLL('libc.so.6') + func = libc.ptr('rand', [], 'i') + first = func() + count = 0 + for i in range(100): + res = func() + if res == first: + count += 1 + assert count != 100 + + def test_pow(self): + import _ffi + libm = _ffi.CDLL('libm.so') + pow = libm.ptr('pow', ['d', 'd'], 'd') + assert pow(2.0, 2.0) == 4.0 + assert pow(3.0, 3.0) == 27.0 + assert pow(2, 2) == 4.0 + raises(TypeError, "pow('x', 2.0)") + + From fijal at codespeak.net Thu Oct 11 14:34:45 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 11 Oct 2007 14:34:45 +0200 (CEST) Subject: [pypy-svn] r47395 - pypy/dist/pypy/lib Message-ID: <20071011123445.429498236@code0.codespeak.net> Author: fijal Date: Thu Oct 11 14:34:44 2007 New Revision: 47395 Removed: pypy/dist/pypy/lib/ctypes.py Log: Delete ctypes.py from here From pypy-svn at codespeak.net Thu Oct 11 14:35:25 2007 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Thu, 11 Oct 2007 14:35:25 +0200 (CEST) Subject: [pypy-svn] NEW: Limited Time Only Message-ID: <75318313836.4746469778209@delivery.net> An HTML attachment was scrubbed... URL: From antocuni at codespeak.net Fri Oct 12 10:26:50 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 12 Oct 2007 10:26:50 +0200 (CEST) Subject: [pypy-svn] r47406 - in pypy/dist/pypy/translator: jvm/test oosupport/test_template Message-ID: <20071012082650.5D2B280C0@code0.codespeak.net> Author: antocuni Date: Fri Oct 12 10:26:49 2007 New Revision: 47406 Modified: pypy/dist/pypy/translator/jvm/test/test_op.py pypy/dist/pypy/translator/jvm/test/test_runtest.py pypy/dist/pypy/translator/oosupport/test_template/operations.py pypy/dist/pypy/translator/oosupport/test_template/runtest.py Log: add r_ulonglong tests that fails with genjvm Modified: pypy/dist/pypy/translator/jvm/test/test_op.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_op.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_op.py Fri Oct 12 10:26:49 2007 @@ -5,5 +5,5 @@ # ====> ../../oosupport/test_template/operations.py class TestOperations(JvmTest, BaseTestOperations): - pass - + def test_compare_big_ulongs(self): + py.test.skip('fixme!') Modified: pypy/dist/pypy/translator/jvm/test/test_runtest.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_runtest.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_runtest.py Fri Oct 12 10:26:49 2007 @@ -2,4 +2,6 @@ from pypy.translator.jvm.test.runtest import JvmTest class TestRunTest(BaseTestRunTest, JvmTest): - pass + def test_big_ullong(self): + import py + py.test.skip('fixme!') Modified: pypy/dist/pypy/translator/oosupport/test_template/operations.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/operations.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/operations.py Fri Oct 12 10:26:49 2007 @@ -205,3 +205,15 @@ x = sys.maxint+1 assert self.interpret(f, [r_ulonglong(x)]) == x >> 1 + def test_compare_big_ulongs(self): + bigval = r_ulonglong(9223372036854775808L) + def fn(x): + if x > bigval: return 1 + if x == bigval: return 0 + if x < bigval: return -1 + return -2 + + for val in (bigval-1, bigval, bigval+1): + expected = fn(val) + res = self.interpret(fn, [val]) + assert res == expected Modified: pypy/dist/pypy/translator/oosupport/test_template/runtest.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/runtest.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/runtest.py Fri Oct 12 10:26:49 2007 @@ -51,6 +51,10 @@ def test_ullong(self): assert self.interpret(ident, [r_ulonglong(sys.maxint+1)]) == sys.maxint+1 + def test_big_ullong(self): + bigval = 9223372036854775807L + assert self.interpret(ident, [r_ulonglong(bigval)]) == bigval + def test_exception(self): def fn(): raise ValueError From antocuni at codespeak.net Fri Oct 12 10:40:39 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 12 Oct 2007 10:40:39 +0200 (CEST) Subject: [pypy-svn] r47407 - in pypy/dist/pypy/translator: jvm/test oosupport/test_template Message-ID: <20071012084039.295EC825E@code0.codespeak.net> Author: antocuni Date: Fri Oct 12 10:40:38 2007 New Revision: 47407 Modified: pypy/dist/pypy/translator/jvm/test/test_op.py pypy/dist/pypy/translator/oosupport/test_template/operations.py Log: fix the name of the test, it was a bit misleading. Modified: pypy/dist/pypy/translator/jvm/test/test_op.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_op.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_op.py Fri Oct 12 10:40:38 2007 @@ -5,5 +5,5 @@ # ====> ../../oosupport/test_template/operations.py class TestOperations(JvmTest, BaseTestOperations): - def test_compare_big_ulongs(self): + def test_compare_big_ullongs(self): py.test.skip('fixme!') Modified: pypy/dist/pypy/translator/oosupport/test_template/operations.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/operations.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/operations.py Fri Oct 12 10:40:38 2007 @@ -205,7 +205,7 @@ x = sys.maxint+1 assert self.interpret(f, [r_ulonglong(x)]) == x >> 1 - def test_compare_big_ulongs(self): + def test_compare_big_ullongs(self): bigval = r_ulonglong(9223372036854775808L) def fn(x): if x > bigval: return 1 From arigo at codespeak.net Fri Oct 12 19:14:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 12 Oct 2007 19:14:22 +0200 (CEST) Subject: [pypy-svn] r47410 - in pypy/dist/pypy: rlib rpython rpython/lltypesystem rpython/test translator/c translator/c/src Message-ID: <20071012171422.BA30F81DD@code0.codespeak.net> Author: arigo Date: Fri Oct 12 19:14:20 2007 New Revision: 47410 Modified: pypy/dist/pypy/rlib/objectmodel.py pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/test/test_rbuiltin.py pypy/dist/pypy/translator/c/funcgen.py pypy/dist/pypy/translator/c/src/support.h Log: Introduce a symbolic and an operation: running_on_llinterp and debug_llinterpcall. Together, they allow flow graphs to do completely different things on top of the llinterp and in a real backend. Modified: pypy/dist/pypy/rlib/objectmodel.py ============================================================================== --- pypy/dist/pypy/rlib/objectmodel.py (original) +++ pypy/dist/pypy/rlib/objectmodel.py Fri Oct 12 19:14:20 2007 @@ -92,6 +92,8 @@ return lltype.Signed malloc_zero_filled = CDefinedIntSymbolic('MALLOC_ZERO_FILLED', default=0) +running_on_llinterp = CDefinedIntSymbolic('RUNNING_ON_LLINTERP', default=1) +# running_on_llinterp is meant to have the value 0 in all backends # ____________________________________________________________ @@ -145,6 +147,35 @@ hop.genop('debug_assert', vlist) +def debug_llinterpcall(RESTYPE, pythonfunction, *args): + """When running on the llinterp, this causes the llinterp to call to + the provided Python function with the run-time value of the given args. + The Python function should return a low-level object of type RESTYPE. + This should never be called after translation: use this only if + running_on_llinterp is true. + """ + raise NotImplementedError + +class Entry(ExtRegistryEntry): + _about_ = debug_llinterpcall + + def compute_result_annotation(self, s_RESTYPE, s_pythonfunction, *args_s): + from pypy.annotation import model as annmodel + assert s_RESTYPE.is_constant() + assert s_pythonfunction.is_constant() + return annmodel.lltype_to_annotation(s_RESTYPE.const) + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + RESTYPE= hop.args_s[0].const + pythonfunction = hop.args_s[1].const + c_pythonfunction = hop.inputconst(lltype.Void, pythonfunction) + args_v = [hop.inputarg(hop.args_r[i], arg=i) + for i in range(2, hop.nb_args)] + return hop.genop('debug_llinterpcall', [c_pythonfunction] + args_v, + resulttype=RESTYPE) + + def hlinvoke(repr, llcallable, *args): raise TypeError, "hlinvoke is meant to be rtyped and not called direclty" Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Fri Oct 12 19:14:20 2007 @@ -507,6 +507,9 @@ ll_exc_type = lltype.cast_pointer(rclass.OBJECTPTR, ll_exc).typeptr raise LLFatalError(msg, LLException(ll_exc_type, ll_exc)) + def op_debug_llinterpcall(self, pythonfunction, *args_ll): + return pythonfunction(*args_ll) + def op_instrument_count(self, ll_tag, ll_label): pass # xxx for now Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Fri Oct 12 19:14:20 2007 @@ -440,6 +440,8 @@ 'debug_pdb': LLOp(), 'debug_assert': LLOp(tryfold=True), 'debug_fatalerror': LLOp(), + 'debug_llinterpcall': LLOp(), # Python func call 'res=arg[0](*arg[1:])' + # in backends, abort() or whatever is fine # __________ instrumentation _________ 'instrument_count': LLOp(), Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/rpython/test/test_rbuiltin.py Fri Oct 12 19:14:20 2007 @@ -1,6 +1,7 @@ from pypy.translator.translator import graphof from pypy.rpython.test import test_llinterp from pypy.rlib.objectmodel import instantiate, we_are_translated +from pypy.rlib.objectmodel import running_on_llinterp, debug_llinterpcall from pypy.rpython.lltypesystem import lltype from pypy.tool import udir from pypy.rlib.rarithmetic import r_uint, intmask @@ -401,6 +402,26 @@ res = self.interpret(fn, [3.25]) assert res == 7.25 + def test_debug_llinterpcall(self): + S = lltype.Struct('S', ('m', lltype.Signed)) + SPTR = lltype.Ptr(S) + def foo(n): + "NOT_RPYTHON" + s = lltype.malloc(S, immortal=True) + s.m = eval("n*6", locals()) + return s + def fn(n): + if running_on_llinterp: + return debug_llinterpcall(SPTR, foo, n).m + else: + return 321 + res = self.interpret(fn, [7]) + assert res == 42 + from pypy.translator.c.test.test_genc import compile + f = compile(fn, [int]) + res = f(7) + assert res == 321 + class TestLLtype(BaseTestRbuiltin, LLRtypeMixin): def test_isinstance_obj(self): Modified: pypy/dist/pypy/translator/c/funcgen.py ============================================================================== --- pypy/dist/pypy/translator/c/funcgen.py (original) +++ pypy/dist/pypy/translator/c/funcgen.py Fri Oct 12 19:14:20 2007 @@ -712,6 +712,9 @@ return 'fprintf(stderr, "%%s\\n", %s); abort();' % msg + def OP_DEBUG_LLINTERPCALL(self, op): + return 'abort(); /* debug_llinterpcall should be unreachable */' + def OP_INSTRUMENT_COUNT(self, op): counter_label = op.args[1].value self.db.instrument_ncounter = max(self.db.instrument_ncounter, Modified: pypy/dist/pypy/translator/c/src/support.h ============================================================================== --- pypy/dist/pypy/translator/c/src/support.h (original) +++ pypy/dist/pypy/translator/c/src/support.h Fri Oct 12 19:14:20 2007 @@ -9,6 +9,8 @@ #define MIN(a,b) (((a)<(b))?(a):(b)) #endif /* MIN */ +#define RUNNING_ON_LLINTERP 0 + #define FAIL_EXCEPTION(exc, msg) \ { \ RPyRaiseSimpleException(exc, msg); \ From arigo at codespeak.net Fri Oct 12 19:19:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 12 Oct 2007 19:19:27 +0200 (CEST) Subject: [pypy-svn] r47411 - pypy/dist/pypy/rpython Message-ID: <20071012171927.2E44D81DF@code0.codespeak.net> Author: arigo Date: Fri Oct 12 19:19:26 2007 New Revision: 47411 Modified: pypy/dist/pypy/rpython/extfunc.py Log: Use the concepts introduced in the previous check-in to get rid of the abuse of suggested_primitive in extfunc.py for cases where we have both llimpl and llfakeimpl. Modified: pypy/dist/pypy/rpython/extfunc.py ============================================================================== --- pypy/dist/pypy/rpython/extfunc.py (original) +++ pypy/dist/pypy/rpython/extfunc.py Fri Oct 12 19:19:26 2007 @@ -182,6 +182,19 @@ impl = getattr(self, method_name, None) fakeimpl = getattr(self, fake_method_name, self.instance) if impl: + if hasattr(self, fake_method_name): + # If we have both an {ll,oo}impl and a {ll,oo}fakeimpl, + # we need a wrapper that selects the proper one and calls it + from pypy.rlib.objectmodel import running_on_llinterp + from pypy.rlib.objectmodel import debug_llinterpcall + from pypy.tool.sourcetools import func_with_new_name + original_impl = impl + def ll_wrapper(*args): + if running_on_llinterp: + return debug_llinterpcall(ll_result, fakeimpl, *args) + else: + return original_impl(*args) + impl = func_with_new_name(ll_wrapper, name + '_wrapper') if rtyper.annotator.translator.config.translation.sandbox: impl.dont_inline = True # store some attributes to the 'impl' function, where @@ -191,12 +204,6 @@ '_name': self.name, '_safe_not_sandboxed': self.safe_not_sandboxed, } - if hasattr(self, fake_method_name): - # we use the suggested_primitive flag to ask the llinterp - # to call the fakeimpl directly. It also disables inlining - # and other optimizations that would remove the call. - impl._llfnobjattrs_['_callable'] = fakeimpl - fakeimpl.suggested_primitive = True obj = rtyper.getannmixlevel().delayedfunction( impl, signature_args, hop.s_result) else: From simonb at codespeak.net Fri Oct 12 20:05:10 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Fri, 12 Oct 2007 20:05:10 +0200 (CEST) Subject: [pypy-svn] r47412 - in pypy/dist/pypy/translator/c: . test Message-ID: <20071012180510.5306D8200@code0.codespeak.net> Author: simonb Date: Fri Oct 12 20:05:09 2007 New Revision: 47412 Modified: pypy/dist/pypy/translator/c/support.py pypy/dist/pypy/translator/c/test/test_typed.py Log: simonb, Amaury: fix comparison. In gen_assignments(), variable names are compared by reference, but source and destination names are are constructed in different ways. Modified: pypy/dist/pypy/translator/c/support.py ============================================================================== --- pypy/dist/pypy/translator/c/support.py (original) +++ pypy/dist/pypy/translator/c/support.py Fri Oct 12 20:05:09 2007 @@ -205,7 +205,7 @@ startingpoint = dst tmpdecl = cdecl(typename, 'tmp') code = ['{ %s = %s;' % (tmpdecl, dst)] - while src is not startingpoint: + while src != startingpoint: code.append('%s = %s;' % (dst, src)) dst = src src, typename = dest2src.pop(dst) Modified: pypy/dist/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_typed.py (original) +++ pypy/dist/pypy/translator/c/test/test_typed.py Fri Oct 12 20:05:09 2007 @@ -772,7 +772,6 @@ assert res == expected def test_swap(self): - py.test.skip("Failing test. ") def func_swap(): a = [] b = range(10) From arigo at codespeak.net Fri Oct 12 20:07:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 12 Oct 2007 20:07:06 +0200 (CEST) Subject: [pypy-svn] r47413 - pypy/dist/pypy/translator/c Message-ID: <20071012180706.D2FC3820B@code0.codespeak.net> Author: arigo Date: Fri Oct 12 20:07:04 2007 New Revision: 47413 Modified: pypy/dist/pypy/translator/c/node.py Log: Update comment. Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Fri Oct 12 20:07:04 2007 @@ -536,8 +536,11 @@ for name in self.T._names: data.append((name, getattr(self.obj, name))) - # You can only initialise the first field of a union in c - # XXX what if later fields have some initialisation? + # Reasonably, you should only initialise one of the fields of a union + # in C. This is possible with the syntax '.fieldname value' or + # '.fieldname = value'. But here we don't know which of the + # fields need initialization, so XXX we pick the first one + # arbitrarily. if hasattr(self.T, "_hints") and self.T._hints.get('union'): data = data[0:1] From arigo at codespeak.net Fri Oct 12 20:57:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 12 Oct 2007 20:57:51 +0200 (CEST) Subject: [pypy-svn] r47414 - in pypy/dist/pypy/rpython/memory: . gctransform Message-ID: <20071012185751.D209281D7@code0.codespeak.net> Author: arigo Date: Fri Oct 12 20:57:51 2007 New Revision: 47414 Modified: pypy/dist/pypy/rpython/memory/gc.py pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: * allow the framework to inline a version of malloc_fixedsize for the most common case. * tweak SemiSpaceGC so that its malloc_fixedsize contains the minimum amount of operations after such an inlining. Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Fri Oct 12 20:57:51 2007 @@ -951,6 +951,7 @@ class SemiSpaceGC(GCBase): _alloc_flavor_ = "raw" moving_gc = True + inline_simple_malloc = True HDR = lltype.Struct('header', ('forw', llmemory.Address), ('typeid', lltype.Signed)) @@ -980,13 +981,14 @@ contains_weakptr=False): size_gc_header = self.gcheaderbuilder.size_gc_header totalsize = size_gc_header + size - if raw_malloc_usage(totalsize) > self.top_of_space - self.free: - if not can_collect or not self.obtain_free_space(totalsize): - raise memoryError result = self.free + if raw_malloc_usage(totalsize) > self.top_of_space - result: + if not can_collect: + raise memoryError + result = self.obtain_free_space(totalsize) llarena.arena_reserve(result, totalsize) self.init_gc_object(result, typeid) - self.free += totalsize + self.free = result + totalsize if has_finalizer: self.objects_with_finalizers.append(result + size_gc_header) if contains_weakptr: @@ -1002,14 +1004,15 @@ totalsize = ovfcheck(nonvarsize + varsize) except OverflowError: raise memoryError - if raw_malloc_usage(totalsize) > self.top_of_space - self.free: - if not can_collect or not self.obtain_free_space(totalsize): - raise memoryError result = self.free + if raw_malloc_usage(totalsize) > self.top_of_space - result: + if not can_collect: + raise memoryError + result = self.obtain_free_space(totalsize) llarena.arena_reserve(result, totalsize) self.init_gc_object(result, typeid) (result + size_gc_header + offset_to_length).signed[0] = length - self.free += llarena.round_up_for_allocation(totalsize) + self.free = result + llarena.round_up_for_allocation(totalsize) if has_finalizer: self.objects_with_finalizers.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) @@ -1019,6 +1022,14 @@ malloc_varsize_clear = malloc_varsize def obtain_free_space(self, needed): + # a bit of tweaking to maximize the performance and minimize the + # amount of code in an inlined version of malloc_fixedsize() + if not self.try_obtain_free_space(needed): + raise memoryError + return self.free + obtain_free_space.dont_inline = True + + def try_obtain_free_space(self, needed): # XXX for bonus points do big objects differently needed = raw_malloc_usage(needed) self.semispace_collect() Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Fri Oct 12 20:57:51 2007 @@ -13,6 +13,7 @@ from pypy.rpython.rbuiltin import gen_cast from pypy.rpython.memory.gctypelayout import ll_weakref_deref, WEAKREF from pypy.rpython.memory.gctypelayout import convert_weakref_to, WEAKREFPTR +from pypy.tool.sourcetools import func_with_new_name import sys @@ -216,6 +217,26 @@ self.collect_ptr = getfn(GCClass.collect.im_func, [s_gc], annmodel.s_None) + # in some GCs we can inline the common case of + # malloc_fixedsize(typeid, size, True, False, False) + if getattr(GCClass, 'inline_simple_malloc', False): + # make a copy of this function so that it gets annotated + # independently and the constants are folded inside + malloc_fast = func_with_new_name( + GCClass.malloc_fixedsize.im_func, + "malloc_fast") + s_False = annmodel.SomeBool(); s_False.const = False + s_True = annmodel.SomeBool(); s_True .const = True + self.malloc_fast_ptr = getfn( + malloc_fast, + [s_gc, annmodel.SomeInteger(nonneg=True), + annmodel.SomeInteger(nonneg=True), + s_True, s_False, + s_False], s_gcref, + inline = True) + else: + self.malloc_fast_ptr = self.malloc_fixedsize_ptr + self.statistics_ptr = getfn(GCClass.statistics.im_func, [s_gc, annmodel.SomeInteger()], annmodel.SomeInteger()) @@ -396,6 +417,8 @@ zero = flags.get('zero', False) if zero: malloc_ptr = self.malloc_fixedsize_clear_ptr + elif c_can_collect.value and not c_has_finalizer.value: + malloc_ptr = self.malloc_fast_ptr else: malloc_ptr = self.malloc_fixedsize_ptr args = [self.c_const_gc, c_type_id, c_size, c_can_collect, From arigo at codespeak.net Sat Oct 13 09:58:39 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 13 Oct 2007 09:58:39 +0200 (CEST) Subject: [pypy-svn] r47416 - in pypy/dist/pypy: rlib rpython Message-ID: <20071013075839.1C22C8288@code0.codespeak.net> Author: arigo Date: Sat Oct 13 09:58:37 2007 New Revision: 47416 Modified: pypy/dist/pypy/rlib/objectmodel.py pypy/dist/pypy/rpython/extfunc.py Log: Annotation fix. Modified: pypy/dist/pypy/rlib/objectmodel.py ============================================================================== --- pypy/dist/pypy/rlib/objectmodel.py (original) +++ pypy/dist/pypy/rlib/objectmodel.py Sat Oct 13 09:58:37 2007 @@ -161,13 +161,23 @@ def compute_result_annotation(self, s_RESTYPE, s_pythonfunction, *args_s): from pypy.annotation import model as annmodel + from pypy.rpython.lltypesystem import lltype assert s_RESTYPE.is_constant() assert s_pythonfunction.is_constant() - return annmodel.lltype_to_annotation(s_RESTYPE.const) + s_result = s_RESTYPE.const + if isinstance(s_result, lltype.LowLevelType): + s_result = annmodel.lltype_to_annotation(s_result) + assert isinstance(s_result, annmodel.SomeObject) + return s_result def specialize_call(self, hop): + from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype - RESTYPE= hop.args_s[0].const + RESTYPE = hop.args_s[0].const + if not isinstance(RESTYPE, lltype.LowLevelType): + assert isinstance(RESTYPE, annmodel.SomeObject) + r_result = hop.rtyper.getrepr(RESTYPE) + RESTYPE = r_result.lowleveltype pythonfunction = hop.args_s[1].const c_pythonfunction = hop.inputconst(lltype.Void, pythonfunction) args_v = [hop.inputarg(hop.args_r[i], arg=i) Modified: pypy/dist/pypy/rpython/extfunc.py ============================================================================== --- pypy/dist/pypy/rpython/extfunc.py (original) +++ pypy/dist/pypy/rpython/extfunc.py Sat Oct 13 09:58:37 2007 @@ -174,7 +174,8 @@ signature_args = self.normalize_args(*hop.args_s) args_r = [rtyper.getrepr(s_arg) for s_arg in signature_args] args_ll = [r_arg.lowleveltype for r_arg in args_r] - r_result = rtyper.getrepr(hop.s_result) + s_result = hop.s_result + r_result = rtyper.getrepr(s_result) ll_result = r_result.lowleveltype name = getattr(self, 'name', None) or self.instance.__name__ method_name = rtyper.type_system.name[:2] + 'typeimpl' @@ -191,7 +192,7 @@ original_impl = impl def ll_wrapper(*args): if running_on_llinterp: - return debug_llinterpcall(ll_result, fakeimpl, *args) + return debug_llinterpcall(s_result, fakeimpl, *args) else: return original_impl(*args) impl = func_with_new_name(ll_wrapper, name + '_wrapper') From arigo at codespeak.net Sat Oct 13 09:59:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 13 Oct 2007 09:59:17 +0200 (CEST) Subject: [pypy-svn] r47417 - pypy/dist/pypy/rpython/module Message-ID: <20071013075917.BF0A78289@code0.codespeak.net> Author: arigo Date: Sat Oct 13 09:59:17 2007 New Revision: 47417 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py Log: Fix an XXX - there are plenty of tables in this module that describe the necessary types, no need to special-case stat fields. Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Sat Oct 13 09:59:17 2007 @@ -207,24 +207,12 @@ def fakeimpl(arg): st = getattr(os, name)(arg) - tup = [st[i] for i in range(len(st))] - extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS)) - tup = tup + list(extra_zeroes) - fields = [] - for i in range(len(tup)): - if i in [1, 2, 6]: - fields.append(rffi.LONGLONG) - else: - fields.append(lltype.Signed) + fields = [TYPE for fieldname, TYPE in LL_STAT_FIELDS] TP = TUPLE_TYPE(fields) ll_tup = lltype.malloc(TP.TO) - for i in range(len(tup)): - # XXX ARGH! - if i in [1, 6, 2]: - val = rffi.cast(rffi.LONGLONG, tup[i]) - else: - val = tup[i] - setattr(ll_tup, 'item%d' % i, val) + for i, (fieldname, TYPE) in enumerate(LL_STAT_FIELDS): + val = getattr(st, fieldname) + rffi.setintfield(ll_tup, 'item%d' % i, val) return ll_tup if arg_is_path: From arigo at codespeak.net Sat Oct 13 10:47:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 13 Oct 2007 10:47:05 +0200 (CEST) Subject: [pypy-svn] r47418 - in pypy/dist/pypy/rpython: . lltypesystem memory Message-ID: <20071013084705.2B6D8826D@code0.codespeak.net> Author: arigo Date: Sat Oct 13 10:47:03 2007 New Revision: 47418 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/llheap.py pypy/dist/pypy/rpython/memory/gcwrapper.py Log: Support for write barriers in gcwrapper. Needs a few changes in the llinterpreter. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Sat Oct 13 10:47:03 2007 @@ -569,27 +569,29 @@ assert not isinstance(ob, lltype._interior_ptr) return ob - def setinterior(self, heap, obj, *fieldnamesval): - prefields, finalfield, fieldvalue = ( - fieldnamesval[:-2], fieldnamesval[-2], fieldnamesval[-1]) - for o in prefields: + def getinneraddr(self, obj, *offsets): + TYPE = lltype.typeOf(obj).TO + addr = llmemory.cast_ptr_to_adr(obj) + for o in offsets: if isinstance(o, str): - obj = getattr(obj, o) + addr += llmemory.offsetof(TYPE, o) + TYPE = getattr(TYPE, o) else: - obj = obj[o] - T = obj._T - if isinstance(finalfield, str): - if getattr(T, finalfield) is not lltype.Void: - heap.setfield(obj, finalfield, fieldvalue) - else: - if T.OF is not lltype.Void: - heap.setarrayitem(obj, finalfield, fieldvalue) + addr += llmemory.itemoffsetof(TYPE, o) + TYPE = TYPE.OF + return addr, TYPE def op_setinteriorfield(self, obj, *fieldnamesval): - self.setinterior(self.heap, obj, *fieldnamesval) + offsets, fieldvalue = fieldnamesval[:-1], fieldnamesval[-1] + inneraddr, FIELD = self.getinneraddr(obj, *offsets) + if FIELD is not lltype.Void: + self.heap.setinterior(obj, inneraddr, FIELD, fieldvalue) def op_bare_setinteriorfield(self, obj, *fieldnamesval): - self.setinterior(llheap, obj, *fieldnamesval) + offsets, fieldvalue = fieldnamesval[:-1], fieldnamesval[-1] + inneraddr, FIELD = self.getinneraddr(obj, *offsets) + if FIELD is not lltype.Void: + llheap.setinterior(obj, inneraddr, FIELD, fieldvalue) def op_getarrayitem(self, array, index): return array[index] Modified: pypy/dist/pypy/rpython/lltypesystem/llheap.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llheap.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llheap.py Sat Oct 13 10:47:03 2007 @@ -1,6 +1,6 @@ # only for the LLInterpreter. Don't use directly. -from pypy.rpython.lltypesystem.lltype import pyobjectptr, malloc, free +from pypy.rpython.lltypesystem.lltype import pyobjectptr, malloc, free, typeOf from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free from pypy.rpython.lltypesystem.llmemory import raw_memclear, raw_memcopy from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage, \ @@ -9,3 +9,8 @@ setfield = setattr from operator import setitem as setarrayitem from gc import collect + +def setinterior(toplevelcontainer, inneraddr, INNERTYPE, newvalue): + assert typeOf(newvalue) == INNERTYPE + # xxx access the address object's ref() directly for performance + inneraddr.ref()[0] = newvalue Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Sat Oct 13 10:47:03 2007 @@ -1,4 +1,4 @@ -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, llheap from pypy.rpython import llinterp from pypy.rpython.memory.support import get_address_linked_list from pypy.rpython.memory import gctypelayout @@ -52,14 +52,25 @@ return lltype.malloc(TYPE, n, flavor=flavor, zero=zero) def setfield(self, obj, fieldname, fieldvalue): - # XXX use write_barrier - but we need the address of the GcStruct - setattr(obj, fieldname, fieldvalue) + STRUCT = lltype.typeOf(obj).TO + addr = llmemory.cast_ptr_to_adr(obj) + addr += llmemory.offsetof(STRUCT, fieldname) + self.setinterior(obj, addr, getattr(STRUCT, fieldname), fieldvalue) def setarrayitem(self, array, index, newitem): - # XXX use write_barrier - but we need the address of the GcStruct - array[index] = newitem - - # XXX do we need a barrier for setinteriorfield too? + ARRAY = lltype.typeOf(array).TO + addr = llmemory.cast_ptr_to_adr(array) + addr += llmemory.itemoffsetof(ARRAY, index) + self.setinterior(array, addr, ARRAY.OF, newitem) + + def setinterior(self, toplevelcontainer, inneraddr, INNERTYPE, newvalue): + if isinstance(INNERTYPE, lltype.Ptr) and INNERTYPE.TO._gckind == 'gc': + self.gc.write_barrier(llmemory.cast_ptr_to_adr(newvalue), + inneraddr, + llmemory.cast_ptr_to_adr(toplevelcontainer)) + else: + llheap.setinterior(toplevelcontainer, inneraddr, + INNERTYPE, newvalue) def collect(self): self.gc.collect() From arigo at codespeak.net Sat Oct 13 11:22:28 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 13 Oct 2007 11:22:28 +0200 (CEST) Subject: [pypy-svn] r47419 - pypy/dist/pypy/rpython/memory Message-ID: <20071013092228.D90128289@code0.codespeak.net> Author: arigo Date: Sat Oct 13 11:22:26 2007 New Revision: 47419 Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py Log: Don't need to record the location of GC pointers inside immutable prebuilt structures, because these cannot change later to point to GC heap objects. Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Sat Oct 13 11:22:26 2007 @@ -146,10 +146,12 @@ adr = llmemory.cast_ptr_to_adr(hdr) gc.init_gc_object_immortal(adr, typeid) - # XXX should skip the gc pointers inside immutable structures, because - # they cannot be dynamically modified to point to GC heap objects + # The following collects the addresses of all the fields that have + # a GC Pointer type, inside the current prebuilt object. All such + # fields are potential roots: unless the structure is immutable, + # they could be changed later to point to GC heap objects. adr = llmemory.cast_ptr_to_adr(value._as_ptr()) - for a in gc_pointers_inside(value, adr): + for a in mutable_gc_pointers_inside(value, adr): self.addresses_of_static_ptrs.append(a) # ____________________________________________________________ @@ -188,22 +190,28 @@ return llmemory.offsetof(WEAKREF, "weakptr") return -1 -def gc_pointers_inside(v, adr): +def mutable_gc_pointers_inside(v, adr): t = lltype.typeOf(v) if isinstance(t, lltype.Struct): + if t._hints.get('immutable'): + return for n, t2 in t._flds.iteritems(): if isinstance(t2, lltype.Ptr) and t2.TO._gckind == 'gc': yield adr + llmemory.offsetof(t, n) elif isinstance(t2, (lltype.Array, lltype.Struct)): - for a in gc_pointers_inside(getattr(v, n), adr + llmemory.offsetof(t, n)): + for a in mutable_gc_pointers_inside(getattr(v, n), + adr + llmemory.offsetof(t, n)): yield a elif isinstance(t, lltype.Array): + if t._hints.get('immutable'): + return if isinstance(t.OF, lltype.Ptr) and t.OF.TO._gckind == 'gc': for i in range(len(v.items)): yield adr + llmemory.itemoffsetof(t, i) elif isinstance(t.OF, lltype.Struct): for i in range(len(v.items)): - for a in gc_pointers_inside(v.items[i], adr + llmemory.itemoffsetof(t, i)): + for a in mutable_gc_pointers_inside(v.items[i], + adr + llmemory.itemoffsetof(t, i)): yield a ########## weakrefs ########## From arigo at codespeak.net Sat Oct 13 11:26:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 13 Oct 2007 11:26:22 +0200 (CEST) Subject: [pypy-svn] r47420 - in pypy/dist/pypy/rpython/memory: . test Message-ID: <20071013092622.AEA6C828A@code0.codespeak.net> Author: arigo Date: Sat Oct 13 11:26:22 2007 New Revision: 47420 Removed: pypy/dist/pypy/rpython/memory/simulator.py pypy/dist/pypy/rpython/memory/test/test_simulator.py Log: Forgot to kill this module, which is not imported any more. From arigo at codespeak.net Sat Oct 13 11:44:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 13 Oct 2007 11:44:00 +0200 (CEST) Subject: [pypy-svn] r47421 - pypy/dist/pypy/rpython/memory Message-ID: <20071013094400.A811081A7@code0.codespeak.net> Author: arigo Date: Sat Oct 13 11:44:00 2007 New Revision: 47421 Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py pypy/dist/pypy/rpython/memory/support.py Log: Stop pretending that get_roots() returns a regular AddressLinkedList. It really returns something that has only a pop() method with a special interface - returning NULL to mean "I'm done". Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Sat Oct 13 11:44:00 2007 @@ -8,7 +8,6 @@ class GCManagedHeap(object): def __init__(self, llinterp, flowgraphs, gc_class, GC_PARAMS={}): - self.RootLinkedList = get_address_linked_list(10, hackishpop=True) self.AddressLinkedList = get_address_linked_list(10) self.gc = gc_class(self.AddressLinkedList, **GC_PARAMS) self.gc.get_roots = self.get_roots_from_llinterp @@ -30,14 +29,14 @@ def get_roots_from_llinterp(self): sizeofaddr = llmemory.sizeof(llmemory.Address) - ll = self.RootLinkedList() + ll = [llmemory.NULL] # end marker for addrofaddr in self.constantroots: if addrofaddr.address[0]: ll.append(addrofaddr) for addrofaddr in self.llinterp.find_roots(): if addrofaddr.address[0]: ll.append(addrofaddr) - return ll + return RootLinkedList(ll) # ____________________________________________________________ # @@ -88,6 +87,13 @@ # ____________________________________________________________ +class RootLinkedList(object): + _alloc_flavor_ = 'raw' + + def __init__(self, lst): + self._lst = lst + self.pop = lst.pop + class DirectRunLayoutBuilder(gctypelayout.TypeLayoutBuilder): Modified: pypy/dist/pypy/rpython/memory/support.py ============================================================================== --- pypy/dist/pypy/rpython/memory/support.py (original) +++ pypy/dist/pypy/rpython/memory/support.py Sat Oct 13 11:44:00 2007 @@ -3,7 +3,7 @@ DEFAULT_CHUNK_SIZE = 1019 -def get_address_linked_list(chunk_size=DEFAULT_CHUNK_SIZE, hackishpop=False): +def get_address_linked_list(chunk_size=DEFAULT_CHUNK_SIZE): CHUNK = lltype.ForwardReference() CHUNK.become(lltype.Struct('AddressLinkedListChunk', @@ -71,8 +71,6 @@ return chunk.length != 0 or bool(chunk.previous) def pop(self): - if hackishpop and not self.non_empty(): - return llmemory.NULL chunk = self.chunk if chunk.length == 0: chunk = self.shrink() From arigo at codespeak.net Sat Oct 13 12:29:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 13 Oct 2007 12:29:15 +0200 (CEST) Subject: [pypy-svn] r47423 - in pypy/dist/pypy/lib: . app_test Message-ID: <20071013102915.A44BD8278@code0.codespeak.net> Author: arigo Date: Sat Oct 13 12:29:14 2007 New Revision: 47423 Modified: pypy/dist/pypy/lib/_structseq.py pypy/dist/pypy/lib/app_test/test_structseq.py Log: Make this test pass. Modified: pypy/dist/pypy/lib/_structseq.py ============================================================================== --- pypy/dist/pypy/lib/_structseq.py (original) +++ pypy/dist/pypy/lib/_structseq.py Sat Oct 13 12:29:14 2007 @@ -60,6 +60,7 @@ dict['_extra_fields'] = tuple(extra_fields) dict['__new__'] = structseq_new dict['__reduce__'] = structseq_reduce + dict['__setattr__'] = structseq_setattr return type.__new__(metacls, classname, (tuple,), dict) @@ -91,7 +92,7 @@ dict[name] = value sequence = sequence[:N] result = tuple.__new__(cls, sequence) - result.__dict__ = dict + object.__setattr__(result, '__dict__', dict) for field in cls._extra_fields: name = field.__name__ if name not in dict: @@ -100,3 +101,7 @@ def structseq_reduce(self): return type(self), (tuple(self), self.__dict__) + +def structseq_setattr(self, attr, value): + raise AttributeError("%r object has no attribute %r" % ( + self.__class__.__name__, attr)) Modified: pypy/dist/pypy/lib/app_test/test_structseq.py ============================================================================== --- pypy/dist/pypy/lib/app_test/test_structseq.py (original) +++ pypy/dist/pypy/lib/app_test/test_structseq.py Sat Oct 13 12:29:14 2007 @@ -76,5 +76,4 @@ def test_no_extra_assignments(): x = mydata(range(100, 113)) - py.test.skip("not working yet") py.test.raises((TypeError, AttributeError), "x.some_random_attribute = 1") From arigo at codespeak.net Sat Oct 13 12:37:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 13 Oct 2007 12:37:04 +0200 (CEST) Subject: [pypy-svn] r47424 - in pypy/dist/pypy/module/__builtin__: . test Message-ID: <20071013103704.7272E828F@code0.codespeak.net> Author: arigo Date: Sat Oct 13 12:37:04 2007 New Revision: 47424 Modified: pypy/dist/pypy/module/__builtin__/operation.py pypy/dist/pypy/module/__builtin__/test/test_builtin.py Log: Don't crash in unichr(out-of-range). Modified: pypy/dist/pypy/module/__builtin__/operation.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/operation.py (original) +++ pypy/dist/pypy/module/__builtin__/operation.py Sat Oct 13 12:37:04 2007 @@ -17,10 +17,16 @@ w_character = space.newstring([w_ascii]) return w_character -def unichr(space, w_code): +def unichr(space, code): "Return a Unicode string of one character with the given ordinal." # XXX range checking! - return space.newunicode([__builtin__.unichr(space.int_w(w_code))]) + try: + c = __builtin__.unichr(code) + except ValueError: + raise OperationError(space.w_ValueError, + space.wrap("unichr() arg out of range")) + return space.newunicode([c]) +unichr.unwrap_spec = [ObjSpace, int] def len(space, w_obj): "len(object) -> integer\n\nReturn the number of items of a sequence or mapping." Modified: pypy/dist/pypy/module/__builtin__/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_builtin.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_builtin.py Sat Oct 13 12:37:04 2007 @@ -28,6 +28,17 @@ raises(ValueError, chr, -1) raises(TypeError, chr, 'a') + def test_unichr(self): + import sys + assert unichr(65) == u'A' + assert type(unicode(65)) is unicode + assert unichr(0x9876) == u'\u9876' + assert ord(unichr(sys.maxunicode)) == sys.maxunicode + if sys.maxunicode > 0x10000: + assert unichr(0x10000) == u'\U00010000' + raises(ValueError, unichr, -1) + raises(ValueError, unichr, sys.maxunicode+1) + def test_intern(self): raises(TypeError, intern) raises(TypeError, intern, 1) From arigo at codespeak.net Sat Oct 13 12:41:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 13 Oct 2007 12:41:51 +0200 (CEST) Subject: [pypy-svn] r47425 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20071013104151.0EF778292@code0.codespeak.net> Author: arigo Date: Sat Oct 13 12:41:50 2007 New Revision: 47425 Modified: pypy/dist/pypy/objspace/std/formatting.py pypy/dist/pypy/objspace/std/test/test_stringformat.py Log: Fix another XXX. Modified: pypy/dist/pypy/objspace/std/formatting.py ============================================================================== --- pypy/dist/pypy/objspace/std/formatting.py (original) +++ pypy/dist/pypy/objspace/std/formatting.py Sat Oct 13 12:41:50 2007 @@ -394,9 +394,11 @@ else: n = space.int_w(w_value) if do_unicode: - c = unichr(n) - # XXX no range checking, but our unichr() builtin needs - # to be fixed too + try: + c = unichr(n) + except ValueError: + raise OperationError(space.w_OverflowError, + space.wrap("unicode character code out of range")) self.std_wp([c]) else: try: Modified: pypy/dist/pypy/objspace/std/test/test_stringformat.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_stringformat.py (original) +++ pypy/dist/pypy/objspace/std/test/test_stringformat.py Sat Oct 13 12:41:50 2007 @@ -131,10 +131,12 @@ raises(ValueError, '%('.__mod__, ({},)) def test_format_char(self): + import sys assert '%c' % 65 == 'A' assert '%c' % 'e' == 'e' raises(OverflowError, '%c'.__mod__, (256,)) raises(OverflowError, '%c'.__mod__, (-1,)) + raises(OverflowError, u'%c'.__mod__, (sys.maxunicode+1,)) raises(TypeError, '%c'.__mod__, ("bla",)) raises(TypeError, '%c'.__mod__, ("",)) raises(TypeError, '%c'.__mod__, (['c'],)) From arigo at codespeak.net Sat Oct 13 19:38:14 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 13 Oct 2007 19:38:14 +0200 (CEST) Subject: [pypy-svn] r47429 - pypy/dist/pypy/rpython/ootypesystem Message-ID: <20071013173814.CD02C8110@code0.codespeak.net> Author: arigo Date: Sat Oct 13 19:38:14 2007 New Revision: 47429 Modified: pypy/dist/pypy/rpython/ootypesystem/rtuple.py Log: It doesn't make much sense to ask for the id() of a tuple. The AbstractTupleRepr provides an rtype_id() that complains appropriately. Modified: pypy/dist/pypy/rpython/ootypesystem/rtuple.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rtuple.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rtuple.py Sat Oct 13 19:38:14 2007 @@ -33,10 +33,6 @@ cname = inputconst(ootype.Void, name) return llops.genop("oogetfield", [v_tuple, cname], resulttype=llresult) - def rtype_id(self, hop): - vinst, = hop.inputargs(self) - return hop.genop('ooidentityhash', [vinst], resulttype=ootype.Signed) - def rtype_bltn_list(self, hop): from pypy.rpython.ootypesystem import rlist v_tup = hop.inputarg(self, 0) From tismer at codespeak.net Sat Oct 13 19:43:43 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sat, 13 Oct 2007 19:43:43 +0200 (CEST) Subject: [pypy-svn] r47430 - in pypy/dist/pypy: rlib rlib/test rpython/lltypesystem Message-ID: <20071013174343.E93F3816E@code0.codespeak.net> Author: tismer Date: Sat Oct 13 19:43:43 2007 New Revision: 47430 Modified: pypy/dist/pypy/rlib/rmmap.py pypy/dist/pypy/rlib/test/test_rmmap.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: some steps in the rmmap windows branch. Some things work, right now there is an annotation problem Modified: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/rmmap.py (original) +++ pypy/dist/pypy/rlib/rmmap.py Sat Oct 13 19:43:43 2007 @@ -80,6 +80,9 @@ def external(name, args, result): return rffi.llexternal(name, args, result, includes=CConfig._includes_) +def winexternal(name, args, result): + return rffi.llexternal(name, args, result, includes=CConfig._includes_, calling_conv='win') + PTR = rffi.CCHARP c_memmove = external('memmove', [PTR, PTR, size_t], lltype.Void) @@ -97,100 +100,84 @@ def _get_error_msg(): errno = rffi.get_errno() - return os.strerror(errno) + return os.strerror(errno) + elif _MS_WINDOWS: - from ctypes import wintypes, Union, Structure - - WORD = wintypes.WORD - DWORD = wintypes.DWORD - BOOL = wintypes.BOOL - LONG = wintypes.LONG - # LPVOID = PTR does not work with ctypes - LPVOID = wintypes.c_void_p + WORD = rffi.UINT + DWORD = rffi.ULONG + BOOL = rffi.LONG + LONG = rffi.LONG + LPVOID = PTR LPCVOID = LPVOID - DWORD_PTR = DWORD # ??? - INT = wintypes.c_int # there is no wintypes.INT - POINTER = wintypes.POINTER - INVALID_c_int_VALUE = wintypes.c_int(-1).value - windll = wintypes.windll - - class SYSINFO_STRUCT(Structure): - _fields_ = [("wProcessorArchitecture", WORD), - ("wReserved", WORD)] - - class SYSINFO_UNION(Union): - _fields_ = [("dwOemId", DWORD), - ("struct", SYSINFO_STRUCT)] - - class SYSTEM_INFO(Structure): - _fields_ = [("union", SYSINFO_UNION), - ("dwPageSize", DWORD), - ("lpMinimumApplicationAddress", LPVOID), - ("lpMaximumApplicationAddress", LPVOID), - ("dwActiveProcessorMask", DWORD_PTR), - ("dwNumberOfProcessors", DWORD), - ("dwProcessorType", DWORD), - ("dwAllocationGranularity", DWORD), - ("wProcessorLevel", WORD), - ("wProcessorRevision", WORD)] - - windll.kernel32.GetSystemInfo.argtypes = [POINTER(SYSTEM_INFO)] - GetFileSize = windll.kernel32.GetFileSize - GetFileSize.argtypes = [rffi.INT, POINTER(rffi.INT)] - GetFileSize.restype = rffi.INT - GetCurrentProcess = windll.kernel32.GetCurrentProcess - GetCurrentProcess.restype = rffi.INT - DuplicateHandle = windll.kernel32.DuplicateHandle - DuplicateHandle.argtypes = [rffi.INT, rffi.INT, rffi.INT, POINTER(rffi.INT), DWORD, - BOOL, DWORD] - DuplicateHandle.restype = BOOL - CreateFileMapping = windll.kernel32.CreateFileMappingA - CreateFileMapping.argtypes = [rffi.INT, PTR, rffi.INT, rffi.INT, rffi.INT, - c_char_p] - CreateFileMapping.restype = rffi.INT - MapViewOfFile = windll.kernel32.MapViewOfFile - MapViewOfFile.argtypes = [rffi.INT, DWORD, DWORD, DWORD, DWORD] - MapViewOfFile.restype = PTR - CloseHandle = windll.kernel32.CloseHandle - CloseHandle.argtypes = [rffi.INT] - CloseHandle.restype = BOOL - UnmapViewOfFile = windll.kernel32.UnmapViewOfFile - UnmapViewOfFile.argtypes = [LPCVOID] - UnmapViewOfFile.restype = BOOL - FlushViewOfFile = windll.kernel32.FlushViewOfFile - FlushViewOfFile.argtypes = [LPCVOID, rffi.INT] - FlushViewOfFile.restype = BOOL - SetFilePointer = windll.kernel32.SetFilePointer - SetFilePointer.argtypes = [rffi.INT, rffi.INT, POINTER(rffi.INT), rffi.INT] - SetEndOfFile = windll.kernel32.SetEndOfFile - SetEndOfFile.argtypes = [rffi.INT] - msvcr71 = cdll.LoadLibrary("msvcr71.dll") - msvcr71._get_osfhandle.argtypes = [rffi.INT] - msvcr71._get_osfhandle.restype = rffi.INT - # libc._lseek.argtypes = [rffi.INT, rffi.INT, rffi.INT] - # libc._lseek.restype = rffi.INT + DWORD_PTR = DWORD + INT = rffi.INT + INT_P = rffi.INTP + LPCTSTR = rffi.CCHARP + + + class ComplexCConfig: + _includes_ = CConfig._includes_ + sysinfo_struct = rffi_platform.Struct( + 'SYSTEM_INFO', [ + #("union_ignored", DWORD), # I'll do this later + ("dwPageSize", DWORD), + ("lpMinimumApplicationAddress", LPVOID), + ("lpMaximumApplicationAddress", LPVOID), + ("dwActiveProcessorMask", DWORD_PTR), + ("dwNumberOfProcessors", DWORD), + ("dwProcessorType", DWORD), + ("dwAllocationGranularity", DWORD), + ("wProcessorLevel", WORD), + ("wProcessorRevision", WORD), + ]) + + config = rffi_platform.configure(ComplexCConfig) + sysinfo_struct = config['sysinfo_struct'] + sysinfo_struct_p = lltype.Ptr(sysinfo_struct) + + GetSystemInfo = winexternal('GetSystemInfo', [sysinfo_struct_p], lltype.Void) + GetFileSize = winexternal('GetFileSize', [INT, INT_P], INT) + GetCurrentProcess = winexternal('getCurrentProcess', [], INT) + DuplicateHandle = winexternal('DuplicateHandle', [INT, INT, INT, INT_P, DWORD, BOOL, DWORD], BOOL) + CreateFileMapping = winexternal('CreateFileMappingA', [INT, PTR, INT, INT, INT, LPCTSTR], INT) + MapViewOfFile = winexternal('MapViewOfFile', [INT, DWORD, DWORD, DWORD, DWORD], PTR) + CloseHandle = winexternal('CloseHandle', [INT], BOOL) + UnmapViewOfFile = winexternal('UnmapViewOfFile', [LPCVOID], BOOL) + FlushViewOfFile = winexternal('FlushViewOfFile', [LPCVOID, INT], BOOL) + SetFilePointer = winexternal('SetFilePointer', [INT, INT, INT_P, INT], DWORD) + SetEndOfFile = winexternal('SetEndOfFile', [INT], INT) + _get_osfhandle = winexternal('_get_osfhandle', [INT], INT) + GetLastError = winexternal('GetLastError', [], DWORD) def _get_page_size(): - si = SYSTEM_INFO() - windll.kernel32.GetSystemInfo(byref(si)) - return int(si.dwPageSize) + try: + si = rffi.make(sysinfo_struct) + GetSystemInfo(si) + return int(si.c_dwPageSize) + finally: + lltype.free(si, flavor="raw") def _get_file_size(handle): # XXX use native Windows types like WORD - high = rffi.INT(0) - low = rffi.INT(windll.kernel32.GetFileSize(rffi.INT(handle.value), byref(high))) - # low might just happen to have the value INVALID_FILE_SIZE - # so we need to check the last error also - INVALID_FILE_SIZE = -1 - NO_ERROR = 0 - dwErr = GetLastError() - if low.value == INVALID_FILE_SIZE and dwErr != NO_ERROR: - raise REnvironmentError(os.strerror(dwErr)) - return low.value, high.value + high_ref = rffi.CFixedArray(INT, 1) + try: + low = GetFileSize(rffi.cast(INT, handle), high_ref) + high = high_ref[0] + # low might just happen to have the value INVALID_FILE_SIZE + # so we need to check the last error also + INVALID_FILE_SIZE = -1 + NO_ERROR = 0 + dwErr = GetLastError() + err = dwErr.value + if low.value == INVALID_FILE_SIZE and err != NO_ERROR: + raise REnvironmentError(os.strerror(err)) + return low.value, high.value + finally: + lltype.free(high_ref, flavor='raw') def _get_error_msg(): - errno = GetLastError() + errno = GetLastError().value return os.strerror(errno) PAGESIZE = _get_page_size() @@ -214,7 +201,7 @@ def check_valid(self): if _MS_WINDOWS: - to_close = self.map_handle.value == INVALID_INT_VALUE + to_close = self.map_handle == INVALID_INT_VALUE elif _POSIX: to_close = self.closed @@ -240,12 +227,14 @@ if self.size > 0: self.unmapview() self.setdata(NODATA, 0) - if self.map_handle.value != INVALID_rffi.INT_VALUE: - CloseHandle(self.map_handle) - self.map_handle.value = INVALID_rffi.INT_VALUE - if self.file_handle.value != INVALID_rffi.INT_VALUE: - CloseHandle(self.file_handle) - self.file_handle.value = INVALID_rffi.INT_VALUE + if self.map_handle != INVALID_INT_VALUE: + CloseHandle(rffi.cast(INT, self.map_handle)) + ##self.map_handle.value = INVALID_INT_VALUE + self.map_handle = INVALID_INT_VALUE + if self.file_handle != INVALID_INT_VALUE: + CloseHandle(rffi.cast(INT, self.file_handle)) + ##self.file_handle.value = INVALID_INT_VALUE + self.file_handle = INVALID_INT_VALUE elif _POSIX: self.closed = True if self.fd != -1: @@ -348,11 +337,11 @@ size = self.size if _MS_WINDOWS: - if self.file_handle.value != INVALID_rffi.INT_VALUE: + if self.file_handle != INVALID_INT_VALUE: low, high = _get_file_size(self.file_handle) if not high and low <= sys.maxint: return low - size = rffi.INT((high << 32) + low).value + size = (high << 32) + low elif _POSIX: st = os.fstat(self.fd) size = st[stat.ST_SIZE] @@ -459,23 +448,28 @@ # move to the desired EOF position if _64BIT: - newsize_high = DWORD(newsize >> 32) - newsize_low = DWORD(newsize & 0xFFFFFFFF) + newsize_high = rffi.cast(DWORD, newsize >> 32) + newsize_low = rffi.cast(DWORD, newsize & 0xFFFFFFFF) else: - newsize_high = rffi.INT(0) - newsize_low = rffi.INT(newsize) + newsize_high = rffi.cast(INT, 0) + newsize_low = rffi.cast(INT, newsize) - FILE_BEGIN = rffi.INT(0) - SetFilePointer(self.file_handle, newsize_low, byref(newsize_high), - FILE_BEGIN) + FILE_BEGIN = rffi.cast(INT, 0) + high_ref = rffi.CFixedArray(INT, 1) + try: + high_ref[0] = newsize_high + SetFilePointer(self.file_handle, newsize_low, high_ref, + FILE_BEGIN) + finally: + lltype.free(high_ref, flavor='raw') # resize the file SetEndOfFile(self.file_handle) # create another mapping object and remap the file view res = CreateFileMapping(self.file_handle, NULL, PAGE_READWRITE, newsize_high, newsize_low, self.tagname) - self.map_handle = rffi.INT(res) + self.map_handle = res.value - dwErrCode = DWORD(0) + dwErrCode = rffi.cast(DWORD, 0) if self.map_handle: data = MapViewOfFile(self.map_handle, FILE_MAP_WRITE, 0, 0, 0) @@ -487,7 +481,7 @@ else: dwErrCode = GetLastError() - raise REnvironmentError(os.strerror(dwErrCode)) + raise REnvironmentError(os.strerror(dwErrCode.value)) def len(self): self.check_valid() @@ -611,7 +605,7 @@ # assume -1 and 0 both mean invalid file descriptor # to 'anonymously' map memory. if fileno != -1 and fileno != 0: - fh = msvcr71._get_osfhandle(fileno) + fh = _get_osfhandle(rffi.cast(INT, fileno)).value if fh == -1: raise REnvironmentError(_get_error_msg()) # Win9x appears to need us seeked to zero @@ -619,27 +613,31 @@ # libc._lseek(fileno, 0, SEEK_SET) m = MMap(access) - # XXX the following two attributes should be plain RPython ints - m.file_handle = rffi.INT(INVALID_rffi.INT_VALUE) - m.map_handle = rffi.INT(INVALID_rffi.INT_VALUE) - + m.file_handle = INVALID_INT_VALUE + m.map_handle = INVALID_INT_VALUE if fh: # it is necessary to duplicate the handle, so the - # Python code can close it on us - res = DuplicateHandle(GetCurrentProcess(), # source process handle - fh, # handle to be duplicated - GetCurrentProcess(), # target process handle - byref(m.file_handle), # result - 0, # access - ignored due to options value - False, # inherited by child procs? - DUPLICATE_SAME_ACCESS) # options - if not res: - raise REnvironmentError(_get_error_msg()) - + # Python code can close it on us + handle_ref = rffi.CFixedArray(INT, 1) + handle_ref[0].value = m.file_handle + try: + res = DuplicateHandle(GetCurrentProcess(), # source process handle + rffi.cast(INT, fh), # handle to be duplicated + GetCurrentProcess(), # target process handle + handle_ref, # result + 0, # access - ignored due to options value + False, # inherited by child procs? + DUPLICATE_SAME_ACCESS).value # options + if not res: + raise REnvironmentError(_get_error_msg()) + m.file_handle = handle_ref[0].value + finally: + lltype.free(handle_ref, flavor='raw') + if not map_size: - low, high = _get_file_size(rffi.INT(fh)) + low, high = _get_file_size(fh) if _64BIT: - map_size = rffi.INT((low << 32) + 1).value + map_size = (low << 32) + 1 else: if high: # file is too large to map completely @@ -652,18 +650,18 @@ # DWORD is a 4-byte int. If int > 4-byte it must be divided if _64BIT: - size_hi = DWORD(map_size >> 32) - size_lo = DWORD(map_size & 0xFFFFFFFF) + size_hi = rffi.cast(DWORD, map_size >> 32) + size_lo = rffi.cast(DWORD, map_size & 0xFFFFFFFF) else: - size_hi = rffi.INT(0) - size_lo = rffi.INT(map_size) + size_hi = rffi.cast(INT, 0) + size_lo = rffi.cast(INT, map_size) - m.map_handle = rffi.INT(CreateFileMapping(m.file_handle, NULL, flProtect, - size_hi, size_lo, m.tagname)) + m.map_handle = CreateFileMapping(rffi.cast(INT, m.file_handle), NULL, flProtect, + size_hi, size_lo, m.tagname).value if m.map_handle: - res = MapViewOfFile(m.map_handle, dwDesiredAccess, - 0, 0, 0) + res = MapViewOfFile(rffi.cast(INT, m.map_handle), dwDesiredAccess, + 0, 0, 0).value if res: m.setdata(res, map_size) return m @@ -672,7 +670,7 @@ else: dwErr = GetLastError() - raise REnvironmentError(os.strerror(dwErr)) + raise REnvironmentError(os.strerror(dwErr.value)) # register_external here? Modified: pypy/dist/pypy/rlib/test/test_rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rmmap.py (original) +++ pypy/dist/pypy/rlib/test/test_rmmap.py Sat Oct 13 19:43:43 2007 @@ -34,8 +34,8 @@ def test_file_size(self): def func(no): - if os.name == "nt": - skip("Only Unix checks file size") + ## if os.name == "nt": + ## skip("Only Unix checks file size") try: mmap.mmap(no, 123) Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Sat Oct 13 19:43:43 2007 @@ -291,9 +291,18 @@ # char * CCHARP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) - -# int *, unsigned int *, etc. -#INTP = ... see setup() above + +# int * +INTP = lltype.Ptr(lltype.Array(lltype.Signed, hints={'nolength': True})) + +# unsigned int * +UINTP = lltype.Ptr(lltype.Array(lltype.Unsigned, hints={'nolength': True})) + +# signed long long * +DINTP = lltype.Ptr(lltype.Array(lltype.SignedLongLong, hints={'nolength': True})) + +# unsigned long long * +UDINTP = lltype.Ptr(lltype.Array(lltype.UnsignedLongLong, hints={'nolength': True})) # double * DOUBLEP = lltype.Ptr(lltype.Array(DOUBLE, hints={'nolength': True})) From arigo at codespeak.net Sat Oct 13 21:27:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 13 Oct 2007 21:27:31 +0200 (CEST) Subject: [pypy-svn] r47431 - pypy/dist/pypy/translator/backendopt/test Message-ID: <20071013192731.3596780E0@code0.codespeak.net> Author: arigo Date: Sat Oct 13 21:27:29 2007 New Revision: 47431 Modified: pypy/dist/pypy/translator/backendopt/test/test_escape.py Log: --view support. Modified: pypy/dist/pypy/translator/backendopt/test/test_escape.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_escape.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_escape.py Sat Oct 13 21:27:29 2007 @@ -3,6 +3,7 @@ from pypy.translator.backendopt.support import find_backedges, find_loop_blocks from pypy.rpython.llinterp import LLInterpreter from pypy.rlib.objectmodel import instantiate +from pypy import conftest import py @@ -10,6 +11,8 @@ t = TranslationContext() t.buildannotator().build_types(function, types) t.buildrtyper().specialize() + if conftest.option.view: + t.view() adi = AbstractDataFlowInterpreter(t) graph = graphof(t, function) adi.schedule_function(graph) From tismer at codespeak.net Sat Oct 13 21:46:16 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sat, 13 Oct 2007 21:46:16 +0200 (CEST) Subject: [pypy-svn] r47432 - in pypy/dist/pypy/rlib: . test Message-ID: <20071013194616.85FA98175@code0.codespeak.net> Author: tismer Date: Sat Oct 13 21:46:16 2007 New Revision: 47432 Modified: pypy/dist/pypy/rlib/rmmap.py pypy/dist/pypy/rlib/test/test_rmmap.py Log: rmmap works. checking in before doing some cleanups Modified: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/rmmap.py (original) +++ pypy/dist/pypy/rlib/rmmap.py Sat Oct 13 21:46:16 2007 @@ -112,6 +112,7 @@ DWORD_PTR = DWORD INT = rffi.INT INT_P = rffi.INTP + DWORD_P = rffi.DINTP LPCTSTR = rffi.CCHARP @@ -137,7 +138,7 @@ GetSystemInfo = winexternal('GetSystemInfo', [sysinfo_struct_p], lltype.Void) GetFileSize = winexternal('GetFileSize', [INT, INT_P], INT) - GetCurrentProcess = winexternal('getCurrentProcess', [], INT) + GetCurrentProcess = winexternal('GetCurrentProcess', [], INT) DuplicateHandle = winexternal('DuplicateHandle', [INT, INT, INT, INT_P, DWORD, BOOL, DWORD], BOOL) CreateFileMapping = winexternal('CreateFileMappingA', [INT, PTR, INT, INT, INT, LPCTSTR], INT) MapViewOfFile = winexternal('MapViewOfFile', [INT, DWORD, DWORD, DWORD, DWORD], PTR) @@ -160,7 +161,7 @@ def _get_file_size(handle): # XXX use native Windows types like WORD - high_ref = rffi.CFixedArray(INT, 1) + high_ref = lltype.malloc(INT_P.TO, 1, flavor='raw') try: low = GetFileSize(rffi.cast(INT, handle), high_ref) high = high_ref[0] @@ -169,15 +170,15 @@ INVALID_FILE_SIZE = -1 NO_ERROR = 0 dwErr = GetLastError() - err = dwErr.value - if low.value == INVALID_FILE_SIZE and err != NO_ERROR: + err = rffi.cast(lltype.Signed, dwErr) + if rffi.cast(lltype.Signed, low) == INVALID_FILE_SIZE and err != NO_ERROR: raise REnvironmentError(os.strerror(err)) - return low.value, high.value + return low, high finally: lltype.free(high_ref, flavor='raw') def _get_error_msg(): - errno = GetLastError().value + errno = rffi.cast(lltype.Signed, GetLastError()) return os.strerror(errno) PAGESIZE = _get_page_size() @@ -229,11 +230,9 @@ self.setdata(NODATA, 0) if self.map_handle != INVALID_INT_VALUE: CloseHandle(rffi.cast(INT, self.map_handle)) - ##self.map_handle.value = INVALID_INT_VALUE self.map_handle = INVALID_INT_VALUE if self.file_handle != INVALID_INT_VALUE: CloseHandle(rffi.cast(INT, self.file_handle)) - ##self.file_handle.value = INVALID_INT_VALUE self.file_handle = INVALID_INT_VALUE elif _POSIX: self.closed = True @@ -450,12 +449,13 @@ if _64BIT: newsize_high = rffi.cast(DWORD, newsize >> 32) newsize_low = rffi.cast(DWORD, newsize & 0xFFFFFFFF) + high_ref = lltype.malloc(DWORD_P.TO, 1, flavor='raw') else: newsize_high = rffi.cast(INT, 0) newsize_low = rffi.cast(INT, newsize) + high_ref = lltype.malloc(INT_P.TO, 1, flavor='raw') FILE_BEGIN = rffi.cast(INT, 0) - high_ref = rffi.CFixedArray(INT, 1) try: high_ref[0] = newsize_high SetFilePointer(self.file_handle, newsize_low, high_ref, @@ -467,7 +467,7 @@ # create another mapping object and remap the file view res = CreateFileMapping(self.file_handle, NULL, PAGE_READWRITE, newsize_high, newsize_low, self.tagname) - self.map_handle = res.value + self.map_handle = res dwErrCode = rffi.cast(DWORD, 0) if self.map_handle: @@ -480,8 +480,8 @@ dwErrCode = GetLastError() else: dwErrCode = GetLastError() - - raise REnvironmentError(os.strerror(dwErrCode.value)) + err = rffi.cast(lltype.Signed, dwErrCode) + raise REnvironmentError(os.strerror(err)) def len(self): self.check_valid() @@ -605,7 +605,7 @@ # assume -1 and 0 both mean invalid file descriptor # to 'anonymously' map memory. if fileno != -1 and fileno != 0: - fh = _get_osfhandle(rffi.cast(INT, fileno)).value + fh = _get_osfhandle(rffi.cast(INT, fileno)) if fh == -1: raise REnvironmentError(_get_error_msg()) # Win9x appears to need us seeked to zero @@ -618,8 +618,8 @@ if fh: # it is necessary to duplicate the handle, so the # Python code can close it on us - handle_ref = rffi.CFixedArray(INT, 1) - handle_ref[0].value = m.file_handle + handle_ref = lltype.malloc(INT_P.TO, 1, flavor='raw') + handle_ref[0] = rffi.cast(INT, m.file_handle) try: res = DuplicateHandle(GetCurrentProcess(), # source process handle rffi.cast(INT, fh), # handle to be duplicated @@ -627,10 +627,10 @@ handle_ref, # result 0, # access - ignored due to options value False, # inherited by child procs? - DUPLICATE_SAME_ACCESS).value # options + DUPLICATE_SAME_ACCESS) # options if not res: raise REnvironmentError(_get_error_msg()) - m.file_handle = handle_ref[0].value + m.file_handle = handle_ref[0] finally: lltype.free(handle_ref, flavor='raw') @@ -657,11 +657,11 @@ size_lo = rffi.cast(INT, map_size) m.map_handle = CreateFileMapping(rffi.cast(INT, m.file_handle), NULL, flProtect, - size_hi, size_lo, m.tagname).value + size_hi, size_lo, m.tagname) if m.map_handle: res = MapViewOfFile(rffi.cast(INT, m.map_handle), dwDesiredAccess, - 0, 0, 0).value + 0, 0, 0) if res: m.setdata(res, map_size) return m @@ -669,8 +669,8 @@ dwErr = GetLastError() else: dwErr = GetLastError() - - raise REnvironmentError(os.strerror(dwErr.value)) + err = rffi.cast(lltype.Signed, dwErr) + raise REnvironmentError(os.strerror(err)) # register_external here? Modified: pypy/dist/pypy/rlib/test/test_rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rmmap.py (original) +++ pypy/dist/pypy/rlib/test/test_rmmap.py Sat Oct 13 21:46:16 2007 @@ -33,9 +33,9 @@ interpret(f, []) def test_file_size(self): + if os.name == "nt": + skip("Only Unix checks file size") def func(no): - ## if os.name == "nt": - ## skip("Only Unix checks file size") try: mmap.mmap(no, 123) From cfbolz at codespeak.net Sat Oct 13 22:22:45 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 13 Oct 2007 22:22:45 +0200 (CEST) Subject: [pypy-svn] r47433 - in pypy/dist/pypy: rlib rpython/memory rpython/memory/gc rpython/memory/gc/test rpython/memory/gctransform rpython/memory/test Message-ID: <20071013202245.3C5688142@code0.codespeak.net> Author: cfbolz Date: Sat Oct 13 22:22:43 2007 New Revision: 47433 Added: pypy/dist/pypy/rpython/memory/gc/ pypy/dist/pypy/rpython/memory/gc/__init__.py (contents, props changed) pypy/dist/pypy/rpython/memory/gc/base.py (contents, props changed) pypy/dist/pypy/rpython/memory/gc/marksweep.py (contents, props changed) pypy/dist/pypy/rpython/memory/gc/semispace.py (contents, props changed) pypy/dist/pypy/rpython/memory/gc/test/ pypy/dist/pypy/rpython/memory/gc/test/__init__.py (contents, props changed) Removed: pypy/dist/pypy/rpython/memory/gc.py Modified: pypy/dist/pypy/rlib/rgc.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/test/test_gc.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: split the growing gc.py file into one file per GC implementation Modified: pypy/dist/pypy/rlib/rgc.py ============================================================================== --- pypy/dist/pypy/rlib/rgc.py (original) +++ pypy/dist/pypy/rlib/rgc.py Sat Oct 13 22:22:43 2007 @@ -47,7 +47,7 @@ return rtyper.getrepr(s_None) else: from pypy.rpython.rmodel import SimplePointerRepr - from pypy.rpython.memory.gc import X_POOL_PTR + from pypy.rpython.memory.gc.marksweep import X_POOL_PTR return SimplePointerRepr(X_POOL_PTR) @@ -92,7 +92,7 @@ from pypy.rpython.error import TyperError from pypy.rpython.lltypesystem import lltype, llmemory, rtuple from pypy.annotation import model as annmodel - from pypy.rpython.memory.gc import X_CLONE, X_CLONE_PTR + from pypy.rpython.memory.gc.marksweep import X_CLONE, X_CLONE_PTR config = hop.rtyper.getconfig() if config.translation.gc != 'framework': Added: pypy/dist/pypy/rpython/memory/gc/__init__.py ============================================================================== Added: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/memory/gc/base.py Sat Oct 13 22:22:43 2007 @@ -0,0 +1,95 @@ +from pypy.rpython.lltypesystem import lltype, llmemory + +class GCBase(object): + _alloc_flavor_ = "raw" + moving_gc = False + + def set_query_functions(self, is_varsize, getfinalizer, + offsets_to_gc_pointers, + fixed_size, varsize_item_sizes, + varsize_offset_to_variable_part, + varsize_offset_to_length, + varsize_offsets_to_gcpointers_in_var_part, + weakpointer_offset): + self.getfinalizer = getfinalizer + self.is_varsize = is_varsize + self.offsets_to_gc_pointers = offsets_to_gc_pointers + self.fixed_size = fixed_size + self.varsize_item_sizes = varsize_item_sizes + self.varsize_offset_to_variable_part = varsize_offset_to_variable_part + self.varsize_offset_to_length = varsize_offset_to_length + self.varsize_offsets_to_gcpointers_in_var_part = varsize_offsets_to_gcpointers_in_var_part + self.weakpointer_offset = weakpointer_offset + + def write_barrier(self, addr, addr_to, addr_struct): + addr_to.address[0] = addr + + def setup(self): + pass + + def statistics(self, index): + return -1 + + def size_gc_header(self, typeid=0): + return self.gcheaderbuilder.size_gc_header + + def malloc(self, typeid, length=0, zero=False): + """For testing. The interface used by the gctransformer is + the four malloc_[fixed,var]size[_clear]() functions. + """ + size = self.fixed_size(typeid) + needs_finalizer = bool(self.getfinalizer(typeid)) + weakptr_offset = self.weakpointer_offset(typeid) + #XXX cannot compare weakptr_offset with -1 + #contains_weakptr = weakpointer_offset. != -1 + if isinstance(weakptr_offset, int): + assert weakptr_offset == -1 + contains_weakptr = False + else: + contains_weakptr = True + assert not (needs_finalizer and contains_weakptr) + if self.is_varsize(typeid): + assert not contains_weakptr + itemsize = self.varsize_item_sizes(typeid) + offset_to_length = self.varsize_offset_to_length(typeid) + if zero: + malloc_varsize = self.malloc_varsize_clear + else: + malloc_varsize = self.malloc_varsize + ref = malloc_varsize(typeid, length, size, itemsize, + offset_to_length, True, needs_finalizer) + else: + if zero: + malloc_fixedsize = self.malloc_fixedsize_clear + else: + malloc_fixedsize = self.malloc_fixedsize + ref = malloc_fixedsize(typeid, size, True, needs_finalizer, + contains_weakptr) + # lots of cast and reverse-cast around... + return llmemory.cast_ptr_to_adr(ref) + + def x_swap_pool(self, newpool): + return newpool + + def x_clone(self, clonedata): + raise RuntimeError("no support for x_clone in the GC") + + def x_become(self, target_addr, source_addr): + raise RuntimeError("no support for x_become in the GC") + + +def choose_gc_from_config(config): + """Return a (GCClass, GC_PARAMS) from the given config object. + """ + config.translation.gc = "framework" + if config.translation.frameworkgc == "marksweep": + GC_PARAMS = {'start_heap_size': 8*1024*1024} # XXX adjust + from pypy.rpython.memory.gc.marksweep import MarkSweepGC + return MarkSweepGC, GC_PARAMS + elif config.translation.frameworkgc == "semispace": + GC_PARAMS = {'space_size': 8*1024*1024} # XXX adjust + from pypy.rpython.memory.gc.marksweep import SemiSpaceGC + return SemiSpaceGC, GC_PARAMS + else: + raise ValueError("unknown value for frameworkgc: %r" % ( + config.translation.frameworkgc,)) Added: pypy/dist/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/memory/gc/marksweep.py Sat Oct 13 22:22:43 2007 @@ -0,0 +1,872 @@ +from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free +from pypy.rpython.lltypesystem.llmemory import raw_memcopy, raw_memclear +from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage +from pypy.rpython.memory.support import get_address_linked_list +from pypy.rpython.memory.gcheader import GCHeaderBuilder +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rlib.objectmodel import free_non_gc_object, debug_assert +from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rlib.rarithmetic import ovfcheck +from pypy.rpython.memory.gc.base import GCBase + + +import sys, os + +X_POOL = lltype.GcOpaqueType('gc.pool') +X_POOL_PTR = lltype.Ptr(X_POOL) +X_CLONE = lltype.GcStruct('CloneData', ('gcobjectptr', llmemory.GCREF), + ('pool', X_POOL_PTR)) +X_CLONE_PTR = lltype.Ptr(X_CLONE) + +DEBUG_PRINT = False +memoryError = MemoryError() +class MarkSweepGC(GCBase): + _alloc_flavor_ = "raw" + + HDR = lltype.ForwardReference() + HDRPTR = lltype.Ptr(HDR) + # need to maintain a linked list of malloced objects, since we used the + # systems allocator and can't walk the heap + HDR.become(lltype.Struct('header', ('typeid', lltype.Signed), + ('next', HDRPTR))) + + POOL = lltype.GcStruct('gc_pool') + POOLPTR = lltype.Ptr(POOL) + + POOLNODE = lltype.ForwardReference() + POOLNODEPTR = lltype.Ptr(POOLNODE) + POOLNODE.become(lltype.Struct('gc_pool_node', ('linkedlist', HDRPTR), + ('nextnode', POOLNODEPTR))) + + def __init__(self, AddressLinkedList, start_heap_size=4096, get_roots=None): + self.heap_usage = 0 # at the end of the latest collection + self.bytes_malloced = 0 # since the latest collection + self.bytes_malloced_threshold = start_heap_size + self.total_collection_time = 0.0 + self.AddressLinkedList = AddressLinkedList + self.malloced_objects = lltype.nullptr(self.HDR) + self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) + # these are usually only the small bits of memory that make a + # weakref object + self.objects_with_weak_pointers = lltype.nullptr(self.HDR) + self.get_roots = get_roots + self.gcheaderbuilder = GCHeaderBuilder(self.HDR) + # pools, for x_swap_pool(): + # 'curpool' is the current pool, lazily allocated (i.e. NULL means + # the current POOL object is not yet malloc'ed). POOL objects are + # usually at the start of a linked list of objects, via the HDRs. + # The exception is 'curpool' whose linked list of objects is in + # 'self.malloced_objects' instead of in the header of 'curpool'. + # POOL objects are never in the middle of a linked list themselves. + # XXX a likely cause for the current problems with pools is: + # not all objects live in malloced_objects, some also live in + # malloced_objects_with_finalizer and objects_with_weak_pointers + self.curpool = lltype.nullptr(self.POOL) + # 'poolnodes' is a linked list of all such linked lists. Each + # linked list will usually start with a POOL object, but it can + # also contain only normal objects if the POOL object at the head + # was already freed. The objects in 'malloced_objects' are not + # found via 'poolnodes'. + self.poolnodes = lltype.nullptr(self.POOLNODE) + self.collect_in_progress = False + self.prev_collect_end_time = 0.0 + + def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, + contains_weakptr=False): + if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: + self.collect() + size_gc_header = self.gcheaderbuilder.size_gc_header + try: + tot_size = size_gc_header + size + usage = raw_malloc_usage(tot_size) + bytes_malloced = ovfcheck(self.bytes_malloced+usage) + ovfcheck(self.heap_usage + bytes_malloced) + except OverflowError: + raise memoryError + result = raw_malloc(tot_size) + if not result: + raise memoryError + hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) + hdr.typeid = typeid << 1 + if has_finalizer: + hdr.next = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = hdr + elif contains_weakptr: + hdr.next = self.objects_with_weak_pointers + self.objects_with_weak_pointers = hdr + else: + hdr.next = self.malloced_objects + self.malloced_objects = hdr + self.bytes_malloced = bytes_malloced + result += size_gc_header + #llop.debug_print(lltype.Void, 'malloc typeid', typeid, + # '->', llmemory.cast_adr_to_int(result)) + return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + + def malloc_fixedsize_clear(self, typeid, size, can_collect, + has_finalizer=False, contains_weakptr=False): + if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: + self.collect() + size_gc_header = self.gcheaderbuilder.size_gc_header + try: + tot_size = size_gc_header + size + usage = raw_malloc_usage(tot_size) + bytes_malloced = ovfcheck(self.bytes_malloced+usage) + ovfcheck(self.heap_usage + bytes_malloced) + except OverflowError: + raise memoryError + result = raw_malloc(tot_size) + if not result: + raise memoryError + raw_memclear(result, tot_size) + hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) + hdr.typeid = typeid << 1 + if has_finalizer: + hdr.next = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = hdr + elif contains_weakptr: + hdr.next = self.objects_with_weak_pointers + self.objects_with_weak_pointers = hdr + else: + hdr.next = self.malloced_objects + self.malloced_objects = hdr + self.bytes_malloced = bytes_malloced + result += size_gc_header + #llop.debug_print(lltype.Void, 'malloc typeid', typeid, + # '->', llmemory.cast_adr_to_int(result)) + return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + + def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, + can_collect, has_finalizer=False): + if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: + self.collect() + size_gc_header = self.gcheaderbuilder.size_gc_header + try: + fixsize = size_gc_header + size + varsize = ovfcheck(itemsize * length) + tot_size = ovfcheck(fixsize + varsize) + usage = raw_malloc_usage(tot_size) + bytes_malloced = ovfcheck(self.bytes_malloced+usage) + ovfcheck(self.heap_usage + bytes_malloced) + except OverflowError: + raise memoryError + result = raw_malloc(tot_size) + if not result: + raise memoryError + (result + size_gc_header + offset_to_length).signed[0] = length + hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) + hdr.typeid = typeid << 1 + if has_finalizer: + hdr.next = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = hdr + else: + hdr.next = self.malloced_objects + self.malloced_objects = hdr + self.bytes_malloced = bytes_malloced + + result += size_gc_header + #llop.debug_print(lltype.Void, 'malloc_varsize length', length, + # 'typeid', typeid, + # '->', llmemory.cast_adr_to_int(result)) + return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + + def malloc_varsize_clear(self, typeid, length, size, itemsize, + offset_to_length, can_collect, + has_finalizer=False): + if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: + self.collect() + size_gc_header = self.gcheaderbuilder.size_gc_header + try: + fixsize = size_gc_header + size + varsize = ovfcheck(itemsize * length) + tot_size = ovfcheck(fixsize + varsize) + usage = raw_malloc_usage(tot_size) + bytes_malloced = ovfcheck(self.bytes_malloced+usage) + ovfcheck(self.heap_usage + bytes_malloced) + except OverflowError: + raise memoryError + result = raw_malloc(tot_size) + if not result: + raise memoryError + raw_memclear(result, tot_size) + (result + size_gc_header + offset_to_length).signed[0] = length + hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) + hdr.typeid = typeid << 1 + if has_finalizer: + hdr.next = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = hdr + else: + hdr.next = self.malloced_objects + self.malloced_objects = hdr + self.bytes_malloced = bytes_malloced + + result += size_gc_header + #llop.debug_print(lltype.Void, 'malloc_varsize length', length, + # 'typeid', typeid, + # '->', llmemory.cast_adr_to_int(result)) + return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + + def collect(self): + # 1. mark from the roots, and also the objects that objects-with-del + # point to (using the list of malloced_objects_with_finalizer) + # 2. walk the list of objects-without-del and free the ones not marked + # 3. walk the list of objects-with-del and for the ones not marked: + # call __del__, move the object to the list of object-without-del + import time + from pypy.rpython.lltypesystem.lloperation import llop + if DEBUG_PRINT: + llop.debug_print(lltype.Void, 'collecting...') + start_time = time.time() + self.collect_in_progress = True + roots = self.get_roots() + size_gc_header = self.gcheaderbuilder.size_gc_header +## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, +## size_gc_header) + + # push the roots on the mark stack + objects = self.AddressLinkedList() # mark stack + while 1: + curr = roots.pop() + if curr == NULL: + break + # roots is a list of addresses to addresses: + objects.append(curr.address[0]) + free_non_gc_object(roots) + # from this point onwards, no more mallocs should be possible + old_malloced = self.bytes_malloced + self.bytes_malloced = 0 + curr_heap_size = 0 + freed_size = 0 + + # mark objects reachable by objects with a finalizer, but not those + # themselves. add their size to curr_heap_size, since they always + # survive the collection + hdr = self.malloced_objects_with_finalizer + while hdr: + next = hdr.next + typeid = hdr.typeid >> 1 + gc_info = llmemory.cast_ptr_to_adr(hdr) + obj = gc_info + size_gc_header + if not hdr.typeid & 1: + self.add_reachable_to_stack(obj, objects) + addr = llmemory.cast_ptr_to_adr(hdr) + size = self.fixed_size(typeid) + if self.is_varsize(typeid): + length = (obj + self.varsize_offset_to_length(typeid)).signed[0] + size += self.varsize_item_sizes(typeid) * length + estimate = raw_malloc_usage(size_gc_header + size) + curr_heap_size += estimate + hdr = next + + # mark thinks on the mark stack and put their descendants onto the + # stack until the stack is empty + while objects.non_empty(): #mark + curr = objects.pop() + gc_info = curr - size_gc_header + hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) + if hdr.typeid & 1: + continue + self.add_reachable_to_stack(curr, objects) + hdr.typeid = hdr.typeid | 1 + objects.delete() + # also mark self.curpool + if self.curpool: + gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header + hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) + hdr.typeid = hdr.typeid | 1 + # go through the list of objects containing weak pointers + # and kill the links if they go to dead objects + # if the object itself is not marked, free it + hdr = self.objects_with_weak_pointers + surviving = lltype.nullptr(self.HDR) + while hdr: + typeid = hdr.typeid >> 1 + next = hdr.next + addr = llmemory.cast_ptr_to_adr(hdr) + size = self.fixed_size(typeid) + estimate = raw_malloc_usage(size_gc_header + size) + if hdr.typeid & 1: + typeid = hdr.typeid >> 1 + offset = self.weakpointer_offset(typeid) + hdr.typeid = hdr.typeid & (~1) + gc_info = llmemory.cast_ptr_to_adr(hdr) + weakref_obj = gc_info + size_gc_header + pointing_to = (weakref_obj + offset).address[0] + if pointing_to: + gc_info_pointing_to = pointing_to - size_gc_header + hdr_pointing_to = llmemory.cast_adr_to_ptr( + gc_info_pointing_to, self.HDRPTR) + # pointed to object will die + # XXX what to do if the object has a finalizer which resurrects + # the object? + if not hdr_pointing_to.typeid & 1: + (weakref_obj + offset).address[0] = NULL + hdr.next = surviving + surviving = hdr + curr_heap_size += estimate + else: + freed_size += estimate + raw_free(addr) + hdr = next + self.objects_with_weak_pointers = surviving + # sweep: delete objects without del if they are not marked + # unmark objects without del that are marked + firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw') + firstpoolnode.linkedlist = self.malloced_objects + firstpoolnode.nextnode = self.poolnodes + prevpoolnode = lltype.nullptr(self.POOLNODE) + poolnode = firstpoolnode + while poolnode: #sweep + ppnext = llmemory.cast_ptr_to_adr(poolnode) + ppnext += llmemory.offsetof(self.POOLNODE, 'linkedlist') + hdr = poolnode.linkedlist + while hdr: #sweep + typeid = hdr.typeid >> 1 + next = hdr.next + addr = llmemory.cast_ptr_to_adr(hdr) + size = self.fixed_size(typeid) + if self.is_varsize(typeid): + length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0] + size += self.varsize_item_sizes(typeid) * length + estimate = raw_malloc_usage(size_gc_header + size) + if hdr.typeid & 1: + hdr.typeid = hdr.typeid & (~1) + ppnext.address[0] = addr + ppnext = llmemory.cast_ptr_to_adr(hdr) + ppnext += llmemory.offsetof(self.HDR, 'next') + curr_heap_size += estimate + else: + freed_size += estimate + raw_free(addr) + hdr = next + ppnext.address[0] = llmemory.NULL + next = poolnode.nextnode + if not poolnode.linkedlist and prevpoolnode: + # completely empty node + prevpoolnode.nextnode = next + lltype.free(poolnode, flavor='raw') + else: + prevpoolnode = poolnode + poolnode = next + self.malloced_objects = firstpoolnode.linkedlist + self.poolnodes = firstpoolnode.nextnode + lltype.free(firstpoolnode, flavor='raw') + #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) + + end_time = time.time() + compute_time = start_time - self.prev_collect_end_time + collect_time = end_time - start_time + + garbage_collected = old_malloced - (curr_heap_size - self.heap_usage) + + if (collect_time * curr_heap_size > + 0.02 * garbage_collected * compute_time): + self.bytes_malloced_threshold += self.bytes_malloced_threshold / 2 + if (collect_time * curr_heap_size < + 0.005 * garbage_collected * compute_time): + self.bytes_malloced_threshold /= 2 + + # Use atleast as much memory as current live objects. + if curr_heap_size > self.bytes_malloced_threshold: + self.bytes_malloced_threshold = curr_heap_size + + # Cap at 1/4 GB + self.bytes_malloced_threshold = min(self.bytes_malloced_threshold, + 256 * 1024 * 1024) + self.total_collection_time += collect_time + self.prev_collect_end_time = end_time + if DEBUG_PRINT: + llop.debug_print(lltype.Void, + " malloced since previous collection:", + old_malloced, "bytes") + llop.debug_print(lltype.Void, + " heap usage at start of collection: ", + self.heap_usage + old_malloced, "bytes") + llop.debug_print(lltype.Void, + " freed: ", + freed_size, "bytes") + llop.debug_print(lltype.Void, + " new heap usage: ", + curr_heap_size, "bytes") + llop.debug_print(lltype.Void, + " total time spent collecting: ", + self.total_collection_time, "seconds") + llop.debug_print(lltype.Void, + " collecting time: ", + collect_time) + llop.debug_print(lltype.Void, + " computing time: ", + collect_time) + llop.debug_print(lltype.Void, + " new threshold: ", + self.bytes_malloced_threshold) +## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, +## size_gc_header) + assert self.heap_usage + old_malloced == curr_heap_size + freed_size + + self.heap_usage = curr_heap_size + hdr = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) + last = lltype.nullptr(self.HDR) + while hdr: + next = hdr.next + if hdr.typeid & 1: + hdr.next = lltype.nullptr(self.HDR) + if not self.malloced_objects_with_finalizer: + self.malloced_objects_with_finalizer = hdr + else: + last.next = hdr + hdr.typeid = hdr.typeid & (~1) + last = hdr + else: + obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header + finalizer = self.getfinalizer(hdr.typeid >> 1) + # make malloced_objects_with_finalizer consistent + # for the sake of a possible collection caused by finalizer + if not self.malloced_objects_with_finalizer: + self.malloced_objects_with_finalizer = next + else: + last.next = next + hdr.next = self.malloced_objects + self.malloced_objects = hdr + #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) + finalizer(obj) + if not self.collect_in_progress: # another collection was caused? + llop.debug_print(lltype.Void, "outer collect interrupted " + "by recursive collect") + return + if not last: + if self.malloced_objects_with_finalizer == next: + self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) + else: + # now it gets annoying: finalizer caused a malloc of something + # with a finalizer + last = self.malloced_objects_with_finalizer + while last.next != next: + last = last.next + last.next = lltype.nullptr(self.HDR) + else: + last.next = lltype.nullptr(self.HDR) + hdr = next + self.collect_in_progress = False + + STAT_HEAP_USAGE = 0 + STAT_BYTES_MALLOCED = 1 + STATISTICS_NUMBERS = 2 + + def add_reachable_to_stack(self, obj, objects): + size_gc_header = self.gcheaderbuilder.size_gc_header + gc_info = obj - size_gc_header + hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) + typeid = hdr.typeid >> 1 + offsets = self.offsets_to_gc_pointers(typeid) + i = 0 + while i < len(offsets): + pointer = obj + offsets[i] + objects.append(pointer.address[0]) + i += 1 + if self.is_varsize(typeid): + offset = self.varsize_offset_to_variable_part( + typeid) + length = (obj + self.varsize_offset_to_length(typeid)).signed[0] + obj += offset + offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) + itemlength = self.varsize_item_sizes(typeid) + i = 0 + while i < length: + item = obj + itemlength * i + j = 0 + while j < len(offsets): + pointer = item + offsets[j] + objects.append(pointer.address[0]) + j += 1 + i += 1 + + def statistics(self, index): + # no memory allocation here! + if index == self.STAT_HEAP_USAGE: + return self.heap_usage + if index == self.STAT_BYTES_MALLOCED: + return self.bytes_malloced + return -1 + + def init_gc_object(self, addr, typeid): + hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) + hdr.typeid = typeid << 1 + + def init_gc_object_immortal(self, addr, typeid): + # prebuilt gc structures always have the mark bit set + hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) + hdr.typeid = (typeid << 1) | 1 + + # experimental support for thread cloning + def x_swap_pool(self, newpool): + # Set newpool as the current pool (create one if newpool == NULL). + # All malloc'ed objects are put into the current pool;this is a + # way to separate objects depending on when they were allocated. + size_gc_header = self.gcheaderbuilder.size_gc_header + # invariant: each POOL GcStruct is at the _front_ of a linked list + # of malloced objects. + oldpool = self.curpool + #llop.debug_print(lltype.Void, 'x_swap_pool', + # lltype.cast_ptr_to_int(oldpool), + # lltype.cast_ptr_to_int(newpool)) + if not oldpool: + # make a fresh pool object, which is automatically inserted at the + # front of the current list + oldpool = lltype.malloc(self.POOL) + addr = llmemory.cast_ptr_to_adr(oldpool) + addr -= size_gc_header + hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) + # put this new POOL object in the poolnodes list + node = lltype.malloc(self.POOLNODE, flavor="raw") + node.linkedlist = hdr + node.nextnode = self.poolnodes + self.poolnodes = node + else: + # manually insert oldpool at the front of the current list + addr = llmemory.cast_ptr_to_adr(oldpool) + addr -= size_gc_header + hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) + hdr.next = self.malloced_objects + + newpool = lltype.cast_opaque_ptr(self.POOLPTR, newpool) + if newpool: + # newpool is at the front of the new linked list to install + addr = llmemory.cast_ptr_to_adr(newpool) + addr -= size_gc_header + hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) + self.malloced_objects = hdr.next + # invariant: now that objects in the hdr.next list are accessible + # through self.malloced_objects, make sure they are not accessible + # via poolnodes (which has a node pointing to newpool): + hdr.next = lltype.nullptr(self.HDR) + else: + # start a fresh new linked list + self.malloced_objects = lltype.nullptr(self.HDR) + self.curpool = newpool + return lltype.cast_opaque_ptr(X_POOL_PTR, oldpool) + + def x_clone(self, clonedata): + # Recursively clone the gcobject and everything it points to, + # directly or indirectly -- but stops at objects that are not + # in the specified pool. A new pool is built to contain the + # copies, and the 'gcobjectptr' and 'pool' fields of clonedata + # are adjusted to refer to the result. + CURPOOL_FLAG = sys.maxint // 2 + 1 + + # install a new pool into which all the mallocs go + curpool = self.x_swap_pool(lltype.nullptr(X_POOL)) + + size_gc_header = self.gcheaderbuilder.size_gc_header + oldobjects = self.AddressLinkedList() + # if no pool specified, use the current pool as the 'source' pool + oldpool = clonedata.pool or curpool + oldpool = lltype.cast_opaque_ptr(self.POOLPTR, oldpool) + addr = llmemory.cast_ptr_to_adr(oldpool) + addr -= size_gc_header + + hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) + hdr = hdr.next # skip the POOL object itself + while hdr: + next = hdr.next + hdr.typeid |= CURPOOL_FLAG # mark all objects from malloced_list + hdr.next = lltype.nullptr(self.HDR) # abused to point to the copy + oldobjects.append(llmemory.cast_ptr_to_adr(hdr)) + hdr = next + + # a stack of addresses of places that still points to old objects + # and that must possibly be fixed to point to a new copy + stack = self.AddressLinkedList() + stack.append(llmemory.cast_ptr_to_adr(clonedata) + + llmemory.offsetof(X_CLONE, 'gcobjectptr')) + while stack.non_empty(): + gcptr_addr = stack.pop() + oldobj_addr = gcptr_addr.address[0] + if not oldobj_addr: + continue # pointer is NULL + oldhdr = llmemory.cast_adr_to_ptr(oldobj_addr - size_gc_header, + self.HDRPTR) + typeid = oldhdr.typeid + if not (typeid & CURPOOL_FLAG): + continue # ignore objects that were not in the malloced_list + newhdr = oldhdr.next # abused to point to the copy + if not newhdr: + typeid = (typeid & ~CURPOOL_FLAG) >> 1 + size = self.fixed_size(typeid) + # XXX! collect() at the beginning if the free heap is low + if self.is_varsize(typeid): + itemsize = self.varsize_item_sizes(typeid) + offset_to_length = self.varsize_offset_to_length(typeid) + length = (oldobj_addr + offset_to_length).signed[0] + newobj = self.malloc_varsize(typeid, length, size, + itemsize, offset_to_length, + False) + size += length*itemsize + else: + newobj = self.malloc_fixedsize(typeid, size, False) + length = -1 + + newobj_addr = llmemory.cast_ptr_to_adr(newobj) + + #llop.debug_print(lltype.Void, 'clone', + # llmemory.cast_adr_to_int(oldobj_addr), + # '->', llmemory.cast_adr_to_int(newobj_addr), + # 'typeid', typeid, + # 'length', length) + + newhdr_addr = newobj_addr - size_gc_header + newhdr = llmemory.cast_adr_to_ptr(newhdr_addr, self.HDRPTR) + + saved_id = newhdr.typeid # XXX hack needed for genc + saved_next = newhdr.next # where size_gc_header == 0 + raw_memcopy(oldobj_addr, newobj_addr, size) + newhdr.typeid = saved_id + newhdr.next = saved_next + + offsets = self.offsets_to_gc_pointers(typeid) + i = 0 + while i < len(offsets): + pointer_addr = newobj_addr + offsets[i] + stack.append(pointer_addr) + i += 1 + + if length > 0: + offsets = self.varsize_offsets_to_gcpointers_in_var_part( + typeid) + itemlength = self.varsize_item_sizes(typeid) + offset = self.varsize_offset_to_variable_part(typeid) + itembaseaddr = newobj_addr + offset + i = 0 + while i < length: + item = itembaseaddr + itemlength * i + j = 0 + while j < len(offsets): + pointer_addr = item + offsets[j] + stack.append(pointer_addr) + j += 1 + i += 1 + + oldhdr.next = newhdr + newobj_addr = llmemory.cast_ptr_to_adr(newhdr) + size_gc_header + gcptr_addr.address[0] = newobj_addr + stack.delete() + + # re-create the original linked list + next = lltype.nullptr(self.HDR) + while oldobjects.non_empty(): + hdr = llmemory.cast_adr_to_ptr(oldobjects.pop(), self.HDRPTR) + hdr.typeid &= ~CURPOOL_FLAG # reset the flag + hdr.next = next + next = hdr + oldobjects.delete() + + # consistency check + addr = llmemory.cast_ptr_to_adr(oldpool) + addr -= size_gc_header + hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) + assert hdr.next == next + + # build the new pool object collecting the new objects, and + # reinstall the pool that was current at the beginning of x_clone() + clonedata.pool = self.x_swap_pool(curpool) + + def add_reachable_to_stack2(self, obj, objects, target_addr, source_addr): + size_gc_header = self.gcheaderbuilder.size_gc_header + gc_info = obj - size_gc_header + hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) + if hdr.typeid & 1: + return + typeid = hdr.typeid >> 1 + offsets = self.offsets_to_gc_pointers(typeid) + i = 0 + while i < len(offsets): + pointer = obj + offsets[i] + # ------------------------------------------------- + # begin difference from collect + if pointer.address[0] == target_addr: + pointer.address[0] = source_addr + # end difference from collect + # ------------------------------------------------- + objects.append(pointer.address[0]) + i += 1 + if self.is_varsize(typeid): + offset = self.varsize_offset_to_variable_part( + typeid) + length = (obj + self.varsize_offset_to_length(typeid)).signed[0] + obj += offset + offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) + itemlength = self.varsize_item_sizes(typeid) + i = 0 + while i < length: + item = obj + itemlength * i + j = 0 + while j < len(offsets): + pointer = item + offsets[j] + # ------------------------------------------------- + # begin difference from collect + if pointer.address[0] == target_addr: + pointer.address[0] = source_addr + ## end difference from collect + # ------------------------------------------------- + objects.append(pointer.address[0]) + j += 1 + i += 1 + + + def x_become(self, target_addr, source_addr): + # 1. mark from the roots, and also the objects that objects-with-del + # point to (using the list of malloced_objects_with_finalizer) + # 2. walk the list of objects-without-del and free the ones not marked + # 3. walk the list of objects-with-del and for the ones not marked: + # call __del__, move the object to the list of object-without-del + import time + from pypy.rpython.lltypesystem.lloperation import llop + if DEBUG_PRINT: + llop.debug_print(lltype.Void, 'collecting...') + start_time = time.time() + roots = self.get_roots() + size_gc_header = self.gcheaderbuilder.size_gc_header +## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, +## size_gc_header) + + # push the roots on the mark stack + objects = self.AddressLinkedList() # mark stack + while 1: + curr = roots.pop() + if curr == NULL: + break + # roots is a list of addresses to addresses: + objects.append(curr.address[0]) + # the last sweep did not clear the mark bit of static roots, + # since they are not in the malloced_objects list + gc_info = curr.address[0] - size_gc_header + hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) + hdr.typeid = hdr.typeid & (~1) + free_non_gc_object(roots) + # from this point onwards, no more mallocs should be possible + old_malloced = self.bytes_malloced + self.bytes_malloced = 0 + curr_heap_size = 0 + freed_size = 0 + + # mark objects reachable by objects with a finalizer, but not those + # themselves. add their size to curr_heap_size, since they always + # survive the collection + hdr = self.malloced_objects_with_finalizer + while hdr: + next = hdr.next + typeid = hdr.typeid >> 1 + gc_info = llmemory.cast_ptr_to_adr(hdr) + obj = gc_info + size_gc_header + self.add_reachable_to_stack2(obj, objects, target_addr, source_addr) + addr = llmemory.cast_ptr_to_adr(hdr) + size = self.fixed_size(typeid) + if self.is_varsize(typeid): + length = (obj + self.varsize_offset_to_length(typeid)).signed[0] + size += self.varsize_item_sizes(typeid) * length + estimate = raw_malloc_usage(size_gc_header + size) + curr_heap_size += estimate + hdr = next + + # mark thinks on the mark stack and put their descendants onto the + # stack until the stack is empty + while objects.non_empty(): #mark + curr = objects.pop() + self.add_reachable_to_stack2(curr, objects, target_addr, source_addr) + gc_info = curr - size_gc_header + hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) + if hdr.typeid & 1: + continue + hdr.typeid = hdr.typeid | 1 + objects.delete() + # also mark self.curpool + if self.curpool: + gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header + hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) + hdr.typeid = hdr.typeid | 1 + + # sweep: delete objects without del if they are not marked + # unmark objects without del that are marked + firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw') + firstpoolnode.linkedlist = self.malloced_objects + firstpoolnode.nextnode = self.poolnodes + prevpoolnode = lltype.nullptr(self.POOLNODE) + poolnode = firstpoolnode + while poolnode: #sweep + ppnext = llmemory.cast_ptr_to_adr(poolnode) + ppnext += llmemory.offsetof(self.POOLNODE, 'linkedlist') + hdr = poolnode.linkedlist + while hdr: #sweep + typeid = hdr.typeid >> 1 + next = hdr.next + addr = llmemory.cast_ptr_to_adr(hdr) + size = self.fixed_size(typeid) + if self.is_varsize(typeid): + length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0] + size += self.varsize_item_sizes(typeid) * length + estimate = raw_malloc_usage(size_gc_header + size) + if hdr.typeid & 1: + hdr.typeid = hdr.typeid & (~1) + ppnext.address[0] = addr + ppnext = llmemory.cast_ptr_to_adr(hdr) + ppnext += llmemory.offsetof(self.HDR, 'next') + curr_heap_size += estimate + else: + freed_size += estimate + raw_free(addr) + hdr = next + ppnext.address[0] = llmemory.NULL + next = poolnode.nextnode + if not poolnode.linkedlist and prevpoolnode: + # completely empty node + prevpoolnode.nextnode = next + lltype.free(poolnode, flavor='raw') + else: + prevpoolnode = poolnode + poolnode = next + self.malloced_objects = firstpoolnode.linkedlist + self.poolnodes = firstpoolnode.nextnode + lltype.free(firstpoolnode, flavor='raw') + + if curr_heap_size > self.bytes_malloced_threshold: + self.bytes_malloced_threshold = curr_heap_size + end_time = time.time() + self.total_collection_time += end_time - start_time + if DEBUG_PRINT: + llop.debug_print(lltype.Void, + " malloced since previous collection:", + old_malloced, "bytes") + llop.debug_print(lltype.Void, + " heap usage at start of collection: ", + self.heap_usage + old_malloced, "bytes") + llop.debug_print(lltype.Void, + " freed: ", + freed_size, "bytes") + llop.debug_print(lltype.Void, + " new heap usage: ", + curr_heap_size, "bytes") + llop.debug_print(lltype.Void, + " total time spent collecting: ", + self.total_collection_time, "seconds") +## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, +## size_gc_header) + assert self.heap_usage + old_malloced == curr_heap_size + freed_size + + # call finalizers if needed + self.heap_usage = curr_heap_size + hdr = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) + while hdr: + next = hdr.next + if hdr.typeid & 1: + hdr.next = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = hdr + hdr.typeid = hdr.typeid & (~1) + else: + obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header + finalizer = self.getfinalizer(hdr.typeid >> 1) + finalizer(obj) + hdr.next = self.malloced_objects + self.malloced_objects = hdr + hdr = next Added: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Sat Oct 13 22:22:43 2007 @@ -0,0 +1,342 @@ +from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free +from pypy.rpython.lltypesystem.llmemory import raw_memcopy, raw_memclear +from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage +from pypy.rpython.memory.support import get_address_linked_list +from pypy.rpython.memory.gcheader import GCHeaderBuilder +from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rlib.objectmodel import free_non_gc_object, debug_assert +from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rlib.rarithmetic import ovfcheck +from pypy.rpython.memory.gc.base import GCBase + + +import sys, os + +memoryError = MemoryError() + +class SemiSpaceGC(GCBase): + _alloc_flavor_ = "raw" + moving_gc = True + inline_simple_malloc = True + + HDR = lltype.Struct('header', ('forw', llmemory.Address), + ('typeid', lltype.Signed)) + + def __init__(self, AddressLinkedList, space_size=4096, + max_space_size=sys.maxint//2+1, + get_roots=None): + self.space_size = space_size + self.max_space_size = max_space_size + self.get_roots = get_roots + self.gcheaderbuilder = GCHeaderBuilder(self.HDR) + self.AddressLinkedList = AddressLinkedList + + def setup(self): + self.tospace = llarena.arena_malloc(self.space_size, True) + debug_assert(bool(self.tospace), "couldn't allocate tospace") + self.top_of_space = self.tospace + self.space_size + self.fromspace = llarena.arena_malloc(self.space_size, True) + debug_assert(bool(self.fromspace), "couldn't allocate fromspace") + self.free = self.tospace + self.objects_with_finalizers = self.AddressLinkedList() + self.run_finalizers = self.AddressLinkedList() + self.executing_finalizers = False + self.objects_with_weakrefs = self.AddressLinkedList() + + def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, + contains_weakptr=False): + size_gc_header = self.gcheaderbuilder.size_gc_header + totalsize = size_gc_header + size + result = self.free + if raw_malloc_usage(totalsize) > self.top_of_space - result: + if not can_collect: + raise memoryError + result = self.obtain_free_space(totalsize) + llarena.arena_reserve(result, totalsize) + self.init_gc_object(result, typeid) + self.free = result + totalsize + if has_finalizer: + self.objects_with_finalizers.append(result + size_gc_header) + if contains_weakptr: + self.objects_with_weakrefs.append(result + size_gc_header) + return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) + + def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, + can_collect, has_finalizer=False): + size_gc_header = self.gcheaderbuilder.size_gc_header + nonvarsize = size_gc_header + size + try: + varsize = ovfcheck(itemsize * length) + totalsize = ovfcheck(nonvarsize + varsize) + except OverflowError: + raise memoryError + result = self.free + if raw_malloc_usage(totalsize) > self.top_of_space - result: + if not can_collect: + raise memoryError + result = self.obtain_free_space(totalsize) + llarena.arena_reserve(result, totalsize) + self.init_gc_object(result, typeid) + (result + size_gc_header + offset_to_length).signed[0] = length + self.free = result + llarena.round_up_for_allocation(totalsize) + if has_finalizer: + self.objects_with_finalizers.append(result + size_gc_header) + return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) + + # for now, the spaces are filled with zeroes in advance + malloc_fixedsize_clear = malloc_fixedsize + malloc_varsize_clear = malloc_varsize + + def obtain_free_space(self, needed): + # a bit of tweaking to maximize the performance and minimize the + # amount of code in an inlined version of malloc_fixedsize() + if not self.try_obtain_free_space(needed): + raise memoryError + return self.free + obtain_free_space.dont_inline = True + + def try_obtain_free_space(self, needed): + # XXX for bonus points do big objects differently + needed = raw_malloc_usage(needed) + self.semispace_collect() + missing = needed - (self.top_of_space - self.free) + if missing <= 0: + return True # success + else: + # first check if the object could possibly fit + proposed_size = self.space_size + while missing > 0: + if proposed_size >= self.max_space_size: + return False # no way + missing -= proposed_size + proposed_size *= 2 + # For address space fragmentation reasons, we double the space + # size possibly several times, moving the objects at each step, + # instead of going directly for the final size. We assume that + # it's a rare case anyway. + while self.space_size < proposed_size: + if not self.double_space_size(): + return False + debug_assert(needed <= self.top_of_space - self.free, + "double_space_size() failed to do its job") + return True + + def double_space_size(self): + old_fromspace = self.fromspace + newsize = self.space_size * 2 + newspace = llarena.arena_malloc(newsize, True) + if not newspace: + return False # out of memory + llarena.arena_free(old_fromspace) + self.fromspace = newspace + # now self.tospace contains the existing objects and + # self.fromspace is the freshly allocated bigger space + + self.semispace_collect(size_changing=True) + self.top_of_space = self.tospace + newsize + # now self.tospace is the freshly allocated bigger space, + # and self.fromspace is the old smaller space, now empty + llarena.arena_free(self.fromspace) + + newspace = llarena.arena_malloc(newsize, True) + if not newspace: + # Complex failure case: we have in self.tospace a big chunk + # of memory, and the two smaller original spaces are already gone. + # Unsure if it's worth these efforts, but we can artificially + # split self.tospace in two again... + self.max_space_size = self.space_size # don't try to grow again, + # because doing arena_free(self.fromspace) would crash + self.fromspace = self.tospace + self.space_size + self.top_of_space = self.fromspace + debug_assert(self.free <= self.top_of_space, + "unexpected growth of GC space usage during collect") + return False # out of memory + + self.fromspace = newspace + self.space_size = newsize + return True # success + + def collect(self): + self.semispace_collect() + # the indirection is required by the fact that collect() is referred + # to by the gc transformer, and the default argument would crash + + def semispace_collect(self, size_changing=False): + #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) + tospace = self.fromspace + fromspace = self.tospace + self.fromspace = fromspace + self.tospace = tospace + self.top_of_space = tospace + self.space_size + scan = self.free = tospace + self.collect_roots() + scan = self.scan_copied(scan) + if self.objects_with_weakrefs.non_empty(): + self.invalidate_weakrefs() + if self.run_finalizers.non_empty(): + self.update_run_finalizers() + if self.objects_with_finalizers.non_empty(): + self.deal_with_objects_with_finalizers() + scan = self.scan_copied(scan) + if not size_changing: + llarena.arena_reset(fromspace, self.space_size, True) + self.execute_finalizers() + + def scan_copied(self, scan): + while scan < self.free: + curr = scan + self.size_gc_header() + self.trace_and_copy(curr) + scan += self.size_gc_header() + self.get_size(curr) + return scan + + def collect_roots(self): + roots = self.get_roots() + while 1: + root = roots.pop() + if root == NULL: + break + root.address[0] = self.copy(root.address[0]) + free_non_gc_object(roots) + + def copy(self, obj): + # Objects not living the GC heap have all been initialized by + # setting their 'forw' address so that it points to themselves. + # The logic below will thus simply return 'obj' if 'obj' is prebuilt. +## print "copying regularly", obj, + if self.is_forwarded(obj): +## print "already copied to", self.get_forwarding_address(obj) + return self.get_forwarding_address(obj) + else: + newaddr = self.free + totalsize = self.size_gc_header() + self.get_size(obj) + llarena.arena_reserve(newaddr, totalsize) + raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize) + self.free += totalsize + newobj = newaddr + self.size_gc_header() +## print "to", newobj + self.set_forwarding_address(obj, newobj) + return newobj + + def trace_and_copy(self, obj): + gc_info = self.header(obj) + typeid = gc_info.typeid + offsets = self.offsets_to_gc_pointers(typeid) + i = 0 + while i < len(offsets): + pointer = obj + offsets[i] + if pointer.address[0] != NULL: + pointer.address[0] = self.copy(pointer.address[0]) + i += 1 + if self.is_varsize(typeid): + offset = self.varsize_offset_to_variable_part( + typeid) + length = (obj + self.varsize_offset_to_length(typeid)).signed[0] + offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) + itemlength = self.varsize_item_sizes(typeid) + i = 0 + while i < length: + item = obj + offset + itemlength * i + j = 0 + while j < len(offsets): + pointer = item + offsets[j] + if pointer.address[0] != NULL: + pointer.address[0] = self.copy(pointer.address[0]) + j += 1 + i += 1 + + def is_forwarded(self, obj): + return self.header(obj).forw != NULL + + def get_forwarding_address(self, obj): + return self.header(obj).forw + + def set_forwarding_address(self, obj, newobj): + gc_info = self.header(obj) + gc_info.forw = newobj + + def get_size(self, obj): + typeid = self.header(obj).typeid + size = self.fixed_size(typeid) + if self.is_varsize(typeid): + lenaddr = obj + self.varsize_offset_to_length(typeid) + length = lenaddr.signed[0] + size += length * self.varsize_item_sizes(typeid) + size = llarena.round_up_for_allocation(size) + return size + + def header(self, addr): + addr -= self.gcheaderbuilder.size_gc_header + return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) + + def init_gc_object(self, addr, typeid): + hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) + #hdr.forw = NULL -- unneeded, the space is initially filled with zero + hdr.typeid = typeid + + def init_gc_object_immortal(self, addr, typeid): + # immortal objects always have forward to themselves + hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) + hdr.forw = addr + self.gcheaderbuilder.size_gc_header + hdr.typeid = typeid + + def deal_with_objects_with_finalizers(self): + # walk over list of objects with finalizers + # if it is not copied, add it to the list of to-be-called finalizers + # and copy it, to me make the finalizer runnable + new_with_finalizer = self.AddressLinkedList() + while self.objects_with_finalizers.non_empty(): + obj = self.objects_with_finalizers.pop() + if self.is_forwarded(obj): + new_with_finalizer.append(self.get_forwarding_address(obj)) + else: + self.run_finalizers.append(self.copy(obj)) + self.objects_with_finalizers.delete() + self.objects_with_finalizers = new_with_finalizer + + def invalidate_weakrefs(self): + # walk over list of objects that contain weakrefs + # if the object it references survives then update the weakref + # otherwise invalidate the weakref + new_with_weakref = self.AddressLinkedList() + while self.objects_with_weakrefs.non_empty(): + obj = self.objects_with_weakrefs.pop() + if not self.is_forwarded(obj): + continue # weakref itself dies + obj = self.get_forwarding_address(obj) + offset = self.weakpointer_offset(self.header(obj).typeid) + pointing_to = (obj + offset).address[0] + if pointing_to: + if self.is_forwarded(pointing_to): + (obj + offset).address[0] = self.get_forwarding_address( + pointing_to) + new_with_weakref.append(obj) + else: + (obj + offset).address[0] = NULL + self.objects_with_weakrefs.delete() + self.objects_with_weakrefs = new_with_weakref + + def update_run_finalizers(self): + # we are in an inner collection, caused by a finalizer + # the run_finalizers objects need to be copied + new_run_finalizer = self.AddressLinkedList() + while self.run_finalizers.non_empty(): + obj = self.run_finalizers.pop() + new_run_finalizer.append(self.copy(obj)) + self.run_finalizers.delete() + self.run_finalizers = new_run_finalizer + + def execute_finalizers(self): + if self.executing_finalizers: + return # the outer invocation of execute_finalizers() will do it + self.executing_finalizers = True + try: + while self.run_finalizers.non_empty(): + obj = self.run_finalizers.pop() + hdr = self.header(obj) + finalizer = self.getfinalizer(hdr.typeid) + finalizer(obj) + finally: + self.executing_finalizers = False + + STATISTICS_NUMBERS = 0 + Added: pypy/dist/pypy/rpython/memory/gc/test/__init__.py ============================================================================== Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Sat Oct 13 22:22:43 2007 @@ -3,7 +3,8 @@ get_rtti, ll_call_destructor, type_contains_pyobjs from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython import rmodel -from pypy.rpython.memory import gc, gctypelayout +from pypy.rpython.memory import gctypelayout +from pypy.rpython.memory.gc import marksweep from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rlib.rarithmetic import ovfcheck from pypy.rlib.objectmodel import debug_assert @@ -33,12 +34,12 @@ def __init__(self, translator): from pypy.rpython.memory.support import get_address_linked_list - from pypy.rpython.memory.gc import choose_gc_from_config + from pypy.rpython.memory.gc.base import choose_gc_from_config super(FrameworkGCTransformer, self).__init__(translator, inline=True) AddressLinkedList = get_address_linked_list() if hasattr(self, 'GC_PARAMS'): # for tests: the GC choice can be specified as class attributes - from pypy.rpython.memory.gc import MarkSweepGC + from pypy.rpython.memory.gc.marksweep import MarkSweepGC GCClass = getattr(self, 'GCClass', MarkSweepGC) GC_PARAMS = self.GC_PARAMS else: @@ -242,8 +243,8 @@ annmodel.SomeInteger()) # experimental gc_x_* operations - s_x_pool = annmodel.SomePtr(gc.X_POOL_PTR) - s_x_clone = annmodel.SomePtr(gc.X_CLONE_PTR) + s_x_pool = annmodel.SomePtr(marksweep.X_POOL_PTR) + s_x_clone = annmodel.SomePtr(marksweep.X_CLONE_PTR) # the x_*() methods use some regular mallocs that must be # transformed in the normal way self.x_swap_pool_ptr = getfn(GCClass.x_swap_pool.im_func, Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Sat Oct 13 22:22:43 2007 @@ -253,10 +253,10 @@ class TestMarkSweepGC(GCTest): - from pypy.rpython.memory.gc import MarkSweepGC as GCClass + from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass class TestSemiSpaceGC(GCTest): - from pypy.rpython.memory.gc import SemiSpaceGC as GCClass + from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass class TestGrowingSemiSpaceGC(TestSemiSpaceGC): GC_PARAMS = {'space_size': 64} Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Sat Oct 13 22:22:43 2007 @@ -7,7 +7,7 @@ from pypy.rpython.memory.gctransform import framework from pypy.rpython.memory.gctransform import stacklessframework from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rpython.memory.gc import X_CLONE, X_POOL, X_POOL_PTR +from pypy.rpython.memory.gc.marksweep import X_CLONE, X_POOL, X_POOL_PTR from pypy import conftest INT_SIZE = struct.calcsize("i") # only for estimates @@ -86,7 +86,7 @@ try: GCClass = self.gcpolicy.transformerclass.GCClass except AttributeError: - from pypy.rpython.memory.gc import MarkSweepGC as GCClass + from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass if hasattr(GCClass, 'STAT_HEAP_USAGE'): return statistics(GCClass.STAT_HEAP_USAGE) else: @@ -632,6 +632,6 @@ class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): - from pypy.rpython.memory.gc import SemiSpaceGC as GCClass + from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass GC_PARAMS = {'space_size': 2048} root_stack_depth = 200 From arigo at codespeak.net Sat Oct 13 22:35:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 13 Oct 2007 22:35:00 +0200 (CEST) Subject: [pypy-svn] r47434 - in pypy/dist/pypy: rpython rpython/lltypesystem rpython/memory rpython/memory/gc rpython/memory/gctransform rpython/memory/test translator/c translator/c/src translator/c/test Message-ID: <20071013203500.31BB5815D@code0.codespeak.net> Author: arigo Date: Sat Oct 13 22:34:59 2007 New Revision: 47434 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/llheap.py pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/lltypesystem/lltype.py pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/gctransform/boehm.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctransform/transform.py pypy/dist/pypy/rpython/memory/gcwrapper.py pypy/dist/pypy/rpython/memory/test/test_gc.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py pypy/dist/pypy/rpython/rmodel.py pypy/dist/pypy/translator/c/gc.py pypy/dist/pypy/translator/c/src/address.h pypy/dist/pypy/translator/c/test/test_boehm.py Log: Fix the id() in the presence of a moving GC. This first attempt is based on a veeeery slow implementation. M pypy/translator/c/test/test_boehm.py In Boehm the RPython-level id() no longer keeps the object alive M pypy/translator/c/gc.py M pypy/translator/c/src/address.h Delete old code M pypy/rpython/memory/test/test_gc.py M pypy/rpython/memory/test/test_transformed_gc.py Add tests for id() M pypy/rpython/lltypesystem/lloperation.py M pypy/rpython/lltypesystem/llheap.py M pypy/rpython/memory/gcwrapper.py M pypy/rpython/llinterp.py M pypy/rpython/rmodel.py Add 'gc_id' as an lloperation M pypy/rpython/memory/gctransform/transform.py M pypy/rpython/memory/gctransform/framework.py M pypy/rpython/memory/gctransform/boehm.py Transform this 'gc_id' in the gctransformer M pypy/rpython/memory/gc/semispace.py M pypy/rpython/memory/gc/base.py Generic but inefficient implementation of id() for all moving GCs M pypy/rpython/lltypesystem/llmemory.py M pypy/rpython/lltypesystem/lltype.py Add more pointer casts when required. Check for freed objects in _wref. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Sat Oct 13 22:34:59 2007 @@ -778,6 +778,9 @@ self.setvar(v_ptr, p) op_gc_reload_possibly_moved.specialform = True + def op_gc_id(self, v_ptr): + return self.heap.gc_id(v_ptr) + def op_gc_set_max_heap_size(self, maxsize): raise NotImplementedError("gc_set_max_heap_size") Modified: pypy/dist/pypy/rpython/lltypesystem/llheap.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llheap.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llheap.py Sat Oct 13 22:34:59 2007 @@ -14,3 +14,5 @@ assert typeOf(newvalue) == INNERTYPE # xxx access the address object's ref() directly for performance inneraddr.ref()[0] = newvalue + +from pypy.rpython.lltypesystem.lltype import cast_ptr_to_int as gc_id Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Sat Oct 13 22:34:59 2007 @@ -366,45 +366,12 @@ raise NullAddressError return self.ptr -## def get(self): -## return self.ref().get() - -## def set(self, value): -## self.ref().set(value) - def _cast_to_ptr(self, EXPECTED_TYPE): if self: - PTRTYPE = lltype.typeOf(self.ptr) - if (isinstance(EXPECTED_TYPE.TO, lltype.OpaqueType) or - isinstance(PTRTYPE.TO, lltype.OpaqueType)): - return lltype.cast_opaque_ptr(EXPECTED_TYPE, self.ptr) - else: - # regular case - return lltype.cast_pointer(EXPECTED_TYPE, self.ptr) + return cast_any_ptr(EXPECTED_TYPE, self.ptr) else: return lltype.nullptr(EXPECTED_TYPE.TO) -## if (isinstance(ref, _arrayitemref) and -## isinstance(EXPECTED_TYPE.TO, lltype.FixedSizeArray) and -## ref.type() == EXPECTED_TYPE.TO.OF): -## # special case that requires direct_arrayitems -## p_items = lltype.direct_arrayitems(ref.array) -## return lltype.direct_ptradd(p_items, ref.index) -## elif (isinstance(ref, _structfieldref) and -## isinstance(EXPECTED_TYPE.TO, lltype.FixedSizeArray) and -## ref.type() == EXPECTED_TYPE.TO.OF): -## # special case that requires direct_fieldptr -## return lltype.direct_fieldptr(ref.struct, -## ref.fieldname) -## else: -## result = ref.get() -## if (isinstance(EXPECTED_TYPE.TO, lltype.OpaqueType) or -## isinstance(lltype.typeOf(result).TO, lltype.OpaqueType)): -## return lltype.cast_opaque_ptr(EXPECTED_TYPE, result) -## else: -## # regular case -## return lltype.cast_pointer(EXPECTED_TYPE, result) - def _cast_to_int(self): if self: return self.ptr._cast_to_int() @@ -538,7 +505,7 @@ if p is None: return lltype.nullptr(PTRTYPE.TO) else: - return lltype.cast_pointer(PTRTYPE, p) + return cast_any_ptr(PTRTYPE, p) class _wref(lltype._container): _gckind = 'gc' @@ -553,7 +520,9 @@ def _dereference(self): obj = self._obref() - if obj is None: + # in combination with a GC like the SemiSpace, the 'obj' can be + # still alive in the CPython sense but freed by the arena logic. + if obj is None or obj._was_freed(): return None else: return obj._as_ptr() @@ -584,7 +553,8 @@ assert lltype.typeOf(pwref) == WeakRefPtr if pwref: assert isinstance(pwref._obj, _gctransformed_wref) - assert PTRTYPE == lltype.typeOf(pwref._obj._ptr) + if PTRTYPE is not None: + assert PTRTYPE == lltype.typeOf(pwref._obj._ptr) return pwref._obj._ptr else: return lltype.nullptr(PTRTYPE.TO) @@ -636,3 +606,20 @@ assert lltype.typeOf(source) == Address assert lltype.typeOf(dest) == Address size.raw_memcopy(source, dest) + +def cast_any_ptr(EXPECTED_TYPE, ptr): + # this is a generalization of the various cast_xxx_ptr() functions. + PTRTYPE = lltype.typeOf(ptr) + if PTRTYPE == EXPECTED_TYPE: + return ptr + elif EXPECTED_TYPE == WeakRefPtr: + return cast_ptr_to_weakrefptr(ptr) + elif PTRTYPE == WeakRefPtr: + ptr = cast_weakrefptr_to_ptr(None, ptr) + return cast_any_ptr(EXPECTED_TYPE, ptr) + elif (isinstance(EXPECTED_TYPE.TO, lltype.OpaqueType) or + isinstance(PTRTYPE.TO, lltype.OpaqueType)): + return lltype.cast_opaque_ptr(EXPECTED_TYPE, ptr) + else: + # regular case + return lltype.cast_pointer(EXPECTED_TYPE, ptr) Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Sat Oct 13 22:34:59 2007 @@ -389,6 +389,7 @@ 'gc_protect': LLOp(), 'gc_unprotect': LLOp(), 'gc_reload_possibly_moved': LLOp(), + 'gc_id': LLOp(canraise=(MemoryError,), sideeffects=False), 'gc_set_max_heap_size': LLOp(), # experimental operations in support of thread cloning, only # implemented by the Mark&Sweep GC Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lltype.py Sat Oct 13 22:34:59 2007 @@ -1278,6 +1278,8 @@ return self def _getid(self): return id(self) + def _was_freed(self): + return False class _parentable(_container): _kind = "?" @@ -1300,6 +1302,9 @@ self._check() # no double-frees self._storage = None + def _was_freed(self): + return self._storage is None + def _setparentstructure(self, parent, parentindex): self._wrparent = weakref.ref(parent) self._parent_type = typeOf(parent) Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Sat Oct 13 22:34:59 2007 @@ -68,6 +68,9 @@ # lots of cast and reverse-cast around... return llmemory.cast_ptr_to_adr(ref) + def id(self, ptr): + return lltype.cast_ptr_to_int(ptr) + def x_swap_pool(self, newpool): return newpool @@ -78,6 +81,38 @@ raise RuntimeError("no support for x_become in the GC") +class MovingGCBase(GCBase): + moving_gc = True + + def __init__(self): + self.wr_to_objects_with_id = [] + + def id(self, ptr): + # XXX linear search! this is probably too slow to be reasonable :-( + # On the other hand, it punishes you for using 'id', so that's good :-) + # XXX this may explode if --no-translation-rweakref is specified + lst = self.wr_to_objects_with_id + i = len(lst) + freeentry = -1 + while i > 0: + i -= 1 + target = llmemory.weakref_deref(llmemory.GCREF, lst[i]) + if not target: + freeentry = i + elif target == ptr: + break # found + else: + # not found + wr = llmemory.weakref_create(ptr) + if freeentry == -1: + i = len(lst) + lst.append(wr) + else: + i = freeentry # reuse the id() of a dead object + lst[i] = wr + return i + 1 # this produces id() values 1, 2, 3, 4... + + def choose_gc_from_config(config): """Return a (GCClass, GC_PARAMS) from the given config object. """ @@ -88,7 +123,7 @@ return MarkSweepGC, GC_PARAMS elif config.translation.frameworkgc == "semispace": GC_PARAMS = {'space_size': 8*1024*1024} # XXX adjust - from pypy.rpython.memory.gc.marksweep import SemiSpaceGC + from pypy.rpython.memory.gc.semispace import SemiSpaceGC return SemiSpaceGC, GC_PARAMS else: raise ValueError("unknown value for frameworkgc: %r" % ( Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Sat Oct 13 22:34:59 2007 @@ -7,16 +7,15 @@ from pypy.rlib.objectmodel import free_non_gc_object, debug_assert from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck -from pypy.rpython.memory.gc.base import GCBase +from pypy.rpython.memory.gc.base import MovingGCBase import sys, os memoryError = MemoryError() -class SemiSpaceGC(GCBase): +class SemiSpaceGC(MovingGCBase): _alloc_flavor_ = "raw" - moving_gc = True inline_simple_malloc = True HDR = lltype.Struct('header', ('forw', llmemory.Address), @@ -25,6 +24,7 @@ def __init__(self, AddressLinkedList, space_size=4096, max_space_size=sys.maxint//2+1, get_roots=None): + MovingGCBase.__init__(self) self.space_size = space_size self.max_space_size = max_space_size self.get_roots = get_roots Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/boehm.py Sat Oct 13 22:34:59 2007 @@ -147,6 +147,12 @@ resulttype=llmemory.Address) hop.cast_result(v_addr) + def gct_gc_id(self, hop): + # this is the logic from the HIDE_POINTER macro in + v_int = hop.genop('cast_ptr_to_int', [hop.spaceop.args[0]], + resulttype = lltype.Signed) + hop.genop('int_invert', [v_int], resultvar=hop.spaceop.result) + ########## weakrefs ########## # Boehm: weakref objects are small structures containing only a Boehm Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Sat Oct 13 22:34:59 2007 @@ -238,6 +238,14 @@ else: self.malloc_fast_ptr = self.malloc_fixedsize_ptr + if GCClass.moving_gc: + self.id_ptr = getfn(GCClass.id.im_func, + [s_gc, s_gcref], annmodel.SomeInteger(), + inline = False, + minimal_transform = False) + else: + self.id_ptr = None + self.statistics_ptr = getfn(GCClass.statistics.im_func, [s_gc, annmodel.SomeInteger()], annmodel.SomeInteger()) @@ -523,6 +531,18 @@ resulttype=llmemory.Address) hop.cast_result(v_addr) + def gct_gc_id(self, hop): + if self.id_ptr is not None: + self.push_roots(hop) + [v_ptr] = hop.spaceop.args + v_ptr = hop.genop("cast_opaque_ptr", [v_ptr], + resulttype=llmemory.GCREF) + hop.genop("direct_call", [self.id_ptr, self.c_const_gc, v_ptr], + resultvar=hop.spaceop.result) + self.pop_roots(hop) + else: + hop.rename('cast_ptr_to_int') # works nicely for non-moving GCs + def push_alive_nopyobj(self, var, llops): pass Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Sat Oct 13 22:34:59 2007 @@ -325,6 +325,10 @@ def gct_zero_gc_pointers_inside(self, hop): pass + def gct_gc_id(self, hop): + # this assumes a non-moving GC. Moving GCs need to override this + hop.rename('cast_ptr_to_int') + class MinimalGCTransformer(BaseGCTransformer): def __init__(self, parenttransformer): Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Sat Oct 13 22:34:59 2007 @@ -85,6 +85,10 @@ addr = gctypelayout.ll_weakref_deref(obj) return llmemory.cast_adr_to_ptr(addr, PTRTYPE) + def gc_id(self, ptr): + ptr = lltype.cast_opaque_ptr(llmemory.GCREF, ptr) + return self.gc.id(ptr) + # ____________________________________________________________ class RootLinkedList(object): Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Sat Oct 13 22:34:59 2007 @@ -251,6 +251,25 @@ res = self.interpret(f, []) assert res + def test_id(self): + class A(object): + pass + a1 = A() + def f(): + a2 = A() + a3 = A() + id1 = id(a1) + id2 = id(a2) + id3 = id(a3) + llop.gc__collect(lltype.Void) + error = 0 + if id1 != id(a1): error += 1 + if id2 != id(a2): error += 2 + if id3 != id(a3): error += 4 + return error + res = self.interpret(f, []) + assert res == 0 + class TestMarkSweepGC(GCTest): from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Sat Oct 13 22:34:59 2007 @@ -376,6 +376,26 @@ res = run([]) assert res == 111111 + def test_id(self): + class A(object): + pass + a1 = A() + def func(): + a2 = A() + a3 = A() + id1 = id(a1) + id2 = id(a2) + id3 = id(a3) + llop.gc__collect(lltype.Void) + error = 0 + if id1 != id(a1): error += 1 + if id2 != id(a2): error += 2 + if id3 != id(a3): error += 4 + return error + run = self.runner(func) + res = run([]) + assert res == 0 + class TestMarkSweepGC(GenericGCTests): class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): Modified: pypy/dist/pypy/rpython/rmodel.py ============================================================================== --- pypy/dist/pypy/rpython/rmodel.py (original) +++ pypy/dist/pypy/rpython/rmodel.py Sat Oct 13 22:34:59 2007 @@ -220,9 +220,7 @@ raise TyperError('id() of an instance of the non-pointer %r' % ( self,)) vobj, = hop.inputargs(self) - # XXX I am still not sure whether this is the right thing to do, - # since the int might keep things alive with boehm - return hop.genop('cast_ptr_to_int', [vobj], resulttype=Signed) + return hop.genop('gc_id', [vobj], resulttype=Signed) def rtype_hash(self, hop): ll_hash = self.get_ll_hash_function() Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Sat Oct 13 22:34:59 2007 @@ -199,7 +199,6 @@ if sys.platform != "win32": # GC_REDIRECT_TO_LOCAL is not supported on Win32 by gc6.8 yield "#define GC_REDIRECT_TO_LOCAL 1" - yield "#define GC_I_HIDE_POINTERS 1" yield '#include ' yield '#define USING_BOEHM_GC' Modified: pypy/dist/pypy/translator/c/src/address.h ============================================================================== --- pypy/dist/pypy/translator/c/src/address.h (original) +++ pypy/dist/pypy/translator/c/src/address.h Sat Oct 13 22:34:59 2007 @@ -18,11 +18,3 @@ #define OP_CAST_ADR_TO_INT(x, r) r = ((long)x) #define OP_CAST_INT_TO_ADR(x, r) r = ((void *)(x)) - -#ifndef HIDE_POINTER -#define HIDE_POINTER(p) (p) -#ifdef REVEAL_POINTER -#error HIDE_POINTER but not REVEAL_POINTER? -#endif -#define REVEAL_POINTER(p) (p) -#endif Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Sat Oct 13 22:34:59 2007 @@ -102,7 +102,6 @@ def test_id_is_weak(self): # test that id(obj) does not keep obj alive - py.test.skip("fails") from pypy.rpython.lltypesystem.lloperation import llop class State: pass From tismer at codespeak.net Sat Oct 13 22:43:18 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sat, 13 Oct 2007 22:43:18 +0200 (CEST) Subject: [pypy-svn] r47435 - pypy/dist/pypy/rlib Message-ID: <20071013204318.074068161@code0.codespeak.net> Author: tismer Date: Sat Oct 13 22:43:18 2007 New Revision: 47435 Modified: pypy/dist/pypy/rlib/rmmap.py Log: some cleanups, still everything is working. todo: fix module\mmap\test\test_mmap.py for windows, which has missing constants Modified: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/rmmap.py (original) +++ pypy/dist/pypy/rlib/rmmap.py Sat Oct 13 22:43:18 2007 @@ -115,12 +115,25 @@ DWORD_P = rffi.DINTP LPCTSTR = rffi.CCHARP - class ComplexCConfig: _includes_ = CConfig._includes_ - sysinfo_struct = rffi_platform.Struct( + + SYSINFO_STRUCT = rffi.CStruct( + 'SYSINFO_STRUCT', + ("wProcessorArchitecture", WORD), + ("wReserved", WORD), + ) + + SYSINFO_UNION = rffi.CStruct( + 'union SYSINFO_UNION', + ("dwOemId", DWORD), + ("_struct_", SYSINFO_STRUCT), + ) + # sorry, I can't find a way to insert the above + # because the union field has no name + SYSTEM_INFO = rffi_platform.Struct( 'SYSTEM_INFO', [ - #("union_ignored", DWORD), # I'll do this later + ## ("_union_", SYSINFO_UNION), ("dwPageSize", DWORD), ("lpMinimumApplicationAddress", LPVOID), ("lpMaximumApplicationAddress", LPVOID), @@ -133,10 +146,10 @@ ]) config = rffi_platform.configure(ComplexCConfig) - sysinfo_struct = config['sysinfo_struct'] - sysinfo_struct_p = lltype.Ptr(sysinfo_struct) + SYSTEM_INFO = config['SYSTEM_INFO'] + SYSTEM_INFO_P = lltype.Ptr(SYSTEM_INFO) - GetSystemInfo = winexternal('GetSystemInfo', [sysinfo_struct_p], lltype.Void) + GetSystemInfo = winexternal('GetSystemInfo', [SYSTEM_INFO_P], lltype.Void) GetFileSize = winexternal('GetFileSize', [INT, INT_P], INT) GetCurrentProcess = winexternal('GetCurrentProcess', [], INT) DuplicateHandle = winexternal('DuplicateHandle', [INT, INT, INT, INT_P, DWORD, BOOL, DWORD], BOOL) @@ -153,7 +166,7 @@ def _get_page_size(): try: - si = rffi.make(sysinfo_struct) + si = rffi.make(SYSTEM_INFO) GetSystemInfo(si) return int(si.c_dwPageSize) finally: @@ -163,7 +176,7 @@ # XXX use native Windows types like WORD high_ref = lltype.malloc(INT_P.TO, 1, flavor='raw') try: - low = GetFileSize(rffi.cast(INT, handle), high_ref) + low = GetFileSize(handle, high_ref) high = high_ref[0] # low might just happen to have the value INVALID_FILE_SIZE # so we need to check the last error also @@ -229,10 +242,10 @@ self.unmapview() self.setdata(NODATA, 0) if self.map_handle != INVALID_INT_VALUE: - CloseHandle(rffi.cast(INT, self.map_handle)) + CloseHandle(self.map_handle) self.map_handle = INVALID_INT_VALUE if self.file_handle != INVALID_INT_VALUE: - CloseHandle(rffi.cast(INT, self.file_handle)) + CloseHandle(self.file_handle) self.file_handle = INVALID_INT_VALUE elif _POSIX: self.closed = True @@ -447,15 +460,15 @@ # move to the desired EOF position if _64BIT: - newsize_high = rffi.cast(DWORD, newsize >> 32) - newsize_low = rffi.cast(DWORD, newsize & 0xFFFFFFFF) + newsize_high = newsize >> 32 + newsize_low = newsize & 0xFFFFFFFF high_ref = lltype.malloc(DWORD_P.TO, 1, flavor='raw') else: - newsize_high = rffi.cast(INT, 0) - newsize_low = rffi.cast(INT, newsize) + newsize_high = 0 + newsize_low = newsize high_ref = lltype.malloc(INT_P.TO, 1, flavor='raw') - FILE_BEGIN = rffi.cast(INT, 0) + FILE_BEGIN = 0 try: high_ref[0] = newsize_high SetFilePointer(self.file_handle, newsize_low, high_ref, @@ -469,10 +482,10 @@ newsize_high, newsize_low, self.tagname) self.map_handle = res - dwErrCode = rffi.cast(DWORD, 0) + dwErrCode = 0 if self.map_handle: data = MapViewOfFile(self.map_handle, FILE_MAP_WRITE, - 0, 0, 0) + 0, 0, 0) if data: self.setdata(data, newsize) return @@ -605,7 +618,7 @@ # assume -1 and 0 both mean invalid file descriptor # to 'anonymously' map memory. if fileno != -1 and fileno != 0: - fh = _get_osfhandle(rffi.cast(INT, fileno)) + fh = _get_osfhandle(fileno) if fh == -1: raise REnvironmentError(_get_error_msg()) # Win9x appears to need us seeked to zero @@ -619,10 +632,10 @@ # it is necessary to duplicate the handle, so the # Python code can close it on us handle_ref = lltype.malloc(INT_P.TO, 1, flavor='raw') - handle_ref[0] = rffi.cast(INT, m.file_handle) + handle_ref[0] = m.file_handle try: res = DuplicateHandle(GetCurrentProcess(), # source process handle - rffi.cast(INT, fh), # handle to be duplicated + fh, # handle to be duplicated GetCurrentProcess(), # target process handle handle_ref, # result 0, # access - ignored due to options value @@ -650,17 +663,17 @@ # DWORD is a 4-byte int. If int > 4-byte it must be divided if _64BIT: - size_hi = rffi.cast(DWORD, map_size >> 32) - size_lo = rffi.cast(DWORD, map_size & 0xFFFFFFFF) + size_hi = map_size >> 32 + size_lo = map_size & 0xFFFFFFFF else: - size_hi = rffi.cast(INT, 0) - size_lo = rffi.cast(INT, map_size) + size_hi = 0 + size_lo = map_size - m.map_handle = CreateFileMapping(rffi.cast(INT, m.file_handle), NULL, flProtect, + m.map_handle = CreateFileMapping(m.file_handle, NULL, flProtect, size_hi, size_lo, m.tagname) if m.map_handle: - res = MapViewOfFile(rffi.cast(INT, m.map_handle), dwDesiredAccess, + res = MapViewOfFile(m.map_handle, dwDesiredAccess, 0, 0, 0) if res: m.setdata(res, map_size) From cfbolz at codespeak.net Sat Oct 13 23:11:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 13 Oct 2007 23:11:30 +0200 (CEST) Subject: [pypy-svn] r47436 - in pypy/dist/pypy/rpython/memory: gc test Message-ID: <20071013211130.B3DDF818A@code0.codespeak.net> Author: cfbolz Date: Sat Oct 13 23:11:29 2007 New Revision: 47436 Modified: pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/marksweep.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: have a subclass of the mark and sweep GC that prints info when it is allocating and freeing memory. trying to do some nice graphs about memory usage in pypy. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Sat Oct 13 23:11:29 2007 @@ -121,6 +121,10 @@ GC_PARAMS = {'start_heap_size': 8*1024*1024} # XXX adjust from pypy.rpython.memory.gc.marksweep import MarkSweepGC return MarkSweepGC, GC_PARAMS + if config.translation.frameworkgc == "statistics": + GC_PARAMS = {'start_heap_size': 8*1024*1024} # XXX adjust + from pypy.rpython.memory.gc.marksweep import PrintingMarkSweepGC + return PrintingMarkSweepGC, GC_PARAMS elif config.translation.frameworkgc == "semispace": GC_PARAMS = {'space_size': 8*1024*1024} # XXX adjust from pypy.rpython.memory.gc.semispace import SemiSpaceGC Modified: pypy/dist/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/dist/pypy/rpython/memory/gc/marksweep.py Sat Oct 13 23:11:29 2007 @@ -870,3 +870,419 @@ hdr.next = self.malloced_objects self.malloced_objects = hdr hdr = next + + +class PrintingMarkSweepGC(MarkSweepGC): + _alloc_flavor_ = "raw" + COLLECT_EVERY = 2000 + + def __init__(self, AddressLinkedList, start_heap_size=4096, get_roots=None): + MarkSweepGC.__init__(self, AddressLinkedList, start_heap_size, get_roots) + self.count_mallocs = 0 + + def write_malloc_statistics(self, typeid, size, result, varsize): + if varsize: + what = "malloc_varsize" + else: + what = "malloc" + llop.debug_print(lltype.Void, what, typeid, " ", size, " ", result) + + def write_free_statistics(self, typeid, result): + llop.debug_print(lltype.Void, "free", typeid, " ", result) + + + #XXX XXX XXX XXX XXX XXX XXX XXX + # the next methods are nearly copies of the MarkSweepGC methods + # not clear how that can be improved + + def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, + contains_weakptr=False): + self.count_mallocs += 1 + if can_collect and self.count_mallocs > self.COLLECT_EVERY: + self.collect() + size_gc_header = self.gcheaderbuilder.size_gc_header + try: + tot_size = size_gc_header + size + usage = raw_malloc_usage(tot_size) + bytes_malloced = ovfcheck(self.bytes_malloced+usage) + ovfcheck(self.heap_usage + bytes_malloced) + except OverflowError: + raise memoryError + result = raw_malloc(tot_size) + if not result: + raise memoryError + hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) + hdr.typeid = typeid << 1 + if has_finalizer: + hdr.next = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = hdr + elif contains_weakptr: + hdr.next = self.objects_with_weak_pointers + self.objects_with_weak_pointers = hdr + else: + hdr.next = self.malloced_objects + self.malloced_objects = hdr + self.bytes_malloced = bytes_malloced + result += size_gc_header + #llop.debug_print(lltype.Void, 'malloc typeid', typeid, + # '->', llmemory.cast_adr_to_int(result)) + self.write_malloc_statistics(typeid, tot_size, result, False) + return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + + def malloc_fixedsize_clear(self, typeid, size, can_collect, + has_finalizer=False, contains_weakptr=False): + self.count_mallocs += 1 + if can_collect and self.count_mallocs > self.COLLECT_EVERY: + self.collect() + size_gc_header = self.gcheaderbuilder.size_gc_header + try: + tot_size = size_gc_header + size + usage = raw_malloc_usage(tot_size) + bytes_malloced = ovfcheck(self.bytes_malloced+usage) + ovfcheck(self.heap_usage + bytes_malloced) + except OverflowError: + raise memoryError + result = raw_malloc(tot_size) + if not result: + raise memoryError + raw_memclear(result, tot_size) + hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) + hdr.typeid = typeid << 1 + if has_finalizer: + hdr.next = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = hdr + elif contains_weakptr: + hdr.next = self.objects_with_weak_pointers + self.objects_with_weak_pointers = hdr + else: + hdr.next = self.malloced_objects + self.malloced_objects = hdr + self.bytes_malloced = bytes_malloced + result += size_gc_header + #llop.debug_print(lltype.Void, 'malloc typeid', typeid, + # '->', llmemory.cast_adr_to_int(result)) + self.write_malloc_statistics(typeid, tot_size, result, False) + return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + + def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, + can_collect, has_finalizer=False): + self.count_mallocs += 1 + if can_collect and self.count_mallocs > self.COLLECT_EVERY: + self.collect() + size_gc_header = self.gcheaderbuilder.size_gc_header + try: + fixsize = size_gc_header + size + varsize = ovfcheck(itemsize * length) + tot_size = ovfcheck(fixsize + varsize) + usage = raw_malloc_usage(tot_size) + bytes_malloced = ovfcheck(self.bytes_malloced+usage) + ovfcheck(self.heap_usage + bytes_malloced) + except OverflowError: + raise memoryError + result = raw_malloc(tot_size) + if not result: + raise memoryError + (result + size_gc_header + offset_to_length).signed[0] = length + hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) + hdr.typeid = typeid << 1 + if has_finalizer: + hdr.next = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = hdr + else: + hdr.next = self.malloced_objects + self.malloced_objects = hdr + self.bytes_malloced = bytes_malloced + + result += size_gc_header + #llop.debug_print(lltype.Void, 'malloc_varsize length', length, + # 'typeid', typeid, + # '->', llmemory.cast_adr_to_int(result)) + self.write_malloc_statistics(typeid, tot_size, result, True) + return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + + def malloc_varsize_clear(self, typeid, length, size, itemsize, + offset_to_length, can_collect, + has_finalizer=False): + self.count_mallocs += 1 + if can_collect and self.count_mallocs > self.COLLECT_EVERY: + self.collect() + size_gc_header = self.gcheaderbuilder.size_gc_header + try: + fixsize = size_gc_header + size + varsize = ovfcheck(itemsize * length) + tot_size = ovfcheck(fixsize + varsize) + usage = raw_malloc_usage(tot_size) + bytes_malloced = ovfcheck(self.bytes_malloced+usage) + ovfcheck(self.heap_usage + bytes_malloced) + except OverflowError: + raise memoryError + result = raw_malloc(tot_size) + if not result: + raise memoryError + raw_memclear(result, tot_size) + (result + size_gc_header + offset_to_length).signed[0] = length + hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) + hdr.typeid = typeid << 1 + if has_finalizer: + hdr.next = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = hdr + else: + hdr.next = self.malloced_objects + self.malloced_objects = hdr + self.bytes_malloced = bytes_malloced + + result += size_gc_header + #llop.debug_print(lltype.Void, 'malloc_varsize length', length, + # 'typeid', typeid, + # '->', llmemory.cast_adr_to_int(result)) + self.write_malloc_statistics(typeid, tot_size, result, True) + return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + + def collect(self): + # 1. mark from the roots, and also the objects that objects-with-del + # point to (using the list of malloced_objects_with_finalizer) + # 2. walk the list of objects-without-del and free the ones not marked + # 3. walk the list of objects-with-del and for the ones not marked: + # call __del__, move the object to the list of object-without-del + import time + from pypy.rpython.lltypesystem.lloperation import llop + self.count_mallocs = 0 + start_time = time.time() + self.collect_in_progress = True + roots = self.get_roots() + size_gc_header = self.gcheaderbuilder.size_gc_header +## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, +## size_gc_header) + + # push the roots on the mark stack + objects = self.AddressLinkedList() # mark stack + while 1: + curr = roots.pop() + if curr == NULL: + break + # roots is a list of addresses to addresses: + objects.append(curr.address[0]) + free_non_gc_object(roots) + # from this point onwards, no more mallocs should be possible + old_malloced = self.bytes_malloced + self.bytes_malloced = 0 + curr_heap_size = 0 + freed_size = 0 + + # mark objects reachable by objects with a finalizer, but not those + # themselves. add their size to curr_heap_size, since they always + # survive the collection + hdr = self.malloced_objects_with_finalizer + while hdr: + next = hdr.next + typeid = hdr.typeid >> 1 + gc_info = llmemory.cast_ptr_to_adr(hdr) + obj = gc_info + size_gc_header + if not hdr.typeid & 1: + self.add_reachable_to_stack(obj, objects) + addr = llmemory.cast_ptr_to_adr(hdr) + size = self.fixed_size(typeid) + if self.is_varsize(typeid): + length = (obj + self.varsize_offset_to_length(typeid)).signed[0] + size += self.varsize_item_sizes(typeid) * length + estimate = raw_malloc_usage(size_gc_header + size) + curr_heap_size += estimate + hdr = next + + # mark thinks on the mark stack and put their descendants onto the + # stack until the stack is empty + while objects.non_empty(): #mark + curr = objects.pop() + gc_info = curr - size_gc_header + hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) + if hdr.typeid & 1: + continue + self.add_reachable_to_stack(curr, objects) + hdr.typeid = hdr.typeid | 1 + objects.delete() + # also mark self.curpool + if self.curpool: + gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header + hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) + hdr.typeid = hdr.typeid | 1 + # go through the list of objects containing weak pointers + # and kill the links if they go to dead objects + # if the object itself is not marked, free it + hdr = self.objects_with_weak_pointers + surviving = lltype.nullptr(self.HDR) + while hdr: + typeid = hdr.typeid >> 1 + next = hdr.next + addr = llmemory.cast_ptr_to_adr(hdr) + size = self.fixed_size(typeid) + estimate = raw_malloc_usage(size_gc_header + size) + if hdr.typeid & 1: + typeid = hdr.typeid >> 1 + offset = self.weakpointer_offset(typeid) + hdr.typeid = hdr.typeid & (~1) + gc_info = llmemory.cast_ptr_to_adr(hdr) + weakref_obj = gc_info + size_gc_header + pointing_to = (weakref_obj + offset).address[0] + if pointing_to: + gc_info_pointing_to = pointing_to - size_gc_header + hdr_pointing_to = llmemory.cast_adr_to_ptr( + gc_info_pointing_to, self.HDRPTR) + # pointed to object will die + # XXX what to do if the object has a finalizer which resurrects + # the object? + if not hdr_pointing_to.typeid & 1: + (weakref_obj + offset).address[0] = NULL + hdr.next = surviving + surviving = hdr + curr_heap_size += estimate + else: + gc_info = llmemory.cast_ptr_to_adr(hdr) + weakref_obj = gc_info + size_gc_header + self.write_free_statistics(typeid, weakref_obj) + freed_size += estimate + raw_free(addr) + hdr = next + self.objects_with_weak_pointers = surviving + # sweep: delete objects without del if they are not marked + # unmark objects without del that are marked + firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw') + firstpoolnode.linkedlist = self.malloced_objects + firstpoolnode.nextnode = self.poolnodes + prevpoolnode = lltype.nullptr(self.POOLNODE) + poolnode = firstpoolnode + while poolnode: #sweep + ppnext = llmemory.cast_ptr_to_adr(poolnode) + ppnext += llmemory.offsetof(self.POOLNODE, 'linkedlist') + hdr = poolnode.linkedlist + while hdr: #sweep + typeid = hdr.typeid >> 1 + next = hdr.next + addr = llmemory.cast_ptr_to_adr(hdr) + size = self.fixed_size(typeid) + if self.is_varsize(typeid): + length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0] + size += self.varsize_item_sizes(typeid) * length + estimate = raw_malloc_usage(size_gc_header + size) + if hdr.typeid & 1: + hdr.typeid = hdr.typeid & (~1) + ppnext.address[0] = addr + ppnext = llmemory.cast_ptr_to_adr(hdr) + ppnext += llmemory.offsetof(self.HDR, 'next') + curr_heap_size += estimate + else: + gc_info = llmemory.cast_ptr_to_adr(hdr) + obj = gc_info + size_gc_header + self.write_free_statistics(typeid, obj) + freed_size += estimate + raw_free(addr) + hdr = next + ppnext.address[0] = llmemory.NULL + next = poolnode.nextnode + if not poolnode.linkedlist and prevpoolnode: + # completely empty node + prevpoolnode.nextnode = next + lltype.free(poolnode, flavor='raw') + else: + prevpoolnode = poolnode + poolnode = next + self.malloced_objects = firstpoolnode.linkedlist + self.poolnodes = firstpoolnode.nextnode + lltype.free(firstpoolnode, flavor='raw') + #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) + + end_time = time.time() + compute_time = start_time - self.prev_collect_end_time + collect_time = end_time - start_time + + garbage_collected = old_malloced - (curr_heap_size - self.heap_usage) + + if (collect_time * curr_heap_size > + 0.02 * garbage_collected * compute_time): + self.bytes_malloced_threshold += self.bytes_malloced_threshold / 2 + if (collect_time * curr_heap_size < + 0.005 * garbage_collected * compute_time): + self.bytes_malloced_threshold /= 2 + + # Use atleast as much memory as current live objects. + if curr_heap_size > self.bytes_malloced_threshold: + self.bytes_malloced_threshold = curr_heap_size + + # Cap at 1/4 GB + self.bytes_malloced_threshold = min(self.bytes_malloced_threshold, + 256 * 1024 * 1024) + self.total_collection_time += collect_time + self.prev_collect_end_time = end_time + if DEBUG_PRINT: + llop.debug_print(lltype.Void, + " malloced since previous collection:", + old_malloced, "bytes") + llop.debug_print(lltype.Void, + " heap usage at start of collection: ", + self.heap_usage + old_malloced, "bytes") + llop.debug_print(lltype.Void, + " freed: ", + freed_size, "bytes") + llop.debug_print(lltype.Void, + " new heap usage: ", + curr_heap_size, "bytes") + llop.debug_print(lltype.Void, + " total time spent collecting: ", + self.total_collection_time, "seconds") + llop.debug_print(lltype.Void, + " collecting time: ", + collect_time) + llop.debug_print(lltype.Void, + " computing time: ", + collect_time) + llop.debug_print(lltype.Void, + " new threshold: ", + self.bytes_malloced_threshold) +## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, +## size_gc_header) + assert self.heap_usage + old_malloced == curr_heap_size + freed_size + + self.heap_usage = curr_heap_size + hdr = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) + last = lltype.nullptr(self.HDR) + while hdr: + next = hdr.next + if hdr.typeid & 1: + hdr.next = lltype.nullptr(self.HDR) + if not self.malloced_objects_with_finalizer: + self.malloced_objects_with_finalizer = hdr + else: + last.next = hdr + hdr.typeid = hdr.typeid & (~1) + last = hdr + else: + obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header + finalizer = self.getfinalizer(hdr.typeid >> 1) + # make malloced_objects_with_finalizer consistent + # for the sake of a possible collection caused by finalizer + if not self.malloced_objects_with_finalizer: + self.malloced_objects_with_finalizer = next + else: + last.next = next + hdr.next = self.malloced_objects + self.malloced_objects = hdr + #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) + finalizer(obj) + if not self.collect_in_progress: # another collection was caused? + llop.debug_print(lltype.Void, "outer collect interrupted " + "by recursive collect") + return + if not last: + if self.malloced_objects_with_finalizer == next: + self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) + else: + # now it gets annoying: finalizer caused a malloc of something + # with a finalizer + last = self.malloced_objects_with_finalizer + while last.next != next: + last = last.next + last.next = lltype.nullptr(self.HDR) + else: + last.next = lltype.nullptr(self.HDR) + hdr = next + self.collect_in_progress = False Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Sat Oct 13 23:11:29 2007 @@ -648,6 +648,14 @@ assert res == 20 +class TestPrintingGC(GenericGCTests): + + class gcpolicy(gc.FrameworkGcPolicy): + class transformerclass(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc.marksweep import PrintingMarkSweepGC as GCClass + GC_PARAMS = {'start_heap_size': 4096 } + root_stack_depth = 200 + class TestSemiSpaceGC(GenericGCTests): class gcpolicy(gc.FrameworkGcPolicy): From tismer at codespeak.net Sat Oct 13 23:15:26 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sat, 13 Oct 2007 23:15:26 +0200 (CEST) Subject: [pypy-svn] r47437 - pypy/dist/pypy/rlib Message-ID: <20071013211526.0D3FD8188@code0.codespeak.net> Author: tismer Date: Sat Oct 13 23:15:26 2007 New Revision: 47437 Modified: pypy/dist/pypy/rlib/rmmap.py Log: activated the relevant branch of the union Modified: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/rmmap.py (original) +++ pypy/dist/pypy/rlib/rmmap.py Sat Oct 13 23:15:26 2007 @@ -134,6 +134,10 @@ SYSTEM_INFO = rffi_platform.Struct( 'SYSTEM_INFO', [ ## ("_union_", SYSINFO_UNION), + ## instead, we put the smaller fields, here + ("wProcessorArchitecture", WORD), + ("wReserved", WORD), + ## should be a union. dwOemId is obsolete, anyway ("dwPageSize", DWORD), ("lpMinimumApplicationAddress", LPVOID), ("lpMaximumApplicationAddress", LPVOID), From cfbolz at codespeak.net Sat Oct 13 23:23:36 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 13 Oct 2007 23:23:36 +0200 (CEST) Subject: [pypy-svn] r47438 - pypy/dist/pypy/config Message-ID: <20071013212336.B06CE80AF@code0.codespeak.net> Author: cfbolz Date: Sat Oct 13 23:23:36 2007 New Revision: 47438 Modified: pypy/dist/pypy/config/translationoption.py Log: add a cmdline option for the printing gc Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Sat Oct 13 23:23:36 2007 @@ -52,7 +52,7 @@ requires=[("translation.gc", "framework"), ("translation.stackless", True)]), ChoiceOption("frameworkgc", "Select one of our custom GCs", - ["marksweep", "semispace"], + ["marksweep", "semispace", "statistics"], "marksweep", cmdline="--frameworkgc", requires={ "marksweep": [("translation.gc", "framework")], "semispace": [("translation.gc", "framework")], From tismer at codespeak.net Sun Oct 14 04:33:47 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sun, 14 Oct 2007 04:33:47 +0200 (CEST) Subject: [pypy-svn] r47441 - pypy/dist/pypy/rlib Message-ID: <20071014023347.D72568154@code0.codespeak.net> Author: tismer Date: Sun Oct 14 04:33:46 2007 New Revision: 47441 Modified: pypy/dist/pypy/rlib/rmmap.py Log: started to use native windows types. I think the module needs more work, sive I don't believe that restricting file size to sys.maxint is correct. Modified: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/rmmap.py (original) +++ pypy/dist/pypy/rlib/rmmap.py Sun Oct 14 04:33:46 2007 @@ -31,6 +31,22 @@ _header_ = '#define _GNU_SOURCE\n' size_t = rffi_platform.SimpleType("size_t", rffi.LONG) off_t = rffi_platform.SimpleType("off_t", rffi.LONG) + if _MS_WINDOWS: + DWORD_PTR = rffi_platform.SimpleType("DWORD_PTR", rffi.LONG) + WORD = rffi_platform.SimpleType("WORD", rffi.UINT) + DWORD = rffi_platform.SimpleType("DWORD", rffi.ULONG) + BOOL = rffi_platform.SimpleType("BOOL", rffi.LONG) + LONG = rffi_platform.SimpleType("LONG", rffi.LONG) + LPVOID = rffi_platform.SimpleType("LPVOID", rffi.INTP) + LPCVOID = rffi_platform.SimpleType("LPCVOID", rffi.CCHARP) + INT = rffi_platform.SimpleType("int", rffi.INT) + INT_P = rffi_platform.SimpleType("int *", rffi.INTP) + # HANDLE = rffi_platform.SimpleType("HANDLE", rffi.UINTP) + # XXX change correct usage of handle throughout all the source + # and make all interfaces conformant. For now, we just cheat. + HANDLE = rffi_platform.SimpleType("int", rffi.LONG) + LPCTSTR = rffi_platform.SimpleType("LPCTSTR", rffi.CCHARP) + LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.INTP) constants = {} if _POSIX: @@ -103,17 +119,6 @@ return os.strerror(errno) elif _MS_WINDOWS: - WORD = rffi.UINT - DWORD = rffi.ULONG - BOOL = rffi.LONG - LONG = rffi.LONG - LPVOID = PTR - LPCVOID = LPVOID - DWORD_PTR = DWORD - INT = rffi.INT - INT_P = rffi.INTP - DWORD_P = rffi.DINTP - LPCTSTR = rffi.CCHARP class ComplexCConfig: _includes_ = CConfig._includes_ @@ -154,7 +159,7 @@ SYSTEM_INFO_P = lltype.Ptr(SYSTEM_INFO) GetSystemInfo = winexternal('GetSystemInfo', [SYSTEM_INFO_P], lltype.Void) - GetFileSize = winexternal('GetFileSize', [INT, INT_P], INT) + GetFileSize = winexternal('GetFileSize', [HANDLE, LPDWORD], DWORD) GetCurrentProcess = winexternal('GetCurrentProcess', [], INT) DuplicateHandle = winexternal('DuplicateHandle', [INT, INT, INT, INT_P, DWORD, BOOL, DWORD], BOOL) CreateFileMapping = winexternal('CreateFileMappingA', [INT, PTR, INT, INT, INT, LPCTSTR], INT) @@ -178,9 +183,12 @@ def _get_file_size(handle): # XXX use native Windows types like WORD - high_ref = lltype.malloc(INT_P.TO, 1, flavor='raw') + high_ref = lltype.malloc(LPDWORD.TO, 1, flavor='raw') try: low = GetFileSize(handle, high_ref) + low = rffi.cast(lltype.Signed, low) + # XXX should be propagate the real type, allowing + # for 2*sys.maxint? high = high_ref[0] # low might just happen to have the value INVALID_FILE_SIZE # so we need to check the last error also @@ -188,7 +196,7 @@ NO_ERROR = 0 dwErr = GetLastError() err = rffi.cast(lltype.Signed, dwErr) - if rffi.cast(lltype.Signed, low) == INVALID_FILE_SIZE and err != NO_ERROR: + if low == INVALID_FILE_SIZE and err != NO_ERROR: raise REnvironmentError(os.strerror(err)) return low, high finally: @@ -466,7 +474,7 @@ if _64BIT: newsize_high = newsize >> 32 newsize_low = newsize & 0xFFFFFFFF - high_ref = lltype.malloc(DWORD_P.TO, 1, flavor='raw') + high_ref = lltype.malloc(DWORD_PTR.TO, 1, flavor='raw') else: newsize_high = 0 newsize_low = newsize From cfbolz at codespeak.net Sun Oct 14 12:46:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 14 Oct 2007 12:46:23 +0200 (CEST) Subject: [pypy-svn] r47444 - pypy/dist/pypy/rpython/tool Message-ID: <20071014104623.D38A78148@code0.codespeak.net> Author: cfbolz Date: Sun Oct 14 12:46:23 2007 New Revision: 47444 Added: pypy/dist/pypy/rpython/tool/gcstat.py (contents, props changed) Log: parse the generated log file for gc statistics Added: pypy/dist/pypy/rpython/tool/gcstat.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/tool/gcstat.py Sun Oct 14 12:46:23 2007 @@ -0,0 +1,46 @@ + +class LifeTime(object): + __slots__ = "typeid address size varsize birth death".split() + def __init__(self, typeid, address, size, varsize, birth, death=-1): + self.typeid = typeid + self.address = address + self.size = size + self.birth = birth + self.death = death + +def parse_file(f, callback): + unknown_lifetime = {} + current = 0 + for line in f: + line = line.split() + if line[0] == "free": + _, typeid, address = line + typeid = int(typeid) + address = int(address, 16) + unknown = unknown_lifetime.pop(address) + unknown.death = current + callback(unknown) + else: + if line[0] == "malloc_varsize": + varsize = True + else: + varsize = False + _, typeid, size, address = line + size = int(size) + typeid = int(typeid) + address = int(address, 16) + new = LifeTime(typeid, address, size, varsize, current) + unknown_lifetime[address] = new + current += size + for unknown in unknown_lifetime.itervalues(): + unknown.death = current + callback(unknown) + return all + +def collect_all(f): + all = [] + def callback(obj): + all.append(obj) + parse_file(f, callback) + return all + From tismer at codespeak.net Sun Oct 14 15:24:50 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sun, 14 Oct 2007 15:24:50 +0200 (CEST) Subject: [pypy-svn] r47446 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071014132450.E8E67814C@code0.codespeak.net> Author: tismer Date: Sun Oct 14 15:24:48 2007 New Revision: 47446 Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: reverted unnecessary changes Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Sun Oct 14 15:24:48 2007 @@ -292,18 +292,9 @@ # char * CCHARP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) -# int * -INTP = lltype.Ptr(lltype.Array(lltype.Signed, hints={'nolength': True})) +# int *, unsigned int *, etc. +#INTP = ... see setup() above -# unsigned int * -UINTP = lltype.Ptr(lltype.Array(lltype.Unsigned, hints={'nolength': True})) - -# signed long long * -DINTP = lltype.Ptr(lltype.Array(lltype.SignedLongLong, hints={'nolength': True})) - -# unsigned long long * -UDINTP = lltype.Ptr(lltype.Array(lltype.UnsignedLongLong, hints={'nolength': True})) - # double * DOUBLEP = lltype.Ptr(lltype.Array(DOUBLE, hints={'nolength': True})) From tismer at codespeak.net Sun Oct 14 16:14:13 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sun, 14 Oct 2007 16:14:13 +0200 (CEST) Subject: [pypy-svn] r47448 - pypy/dist/pypy/module/mmap Message-ID: <20071014141413.20DD58147@code0.codespeak.net> Author: tismer Date: Sun Oct 14 16:14:09 2007 New Revision: 47448 Modified: pypy/dist/pypy/module/mmap/interp_mmap.py Log: module/mmap passes all tests on windows Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Sun Oct 14 16:14:09 2007 @@ -205,6 +205,7 @@ space.wrap("memory mapped size is too large (limited by C int)")) if rmmap._POSIX: + def mmap(space, fileno, length, flags=rmmap.MAP_SHARED, prot=rmmap.PROT_WRITE | rmmap.PROT_READ, access=rmmap._ACCESS_DEFAULT): @@ -218,9 +219,19 @@ except RTypeError, e: raise OperationError(space.w_TypeError, space.wrap(e.message)) mmap.unwrap_spec = [ObjSpace, int, 'index', int, int, int] + elif rmmap._MS_WINDOWS: - def mmap(space, fileno, length, tagname="", access=_ACCESS_DEFAULT): - XXX + + def mmap(space, fileno, length, tagname="", access=rmmap._ACCESS_DEFAULT): + try: + return space.wrap(W_MMap(space, rmmap.mmap(fileno, length, + tagname, access))) + except REnvironmentError, e: + raise OperationError(space.w_EnvironmentError, space.wrap(e.message)) + except RValueError, e: + raise OperationError(space.w_ValueError, space.wrap(e.message)) + except RTypeError, e: + raise OperationError(space.w_TypeError, space.wrap(e.message)) mmap.unwrap_spec = [ObjSpace, int, 'index', str, int] constants = rmmap.constants From cfbolz at codespeak.net Sun Oct 14 16:43:01 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 14 Oct 2007 16:43:01 +0200 (CEST) Subject: [pypy-svn] r47449 - pypy/dist/pypy/rpython/tool Message-ID: <20071014144301.0A7C780D4@code0.codespeak.net> Author: cfbolz Date: Sun Oct 14 16:43:01 2007 New Revision: 47449 Modified: pypy/dist/pypy/rpython/tool/gcstat.py Log: huh, fix this. thanks christian Modified: pypy/dist/pypy/rpython/tool/gcstat.py ============================================================================== --- pypy/dist/pypy/rpython/tool/gcstat.py (original) +++ pypy/dist/pypy/rpython/tool/gcstat.py Sun Oct 14 16:43:01 2007 @@ -7,6 +7,7 @@ self.size = size self.birth = birth self.death = death + self.varsize = varsize def parse_file(f, callback): unknown_lifetime = {} @@ -35,7 +36,6 @@ for unknown in unknown_lifetime.itervalues(): unknown.death = current callback(unknown) - return all def collect_all(f): all = [] From mwh at codespeak.net Sun Oct 14 18:57:06 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sun, 14 Oct 2007 18:57:06 +0200 (CEST) Subject: [pypy-svn] r47450 - pypy/dist/pypy/rpython/memory/gctransform/test Message-ID: <20071014165706.74EC98159@code0.codespeak.net> Author: mwh Date: Sun Oct 14 18:57:05 2007 New Revision: 47450 Modified: pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py Log: a test for write_barrier support in the gctransformer, skipped of course as there is no such support yet. Modified: pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py Sun Oct 14 18:57:05 2007 @@ -1,8 +1,14 @@ +from pypy.objspace.flow.model import Constant, SpaceOperation +from pypy.annotation.model import SomeInteger +from pypy.rpython.memory.gc.base import GCBase from pypy.rpython.memory.gctransform.test.test_transform import rtype +from pypy.rpython.memory.gctransform.transform import GcHighLevelOp from pypy.rpython.memory.gctransform.framework import FrameworkGCTransformer, CollectAnalyzer from pypy.rpython.lltypesystem import lltype +from pypy.rpython.rtyper import LowLevelOpList from pypy.translator.c.gc import FrameworkGcPolicy from pypy.translator.translator import TranslationContext, graphof +from pypy.translator.unsimplify import varoftype from pypy import conftest import py @@ -53,3 +59,31 @@ t = rtype(g, []) gg = graphof(t, g) assert CollectAnalyzer(t).analyze_direct_call(gg) + + +class WriteBarrierTransformer(FrameworkGCTransformer): + GC_PARAMS = {} + class GC_CLASS(GCBase): + def write_barrier(self, addr, addr_to, addr_struct): + addr_to.address[0] = addr + +def test_write_barrier_support(): + py.test.skip("no write barrier support yet!") + t = TranslationContext() + t.buildannotator().build_types(lambda x:x, [SomeInteger()]) + t.buildrtyper().specialize() + llops = LowLevelOpList() + PTR_TYPE = lltype.Ptr(lltype.GcStruct('S', ('x', lltype.Signed))) + spaceop = SpaceOperation( + "setfield", + [varoftype(PTR_TYPE), Constant('x', lltype.Void)], + varoftype(lltype.Void)) + transformer = WriteBarrierTransformer(t) + hop = GcHighLevelOp(transformer, spaceop, llops) + hop.dispatch() + found = False + for op in llops: + if op.opname == 'direct_call': + found = True + break + assert found From arigo at codespeak.net Sun Oct 14 19:26:30 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 14 Oct 2007 19:26:30 +0200 (CEST) Subject: [pypy-svn] r47451 - in pypy/dist/pypy/rpython/memory: . test Message-ID: <20071014172630.178828157@code0.codespeak.net> Author: arigo Date: Sun Oct 14 19:26:29 2007 New Revision: 47451 Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py pypy/dist/pypy/rpython/memory/test/test_gc.py Log: A test that asks for the id() of a relatively large number of objects. The goal is to make this test run faster than it does now on a SemiSpaceGC. Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Sun Oct 14 19:26:29 2007 @@ -50,6 +50,10 @@ else: return lltype.malloc(TYPE, n, flavor=flavor, zero=zero) + def free(self, TYPE, flavor='gc'): + assert flavor != 'gc' + return lltype.free(TYPE, flavor=flavor) + def setfield(self, obj, fieldname, fieldvalue): STRUCT = lltype.typeOf(obj).TO addr = llmemory.cast_ptr_to_adr(obj) Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Sun Oct 14 19:26:29 2007 @@ -270,6 +270,26 @@ res = self.interpret(f, []) assert res == 0 + def test_many_ids(self): + class A(object): + pass + def f(): + from pypy.rpython.lltypesystem import lltype, rffi + alist = [A() for i in range(500)] + idarray = lltype.malloc(rffi.INTP.TO, len(alist), flavor='raw') + # Compute the id of all elements of the list. The goal is + # to not allocate memory, so that if the GC needs memory to + # remember the ids, it will trigger some collections itself + for i in range(len(alist)): + idarray[i] = id(alist[i]) + for j in range(2): + if j == 1: # allocate some stuff between the two iterations + [A() for i in range(200)] + for i in range(len(alist)): + assert idarray[i] == id(alist[i]) + lltype.free(idarray, flavor='raw') + self.interpret(f, []) + class TestMarkSweepGC(GCTest): from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass From arigo at codespeak.net Sun Oct 14 21:13:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 14 Oct 2007 21:13:40 +0200 (CEST) Subject: [pypy-svn] r47452 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071014191340.0D018814A@code0.codespeak.net> Author: arigo Date: Sun Oct 14 21:13:38 2007 New Revision: 47452 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Log: Better error message when we switch the two arguments to rffi.cast() Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Sun Oct 14 21:13:38 2007 @@ -599,6 +599,8 @@ def force_cast(RESTYPE, value): """Cast a value to a result type, trying to use the same rules as C.""" + if not isinstance(RESTYPE, lltype.LowLevelType): + raise TypeError("rffi.cast() first arg should be a TYPE") TYPE1 = lltype.typeOf(value) cvalue = lltype2ctypes(value) cresulttype = get_ctypes_type(RESTYPE) From arigo at codespeak.net Sun Oct 14 22:54:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 14 Oct 2007 22:54:04 +0200 (CEST) Subject: [pypy-svn] r47453 - in pypy/dist/pypy: rpython/lltypesystem/test rpython/memory/gc rpython/memory/test translator/c/test Message-ID: <20071014205404.4E0678141@code0.codespeak.net> Author: arigo Date: Sun Oct 14 22:54:02 2007 New Revision: 47453 Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/test/test_gc.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py pypy/dist/pypy/translator/c/test/test_newgc.py Log: Implement some cleverness and safely in the id() for moving collectors. Sadly, this is based on address trickery that makes testing on top of the llinterp a bit annoying. So this check-in contains many new tests, most of them skipped :-/ At least, the genc test passes. Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py Sun Oct 14 22:54:02 2007 @@ -114,10 +114,10 @@ SX = lltype.Struct('S', ('x',lltype.Signed)) +SPTR = lltype.Ptr(SX) precomputed_size = round_up_for_allocation(llmemory.sizeof(SX)) def test_look_inside_object(): - SPTR = lltype.Ptr(SX) myarenasize = 50 a = arena_malloc(myarenasize, False) b = a + round_up_for_allocation(llmemory.sizeof(lltype.Char)) @@ -132,6 +132,19 @@ arena_free(a) return 42 +def test_address_eq_as_int(): + a = arena_malloc(50, False) + arena_reserve(a, precomputed_size) + p = llmemory.cast_adr_to_ptr(a, SPTR) + a1 = llmemory.cast_ptr_to_adr(p) + assert a == a1 + assert not (a != a1) + assert (a+1) != a1 + assert not ((a+1) == a1) + py.test.skip("cast_adr_to_int() is hard to get consistent") + assert llmemory.cast_adr_to_int(a) == llmemory.cast_adr_to_int(a1) + assert llmemory.cast_adr_to_int(a+1) == llmemory.cast_adr_to_int(a1) + 1 + def test_llinterpreted(): from pypy.rpython.test.test_llinterp import interpret Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Sun Oct 14 22:54:02 2007 @@ -1,4 +1,5 @@ from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rlib.objectmodel import debug_assert class GCBase(object): _alloc_flavor_ = "raw" @@ -86,32 +87,87 @@ def __init__(self): self.wr_to_objects_with_id = [] + self.object_id_dict = {} + self.object_id_dict_ends_at = 0 def id(self, ptr): - # XXX linear search! this is probably too slow to be reasonable :-( - # On the other hand, it punishes you for using 'id', so that's good :-) + self.disable_finalizers() + try: + return self._compute_id(ptr) + finally: + self.enable_finalizers() + + def _compute_id(self, ptr): # XXX this may explode if --no-translation-rweakref is specified + # ---------------------------------------------------------------- + # Basic logic: the list item wr_to_objects_with_id[i] contains a + # weakref to the object whose id is i + 1. The object_id_dict is + # an optimization that tries to reduce the number of linear + # searches in this list. + # ---------------------------------------------------------------- + # Invariant: if object_id_dict_ends_at >= 0, then object_id_dict + # contains (at least) all pairs {address: id}, for the addresses + # of all objects that are the targets of the weakrefs of the + # following slice: wr_to_objects_with_id[:object_id_dict_ends_at]. + # ---------------------------------------------------------------- + # Essential: as long as notify_objects_just_moved() is not called, + # we assume that the objects' addresses did not change. + # ---------------------------------------------------------------- + # First check the dictionary + i = self.object_id_dict_ends_at + if i < 0: + self.object_id_dict.clear() # dictionary invalid + self.object_id_dict_ends_at = 0 + i = 0 + else: + adr = llmemory.cast_ptr_to_adr(ptr) + try: + i = self.object_id_dict[adr] + except KeyError: + pass + else: + # double-check that the answer we got is correct + lst = self.wr_to_objects_with_id + target = llmemory.weakref_deref(llmemory.GCREF, lst[i]) + debug_assert(target == ptr, "bogus object_id_dict") + return i + 1 # found via the dict + # Walk the tail of the list, where entries are not also in the dict lst = self.wr_to_objects_with_id - i = len(lst) + end = len(lst) freeentry = -1 - while i > 0: - i -= 1 + while i < end: target = llmemory.weakref_deref(llmemory.GCREF, lst[i]) if not target: freeentry = i - elif target == ptr: - break # found + else: + # record this entry in the dict + adr = llmemory.cast_ptr_to_adr(target) + self.object_id_dict[adr] = i + if target == ptr: + break # found + i += 1 else: # not found wr = llmemory.weakref_create(ptr) - if freeentry == -1: - i = len(lst) + if freeentry < 0: + debug_assert(end == len(lst), "unexpected lst growth in gc_id") + i = end lst.append(wr) else: i = freeentry # reuse the id() of a dead object lst[i] = wr + adr = llmemory.cast_ptr_to_adr(ptr) + self.object_id_dict[adr] = i + # all entries up to and including index 'i' are now valid in the dict + # unless a collection occurred while we were working, in which case + # the object_id_dict is bogus anyway + if self.object_id_dict_ends_at >= 0: + self.object_id_dict_ends_at = i + 1 return i + 1 # this produces id() values 1, 2, 3, 4... + def notify_objects_just_moved(self): + self.object_id_dict_ends_at = -1 + def choose_gc_from_config(config): """Return a (GCClass, GC_PARAMS) from the given config object. Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Sun Oct 14 22:54:02 2007 @@ -40,8 +40,16 @@ self.free = self.tospace self.objects_with_finalizers = self.AddressLinkedList() self.run_finalizers = self.AddressLinkedList() - self.executing_finalizers = False self.objects_with_weakrefs = self.AddressLinkedList() + self.finalizer_lock_count = 0 + + def disable_finalizers(self): + self.finalizer_lock_count += 1 + + def enable_finalizers(self): + self.finalizer_lock_count -= 1 + if self.run_finalizers.non_empty(): + self.execute_finalizers() def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): @@ -171,13 +179,14 @@ scan = self.free = tospace self.collect_roots() scan = self.scan_copied(scan) - if self.objects_with_weakrefs.non_empty(): - self.invalidate_weakrefs() if self.run_finalizers.non_empty(): self.update_run_finalizers() if self.objects_with_finalizers.non_empty(): self.deal_with_objects_with_finalizers() scan = self.scan_copied(scan) + if self.objects_with_weakrefs.non_empty(): + self.invalidate_weakrefs() + self.notify_objects_just_moved() if not size_changing: llarena.arena_reset(fromspace, self.space_size, True) self.execute_finalizers() @@ -326,9 +335,9 @@ self.run_finalizers = new_run_finalizer def execute_finalizers(self): - if self.executing_finalizers: + if self.finalizer_lock_count > 0: return # the outer invocation of execute_finalizers() will do it - self.executing_finalizers = True + self.finalizer_lock_count = 1 try: while self.run_finalizers.non_empty(): obj = self.run_finalizers.pop() @@ -336,7 +345,7 @@ finalizer = self.getfinalizer(hdr.typeid) finalizer(obj) finally: - self.executing_finalizers = False + self.finalizer_lock_count = 0 STATISTICS_NUMBERS = 0 Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Sun Oct 14 22:54:02 2007 @@ -252,6 +252,10 @@ assert res def test_id(self): + py.test.skip("the MovingGCBase.id() logic can't be directly run") + # XXX ^^^ the problem is that the MovingGCBase instance holds + # references to GC objects - a list of weakrefs and a dict - and + # there is no way we can return these from get_roots_from_llinterp(). class A(object): pass a1 = A() @@ -270,26 +274,6 @@ res = self.interpret(f, []) assert res == 0 - def test_many_ids(self): - class A(object): - pass - def f(): - from pypy.rpython.lltypesystem import lltype, rffi - alist = [A() for i in range(500)] - idarray = lltype.malloc(rffi.INTP.TO, len(alist), flavor='raw') - # Compute the id of all elements of the list. The goal is - # to not allocate memory, so that if the GC needs memory to - # remember the ids, it will trigger some collections itself - for i in range(len(alist)): - idarray[i] = id(alist[i]) - for j in range(2): - if j == 1: # allocate some stuff between the two iterations - [A() for i in range(200)] - for i in range(len(alist)): - assert idarray[i] == id(alist[i]) - lltype.free(idarray, flavor='raw') - self.interpret(f, []) - class TestMarkSweepGC(GCTest): from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Sun Oct 14 22:54:02 2007 @@ -663,3 +663,31 @@ from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass GC_PARAMS = {'space_size': 2048} root_stack_depth = 200 + + def test_many_ids(self): + py.test.skip("fails for bad reasons in lltype.py :-(") + class A(object): + pass + def f(): + from pypy.rpython.lltypesystem import lltype, rffi + alist = [A() for i in range(50)] + idarray = lltype.malloc(rffi.INTP.TO, len(alist), flavor='raw') + # Compute the id of all the elements of the list. The goal is + # to not allocate memory, so that if the GC needs memory to + # remember the ids, it will trigger some collections itself + i = 0 + while i < len(alist): + idarray[i] = id(alist[i]) + i += 1 + j = 0 + while j < 2: + if j == 1: # allocate some stuff between the two iterations + [A() for i in range(20)] + i = 0 + while i < len(alist): + assert idarray[i] == id(alist[i]) + i += 1 + j += 1 + lltype.free(idarray, flavor='raw') + run = self.runner(f) + run([]) Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Sun Oct 14 22:54:02 2007 @@ -827,3 +827,33 @@ class TestSemiSpaceGC(TestUsingFramework): frameworkgc = "semispace" should_be_moving = True + + def test_many_ids(self): + class A(object): + pass + def f(): + from pypy.rpython.lltypesystem import lltype, rffi + alist = [A() for i in range(50000)] + idarray = lltype.malloc(rffi.INTP.TO, len(alist), flavor='raw') + # Compute the id of all elements of the list. The goal is + # to not allocate memory, so that if the GC needs memory to + # remember the ids, it will trigger some collections itself + i = 0 + while i < len(alist): + idarray[i] = id(alist[i]) + i += 1 + j = 0 + while j < 2: + if j == 1: # allocate some stuff between the two iterations + [A() for i in range(20000)] + i = 0 + while i < len(alist): + if idarray[i] != id(alist[i]): + return j * 1000000 + i + i += 1 + j += 1 + lltype.free(idarray, flavor='raw') + return -2 + fn = self.getcompiled(f) + res = fn() + assert res == -2 From cfbolz at codespeak.net Sun Oct 14 22:54:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 14 Oct 2007 22:54:50 +0200 (CEST) Subject: [pypy-svn] r47454 - in pypy/dist/pypy/rpython/memory: . gctransform Message-ID: <20071014205450.5F86F8141@code0.codespeak.net> Author: cfbolz Date: Sun Oct 14 22:54:49 2007 New Revision: 47454 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctypelayout.py Log: fix old typo: finalyzer -> finalizer Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Sun Oct 14 22:54:49 2007 @@ -52,7 +52,7 @@ OFFSETS_TO_GC_PTR = lltype.Array(lltype.Signed) TYPE_INFO = lltype.Struct("type_info", ("isvarsize", lltype.Bool), - ("finalyzer", self.FINALIZERTYPE), + ("finalizer", self.FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), ("varitemsize", lltype.Signed), @@ -67,9 +67,9 @@ debug_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].isvarsize - def q_finalyzer(typeid): + def q_finalizer(typeid): debug_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].finalyzer + return gcdata.type_info_table[typeid].finalizer def q_offsets_to_gc_pointers(typeid): debug_assert(typeid > 0, "invalid type_id") @@ -126,7 +126,7 @@ gcdata.gc.setup() gcdata.gc.set_query_functions( q_is_varsize, - q_finalyzer, + q_finalizer, q_offsets_to_gc_pointers, q_fixed_size, q_varsize_item_sizes, @@ -396,6 +396,7 @@ newgcdependencies.append(ll_static_roots_inside) return newgcdependencies + def gct_direct_call(self, hop): if self.collect_analyzer.analyze(hop.spaceop): self.push_roots(hop) Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Sun Oct 14 22:54:49 2007 @@ -32,7 +32,7 @@ self.id_of_type[TYPE] = type_id offsets = offsets_to_gc_pointers(TYPE) info["ofstoptrs"] = self.offsets2table(offsets, TYPE) - info["finalyzer"] = self.make_finalizer_funcptr_for_type(TYPE) + info["finalizer"] = self.make_finalizer_funcptr_for_type(TYPE) info["weakptrofs"] = weakpointer_offset(TYPE) if not TYPE._is_varsize(): info["isvarsize"] = False @@ -90,9 +90,9 @@ assert typeid > 0 return self.type_info_list[typeid]["isvarsize"] - def q_finalyzer(self, typeid): + def q_finalizer(self, typeid): assert typeid > 0 - return self.type_info_list[typeid]["finalyzer"] + return self.type_info_list[typeid]["finalizer"] def q_offsets_to_gc_pointers(self, typeid): assert typeid > 0 @@ -124,7 +124,7 @@ def get_query_functions(self): return (self.q_is_varsize, - self.q_finalyzer, + self.q_finalizer, self.q_offsets_to_gc_pointers, self.q_fixed_size, self.q_varsize_item_sizes, From cfbolz at codespeak.net Sun Oct 14 22:56:06 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 14 Oct 2007 22:56:06 +0200 (CEST) Subject: [pypy-svn] r47455 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071014205606.C6FBC8141@code0.codespeak.net> Author: cfbolz Date: Sun Oct 14 22:56:06 2007 New Revision: 47455 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: write out a list of typeids and the repr of their types Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Sun Oct 14 22:56:06 2007 @@ -394,8 +394,19 @@ newgcdependencies = [] newgcdependencies.append(table) newgcdependencies.append(ll_static_roots_inside) + self.write_typeid_list() return newgcdependencies + def write_typeid_list(self): + """write out the list of type ids together with some info""" + from pypy.tool.udir import udir + # XXX not ideal since it is not per compilation, but per run + f = udir.join("typeids.txt").open("w") + all = [(typeid, TYPE) + for TYPE, typeid in self.layoutbuilder.id_of_type.iteritems()] + all.sort() + for typeid, TYPE in all: + f.write("%s %s\n" % (typeid, TYPE)) def gct_direct_call(self, hop): if self.collect_analyzer.analyze(hop.spaceop): From arigo at codespeak.net Sun Oct 14 22:56:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 14 Oct 2007 22:56:34 +0200 (CEST) Subject: [pypy-svn] r47456 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071014205634.2E4698141@code0.codespeak.net> Author: arigo Date: Sun Oct 14 22:56:33 2007 New Revision: 47456 Modified: pypy/dist/pypy/rpython/memory/gc/base.py Log: Fix comments. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Sun Oct 14 22:56:33 2007 @@ -106,12 +106,15 @@ # searches in this list. # ---------------------------------------------------------------- # Invariant: if object_id_dict_ends_at >= 0, then object_id_dict - # contains (at least) all pairs {address: id}, for the addresses + # contains all pairs {address: id}, for the addresses # of all objects that are the targets of the weakrefs of the # following slice: wr_to_objects_with_id[:object_id_dict_ends_at]. # ---------------------------------------------------------------- # Essential: as long as notify_objects_just_moved() is not called, - # we assume that the objects' addresses did not change. + # we assume that the objects' addresses did not change. We also + # assume that the address of a live object cannot be reused for + # another object without an intervening notify_objects_just_moved() + # call, but this could be fixed easily if needed. # ---------------------------------------------------------------- # First check the dictionary i = self.object_id_dict_ends_at From mwh at codespeak.net Mon Oct 15 01:04:48 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Mon, 15 Oct 2007 01:04:48 +0200 (CEST) Subject: [pypy-svn] r47458 - pypy/dist/pypy/interpreter Message-ID: <20071014230448.E02A5814B@code0.codespeak.net> Author: mwh Date: Mon Oct 15 01:04:48 2007 New Revision: 47458 Modified: pypy/dist/pypy/interpreter/gateway.py Log: this makes test_gateway pass on my 64-bit python, but it's obviously a bit of a hack. translation gets as far as rtyping now... Modified: pypy/dist/pypy/interpreter/gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/gateway.py (original) +++ pypy/dist/pypy/interpreter/gateway.py Mon Oct 15 01:04:48 2007 @@ -20,7 +20,7 @@ from pypy.interpreter.argument import Arguments, AbstractArguments from pypy.tool.sourcetools import NiceCompile, compile2 from pypy.rlib.jit import hint -from pypy.rlib.rarithmetic import r_longlong +from pypy.rlib.rarithmetic import r_longlong, r_int # internal non-translatable parts: import py @@ -212,8 +212,12 @@ def visit__object(self, typ): if typ not in (int, str, float, r_longlong): assert False, "unsupported basic type in uwnrap_spec" + if typ is r_int is r_longlong: + name = 'r_longlong' + else: + name = typ.__name__ self.run_args.append("space.%s_w(%s)" % - (typ.__name__, self.scopenext())) + (name, self.scopenext())) def visit_index(self, typ): self.run_args.append("space.getindex_w(%s, space.w_OverflowError)" From tismer at codespeak.net Mon Oct 15 02:30:40 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Mon, 15 Oct 2007 02:30:40 +0200 (CEST) Subject: [pypy-svn] r47459 - pypy/dist/pypy/rlib Message-ID: <20071015003040.2EA3D8142@code0.codespeak.net> Author: tismer Date: Mon Oct 15 02:30:38 2007 New Revision: 47459 Added: pypy/dist/pypy/rlib/rmmap2.py Log: added this version of rmmap.py for inspection. I get obscure errors from llinterp, although it works fine in the ctypes emulation (replacing interpret by apply in the test). I had problems with casting to pointer types, llinterp seems to have different rules (odd pointers) than rffi. Also, after avoiding pointer casts so far, this version says [llinterp:error] AN ERROR OCCURED: assert isinstance(, ) [llinterp:error] + where = lltype.Number Since I don't use any floats, there must something else be quite wrong. Please look into this to make rffi really work. (or make it tell be about my real errors) Added: pypy/dist/pypy/rlib/rmmap2.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/rmmap2.py Mon Oct 15 02:30:38 2007 @@ -0,0 +1,717 @@ + +from pypy.rpython.tool import rffi_platform +from pypy.rpython.lltypesystem import rffi, lltype, llmemory + +import sys +import os +import platform +import stat + +_POSIX = os.name == "posix" +_MS_WINDOWS = os.name == "nt" +_LINUX = "linux" in sys.platform +_64BIT = "64bit" in platform.architecture()[0] + +class RValueError(Exception): + def __init__(self, message): + self.message = message + +class REnvironmentError(Exception): + def __init__(self, message): + self.message = message + +class RTypeError(Exception): + def __init__(self, message): + self.message = message + +class CConfig: + _includes_ = ["sys/types.h"] + if _POSIX: + _includes_.append('unistd.h') + _header_ = '#define _GNU_SOURCE\n' + size_t = rffi_platform.SimpleType("size_t", rffi.LONG) + off_t = rffi_platform.SimpleType("off_t", rffi.LONG) + if _MS_WINDOWS: + DWORD_PTR = rffi_platform.SimpleType("DWORD_PTR", rffi.LONG) + WORD = rffi_platform.SimpleType("WORD", rffi.UINT) + DWORD = rffi_platform.SimpleType("DWORD", rffi.ULONG) + BOOL = rffi_platform.SimpleType("BOOL", rffi.LONG) + INT = rffi_platform.SimpleType("INT", rffi.INT) + LONG = rffi_platform.SimpleType("LONG", rffi.LONG) + PLONG = rffi_platform.SimpleType("PLONG", rffi.LONGP) + LPVOID = rffi_platform.SimpleType("LPVOID", rffi.INTP) + LPCVOID = rffi_platform.SimpleType("LPCVOID", rffi.VOIDP) + HANDLE = rffi_platform.SimpleType("HANDLE", rffi.VOIDP) + LPHANDLE = rffi_platform.SimpleType("LPHANDLE", rffi.CCHARPP) + LPCTSTR = rffi_platform.SimpleType("LPCTSTR", rffi.CCHARP) + LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.INTP) + LPSECURITY_ATTRIBUTES = rffi_platform.SimpleType("LPSECURITY_ATTRIBUTES", rffi.CCHARP) + SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T) + +constants = {} +if _POSIX: + CConfig._includes_ += ("sys/mman.h",) + # constants, look in sys/mman.h and platform docs for the meaning + # some constants are linux only so they will be correctly exposed outside + # depending on the OS + constant_names = ['MAP_SHARED', 'MAP_PRIVATE', + 'PROT_READ', 'PROT_WRITE', + 'MS_SYNC'] + opt_constant_names = ['MAP_ANON', 'MAP_ANONYMOUS', + 'PROT_EXEC', + 'MAP_DENYWRITE', 'MAP_EXECUTABLE'] + for name in constant_names: + setattr(CConfig, name, rffi_platform.ConstantInteger(name)) + for name in opt_constant_names: + setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name)) + + CConfig.MREMAP_MAYMOVE = ( + rffi_platform.DefinedConstantInteger("MREMAP_MAYMOVE")) + CConfig.has_mremap = rffi_platform.Has('mremap(NULL, 0, 0, 0)') + # a dirty hack, this is probably a macro + +elif _MS_WINDOWS: + CConfig._includes_ += ("windows.h",) + constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', + 'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY', + 'DUPLICATE_SAME_ACCESS'] + for name in constant_names: + setattr(CConfig, name, rffi_platform.ConstantInteger(name)) + +# export the constants inside and outside. see __init__.py +cConfig = rffi_platform.configure(CConfig) +constants.update(cConfig) + +if _POSIX: + # MAP_ANONYMOUS is not always present but it's always available at CPython level + if constants["MAP_ANONYMOUS"] is None: + constants["MAP_ANONYMOUS"] = constants["MAP_ANON"] + assert constants["MAP_ANONYMOUS"] is not None + constants["MAP_ANON"] = constants["MAP_ANONYMOUS"] + +locals().update(constants) + +_ACCESS_DEFAULT, ACCESS_READ, ACCESS_WRITE, ACCESS_COPY = range(4) + +def external(name, args, result): + return rffi.llexternal(name, args, result, includes=CConfig._includes_) + +def winexternal(name, args, result): + return rffi.llexternal(name, args, result, includes=CConfig._includes_, calling_conv='win') + +PTR = rffi.CCHARP + +c_memmove = external('memmove', [PTR, PTR, size_t], lltype.Void) + +if _POSIX: + has_mremap = cConfig['has_mremap'] + c_mmap = external('mmap', [PTR, size_t, rffi.INT, rffi.INT, + rffi.INT, off_t], PTR) + c_munmap = external('munmap', [PTR, size_t], rffi.INT) + c_msync = external('msync', [PTR, size_t, rffi.INT], rffi.INT) + if has_mremap: + c_mremap = external('mremap', [PTR, size_t, size_t, rffi.ULONG], PTR) + + _get_page_size = external('getpagesize', [], rffi.INT) + + def _get_error_msg(): + errno = rffi.get_errno() + return os.strerror(errno) + +elif _MS_WINDOWS: + + class ComplexCConfig: + _includes_ = CConfig._includes_ + + SYSINFO_STRUCT = rffi.CStruct( + 'SYSINFO_STRUCT', + ("wProcessorArchitecture", WORD), + ("wReserved", WORD), + ) + + SYSINFO_UNION = rffi.CStruct( + 'union SYSINFO_UNION', + ("dwOemId", DWORD), + ("_struct_", SYSINFO_STRUCT), + ) + # sorry, I can't find a way to insert the above + # because the union field has no name + SYSTEM_INFO = rffi_platform.Struct( + 'SYSTEM_INFO', [ + ## ("_union_", SYSINFO_UNION), + ## instead, we put the smaller fields, here + ("wProcessorArchitecture", WORD), + ("wReserved", WORD), + ## should be a union. dwOemId is obsolete, anyway + ("dwPageSize", DWORD), + ("lpMinimumApplicationAddress", LPVOID), + ("lpMaximumApplicationAddress", LPVOID), + ("dwActiveProcessorMask", DWORD_PTR), + ("dwNumberOfProcessors", DWORD), + ("dwProcessorType", DWORD), + ("dwAllocationGranularity", DWORD), + ("wProcessorLevel", WORD), + ("wProcessorRevision", WORD), + ]) + + SECURITY_ATTRIBUTES = rffi_platform.Struct( + 'SECURITY_ATTRIBUTES', [ + ("nLength", DWORD), + ("lpSecurityDescriptor", LPVOID), + ("bInheritHandle", BOOL), + ]) + + config = rffi_platform.configure(ComplexCConfig) + SYSTEM_INFO = config['SYSTEM_INFO'] + SYSTEM_INFO_P = lltype.Ptr(SYSTEM_INFO) + + GetSystemInfo = winexternal('GetSystemInfo', [SYSTEM_INFO_P], lltype.Void) + GetFileSize = winexternal('GetFileSize', [HANDLE, LPDWORD], DWORD) + GetCurrentProcess = winexternal('GetCurrentProcess', [], HANDLE) + DuplicateHandle = winexternal('DuplicateHandle', [HANDLE, HANDLE, HANDLE, LPHANDLE, DWORD, BOOL, DWORD], BOOL) + CreateFileMapping = winexternal('CreateFileMappingA', [HANDLE, LPSECURITY_ATTRIBUTES, DWORD, DWORD, DWORD, LPCTSTR], HANDLE) + MapViewOfFile = winexternal('MapViewOfFile', [HANDLE, DWORD, DWORD, DWORD, SIZE_T], LPCTSTR)##!!LPVOID) + CloseHandle = winexternal('CloseHandle', [HANDLE], BOOL) + UnmapViewOfFile = winexternal('UnmapViewOfFile', [LPCVOID], BOOL) + FlushViewOfFile = winexternal('FlushViewOfFile', [LPCVOID, SIZE_T], BOOL) + SetFilePointer = winexternal('SetFilePointer', [HANDLE, LONG, PLONG, DWORD], DWORD) + SetEndOfFile = winexternal('SetEndOfFile', [HANDLE], BOOL) + ##_get_osfhandle = winexternal('_get_osfhandle', [INT], LONG) + # casting from int to handle did not work, so I changed this + # but it should not be so! + _get_osfhandle = winexternal('_get_osfhandle', [INT], HANDLE) + GetLastError = winexternal('GetLastError', [], DWORD) + + + def _get_page_size(): + try: + si = rffi.make(SYSTEM_INFO) + GetSystemInfo(si) + return int(si.c_dwPageSize) + finally: + lltype.free(si, flavor="raw") + + def _get_file_size(handle): + # XXX use native Windows types like WORD + high_ref = lltype.malloc(LPDWORD.TO, 1, flavor='raw') + try: + low = GetFileSize(handle, high_ref) + low = rffi.cast(lltype.Signed, low) + # XXX should be propagate the real type, allowing + # for 2*sys.maxint? + high = high_ref[0] + # low might just happen to have the value INVALID_FILE_SIZE + # so we need to check the last error also + INVALID_FILE_SIZE = -1 + NO_ERROR = 0 + dwErr = GetLastError() + err = rffi.cast(lltype.Signed, dwErr) + if low == INVALID_FILE_SIZE and err != NO_ERROR: + raise REnvironmentError(os.strerror(err)) + return low, high + finally: + lltype.free(high_ref, flavor='raw') + + def _get_error_msg(): + errno = rffi.cast(lltype.Signed, GetLastError()) + return os.strerror(errno) + +PAGESIZE = _get_page_size() +NULL = lltype.nullptr(PTR.TO) +NODATA = lltype.nullptr(PTR.TO) +NULL_HANDLE = rffi.cast(HANDLE, 0) +INVALID_HANDLE = rffi.cast(HANDLE, -1) + +class MMap(object): + def __init__(self, access): + self.size = 0 + self.pos = 0 + self.access = access + + if _MS_WINDOWS: + self.map_handle = NULL_HANDLE + self.file_handle = NULL_HANDLE + self.tagname = "" + elif _POSIX: + self.fd = -1 + self.closed = False + + def check_valid(self): + if _MS_WINDOWS: + to_close = self.map_handle == INVALID_HANDLE + elif _POSIX: + to_close = self.closed + + if to_close: + raise RValueError("map closed or invalid") + + def check_writeable(self): + if not (self.access != ACCESS_READ): + raise RTypeError("mmap can't modify a readonly memory map.") + + def check_resizeable(self): + if not (self.access == ACCESS_WRITE or self.access == _ACCESS_DEFAULT): + raise RTypeError("mmap can't resize a readonly or copy-on-write memory map.") + + def setdata(self, data, size): + """Set the internal data and map size from a PTR.""" + assert size >= 0 + self.data = data + self.size = size + + def close(self): + if _MS_WINDOWS: + if self.size > 0: + self.unmapview() + self.setdata(NODATA, 0) + if self.map_handle != INVALID_HANDLE: + CloseHandle(self.map_handle) + self.map_handle = INVALID_HANDLE + if self.file_handle != INVALID_HANDLE: + CloseHandle(self.file_handle) + self.file_handle = INVALID_HANDLE + elif _POSIX: + self.closed = True + if self.fd != -1: + os.close(self.fd) + self.fd = -1 + if self.size > 0: + c_munmap(self.getptr(0), self.size) + self.setdata(NODATA, 0) + + def unmapview(self): + UnmapViewOfFile(self.getptr(0)) + + def read_byte(self): + self.check_valid() + + if self.pos < self.size: + value = self.data[self.pos] + self.pos += 1 + return value + else: + raise RValueError("read byte out of range") + + def readline(self): + self.check_valid() + + data = self.data + for pos in xrange(self.pos, self.size): + if data[pos] == '\n': + eol = pos + 1 # we're interested in the position after new line + break + else: # no '\n' found + eol = self.size + + res = "".join([data[i] for i in range(self.pos, eol)]) + self.pos += len(res) + return res + + def read(self, num=-1): + self.check_valid() + + if num < 0: + # read all + eol = self.size + else: + eol = self.pos + num + # silently adjust out of range requests + if eol > self.size: + eol = self.size + + res = [self.data[i] for i in range(self.pos, eol)] + res = "".join(res) + self.pos += len(res) + return res + + def find(self, tofind, start=0): + self.check_valid() + + # XXX naive! how can we reuse the rstr algorithm? + if start < 0: + start += self.size + if start < 0: + start = 0 + data = self.data + for p in xrange(start, self.size - len(tofind) + 1): + for q in range(len(tofind)): + if data[p+q] != tofind[q]: + break # position 'p' is not a match + else: + # full match + return p + # failure + return -1 + + def seek(self, pos, whence=0): + self.check_valid() + + dist = pos + how = whence + + if how == 0: # relative to start + where = dist + elif how == 1: # relative to current position + where = self.pos + dist + elif how == 2: # relative to the end + where = self.size + dist + else: + raise RValueError("unknown seek type") + + if not (0 <= where <= self.size): + raise RValueError("seek out of range") + + self.pos = where + + def tell(self): + self.check_valid() + return self.pos + + def file_size(self): + self.check_valid() + + size = self.size + if _MS_WINDOWS: + if self.file_handle != INVALID_HANDLE: + low, high = _get_file_size(self.file_handle) + if not high and low <= sys.maxint: + return low + size = (high << 32) + low + elif _POSIX: + st = os.fstat(self.fd) + size = st[stat.ST_SIZE] + if size > sys.maxint: + size = sys.maxint + else: + size = int(size) + return size + + def write(self, data): + self.check_valid() + self.check_writeable() + + data_len = len(data) + if self.pos + data_len > self.size: + raise RValueError("data out of range") + + internaldata = self.data + start = self.pos + for i in range(data_len): + internaldata[start+i] = data[i] + self.pos = start + data_len + + def write_byte(self, byte): + self.check_valid() + + if len(byte) != 1: + raise RTypeError("write_byte() argument must be char") + + self.check_writeable() + self.data[self.pos] = byte[0] + self.pos += 1 + + def getptr(self, offset): + return rffi.ptradd(self.data, offset) + + def flush(self, offset=0, size=0): + self.check_valid() + + if size == 0: + size = self.size + if offset < 0 or size < 0 or offset + size > self.size: + raise RValueError("flush values out of range") + else: + start = self.getptr(offset) + if _MS_WINDOWS: + res = FlushViewOfFile(start, size) + # XXX res == 0 means that an error occurred, but in CPython + # this is not checked + return res + elif _POSIX: +## XXX why is this code here? There is no equivalent in CPython +## if _LINUX: +## # alignment of the address +## value = cast(self.data, c_void_p).value +## aligned_value = value & ~(PAGESIZE - 1) +## # the size should be increased too. otherwise the final +## # part is not "msynced" +## new_size = size + value & (PAGESIZE - 1) + res = c_msync(start, size, MS_SYNC) + if res == -1: + raise REnvironmentError(_get_error_msg()) + + return 0 + + def move(self, dest, src, count): + self.check_valid() + + self.check_writeable() + + # check boundings + if (src < 0 or dest < 0 or count < 0 or + src + count > self.size or dest + count > self.size): + raise RValueError("source or destination out of range") + + datasrc = self.getptr(src) + datadest = self.getptr(dest) + c_memmove(datadest, datasrc, count) + + def resize(self, newsize): + self.check_valid() + + self.check_resizeable() + + if _POSIX: + if not has_mremap: + msg = "mmap: resizing not available -- no mremap()" + raise REnvironmentError(msg) + + # resize the underlying file first + try: + os.ftruncate(self.fd, newsize) + except OSError, e: + raise REnvironmentError(os.strerror(e.errno)) + + # now resize the mmap + newdata = c_mremap(self.getptr(0), self.size, newsize, + MREMAP_MAYMOVE or 0) + self.setdata(newdata, newsize) + elif _MS_WINDOWS: + # disconnect the mapping + self.unmapview() + CloseHandle(self.map_handle) + + # move to the desired EOF position + if _64BIT: + newsize_high = newsize >> 32 + newsize_low = newsize & 0xFFFFFFFF + else: + newsize_high = 0 + newsize_low = newsize + + FILE_BEGIN = 0 + high_ref = lltype.malloc(PLONG.TO, 1, flavor='raw') + try: + high_ref[0] = newsize_high + SetFilePointer(self.file_handle, newsize_low, high_ref, + FILE_BEGIN) + finally: + lltype.free(high_ref, flavor='raw') + # resize the file + SetEndOfFile(self.file_handle) + # create another mapping object and remap the file view + res = CreateFileMapping(self.file_handle, NULL, PAGE_READWRITE, + newsize_high, newsize_low, self.tagname) + self.map_handle = res + + dwErrCode = 0 + if self.map_handle: + data = MapViewOfFile(self.map_handle, FILE_MAP_WRITE, + 0, 0, 0) + if data: + # XXX we should have a real LPVOID which must always be casted + charp = rffi.cast(LPCTSTR, data) + self.setdata(charp, newsize) + return + else: + dwErrCode = GetLastError() + else: + dwErrCode = GetLastError() + err = rffi.cast(lltype.Signed, dwErrCode) + raise REnvironmentError(os.strerror(err)) + + def len(self): + self.check_valid() + + return self.size + + def getitem(self, index): + self.check_valid() + # simplified version, for rpython + if index < 0: + index += self.size + return self.data[index] + + def setitem(self, index, value): + self.check_valid() + self.check_writeable() + + if len(value) != 1: + raise RValueError("mmap assignment must be " + "single-character string") + if index < 0: + index += self.size + self.data[index] = value[0] + +def _check_map_size(size): + if size < 0: + raise RTypeError("memory mapped size must be positive") + if rffi.cast(size_t, size) != size: + raise OverflowError("memory mapped size is too large (limited by C int)") + +if _POSIX: + def mmap(fileno, length, flags=MAP_SHARED, + prot=PROT_WRITE | PROT_READ, access=_ACCESS_DEFAULT): + + fd = fileno + + # check size boundaries + _check_map_size(length) + map_size = length + + # check access is not there when flags and prot are there + if access != _ACCESS_DEFAULT and ((flags != MAP_SHARED) or\ + (prot != (PROT_WRITE | PROT_READ))): + raise RValueError("mmap can't specify both access and flags, prot.") + + if access == ACCESS_READ: + flags = MAP_SHARED + prot = PROT_READ + elif access == ACCESS_WRITE: + flags = MAP_SHARED + prot = PROT_READ | PROT_WRITE + elif access == ACCESS_COPY: + flags = MAP_PRIVATE + prot = PROT_READ | PROT_WRITE + elif access == _ACCESS_DEFAULT: + pass + else: + raise RValueError("mmap invalid access parameter.") + + # check file size + try: + st = os.fstat(fd) + except OSError: + pass # ignore errors and trust map_size + else: + mode = st[stat.ST_MODE] + size = st[stat.ST_SIZE] + if size > sys.maxint: + size = sys.maxint + else: + size = int(size) + if stat.S_ISREG(mode): + if map_size == 0: + map_size = size + elif map_size > size: + raise RValueError("mmap length is greater than file size") + + m = MMap(access) + if fd == -1: + # Assume the caller wants to map anonymous memory. + # This is the same behaviour as Windows. mmap.mmap(-1, size) + # on both Windows and Unix map anonymous memory. + m.fd = -1 + + flags |= MAP_ANONYMOUS + + else: + try: + m.fd = os.dup(fd) + except OSError, e: + raise REnvironmentError(os.strerror(e.errno)) + + res = c_mmap(NULL, map_size, prot, flags, fd, 0) + if res == rffi.cast(PTR, -1): + raise REnvironmentError(_get_error_msg()) + + m.setdata(res, map_size) + return m +elif _MS_WINDOWS: + def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT): + # check size boundaries + _check_map_size(length) + map_size = length + + flProtect = 0 + dwDesiredAccess = 0 + fh = NULL_HANDLE + + if access == ACCESS_READ: + flProtect = PAGE_READONLY + dwDesiredAccess = FILE_MAP_READ + elif access == _ACCESS_DEFAULT or access == ACCESS_WRITE: + flProtect = PAGE_READWRITE + dwDesiredAccess = FILE_MAP_WRITE + elif access == ACCESS_COPY: + flProtect = PAGE_WRITECOPY + dwDesiredAccess = FILE_MAP_COPY + else: + raise RValueError("mmap invalid access parameter.") + + # assume -1 and 0 both mean invalid file descriptor + # to 'anonymously' map memory. + if fileno != -1 and fileno != 0: + fh = _get_osfhandle(fileno) + # parts of the C library use HANDLE, others just ints + # XXX hack - made _get_osfhandle compatible + if fh == INVALID_HANDLE: + raise REnvironmentError(_get_error_msg()) + # Win9x appears to need us seeked to zero + # SEEK_SET = 0 + # libc._lseek(fileno, 0, SEEK_SET) + + m = MMap(access) + m.file_handle = INVALID_HANDLE + m.map_handle = INVALID_HANDLE + if fh: + # it is necessary to duplicate the handle, so the + # Python code can close it on us + handle_ref = lltype.malloc(LPHANDLE.TO, 1, flavor='raw') + handle_ref[0] = m.file_handle + try: + res = DuplicateHandle(GetCurrentProcess(), # source process handle + fh, # handle to be duplicated + GetCurrentProcess(), # target process handle + handle_ref, # result + 0, # access - ignored due to options value + False, # inherited by child procs? + DUPLICATE_SAME_ACCESS) # options + if not res: + raise REnvironmentError(_get_error_msg()) + m.file_handle = handle_ref[0] + finally: + lltype.free(handle_ref, flavor='raw') + + if not map_size: + low, high = _get_file_size(fh) + if _64BIT: + map_size = (low << 32) + 1 + else: + if high: + # file is too large to map completely + map_size = -1 + else: + map_size = low + + if tagname: + m.tagname = tagname + + # DWORD is a 4-byte int. If int > 4-byte it must be divided + if _64BIT: + size_hi = map_size >> 32 + size_lo = map_size & 0xFFFFFFFF + else: + size_hi = 0 + size_lo = map_size + + m.map_handle = CreateFileMapping(m.file_handle, NULL, flProtect, + size_hi, size_lo, m.tagname) + + if m.map_handle: + res = MapViewOfFile(m.map_handle, dwDesiredAccess, + 0, 0, 0) + if res: + # XXX we should have a real LPVOID which must always be casted + charp = rffi.cast(LPCTSTR, res) + m.setdata(charp, map_size) + return m + else: + dwErr = GetLastError() + else: + dwErr = GetLastError() + err = rffi.cast(lltype.Signed, dwErr) + raise REnvironmentError(os.strerror(err)) + + +# register_external here? From arigo at codespeak.net Mon Oct 15 10:00:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 15 Oct 2007 10:00:37 +0200 (CEST) Subject: [pypy-svn] r47463 - pypy/dist/pypy/rpython/module Message-ID: <20071015080037.0AE9E8135@code0.codespeak.net> Author: arigo Date: Mon Oct 15 10:00:37 2007 New Revision: 47463 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py Log: Maybe 'val' could be a float here. Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Mon Oct 15 10:00:37 2007 @@ -212,7 +212,7 @@ ll_tup = lltype.malloc(TP.TO) for i, (fieldname, TYPE) in enumerate(LL_STAT_FIELDS): val = getattr(st, fieldname) - rffi.setintfield(ll_tup, 'item%d' % i, val) + rffi.setintfield(ll_tup, 'item%d' % i, int(val)) return ll_tup if arg_is_path: From tismer at codespeak.net Mon Oct 15 12:52:35 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Mon, 15 Oct 2007 12:52:35 +0200 (CEST) Subject: [pypy-svn] r47465 - pypy/dist/pypy/rlib Message-ID: <20071015105235.7BA34812F@code0.codespeak.net> Author: tismer Date: Mon Oct 15 12:52:34 2007 New Revision: 47465 Modified: pypy/dist/pypy/rlib/rmmap.py Log: this version worked already. I was blinded by errors coming from ll_os_stat Modified: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/rmmap.py (original) +++ pypy/dist/pypy/rlib/rmmap.py Mon Oct 15 12:52:34 2007 @@ -36,17 +36,17 @@ WORD = rffi_platform.SimpleType("WORD", rffi.UINT) DWORD = rffi_platform.SimpleType("DWORD", rffi.ULONG) BOOL = rffi_platform.SimpleType("BOOL", rffi.LONG) + INT = rffi_platform.SimpleType("INT", rffi.INT) LONG = rffi_platform.SimpleType("LONG", rffi.LONG) + PLONG = rffi_platform.SimpleType("PLONG", rffi.LONGP) LPVOID = rffi_platform.SimpleType("LPVOID", rffi.INTP) - LPCVOID = rffi_platform.SimpleType("LPCVOID", rffi.CCHARP) - INT = rffi_platform.SimpleType("int", rffi.INT) - INT_P = rffi_platform.SimpleType("int *", rffi.INTP) - # HANDLE = rffi_platform.SimpleType("HANDLE", rffi.UINTP) - # XXX change correct usage of handle throughout all the source - # and make all interfaces conformant. For now, we just cheat. - HANDLE = rffi_platform.SimpleType("int", rffi.LONG) + LPCVOID = rffi_platform.SimpleType("LPCVOID", rffi.VOIDP) + HANDLE = rffi_platform.SimpleType("HANDLE", rffi.VOIDP) + LPHANDLE = rffi_platform.SimpleType("LPHANDLE", rffi.CCHARPP) LPCTSTR = rffi_platform.SimpleType("LPCTSTR", rffi.CCHARP) LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.INTP) + LPSECURITY_ATTRIBUTES = rffi_platform.SimpleType("LPSECURITY_ATTRIBUTES", rffi.CCHARP) + SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T) constants = {} if _POSIX: @@ -154,22 +154,32 @@ ("wProcessorRevision", WORD), ]) + SECURITY_ATTRIBUTES = rffi_platform.Struct( + 'SECURITY_ATTRIBUTES', [ + ("nLength", DWORD), + ("lpSecurityDescriptor", LPVOID), + ("bInheritHandle", BOOL), + ]) + config = rffi_platform.configure(ComplexCConfig) SYSTEM_INFO = config['SYSTEM_INFO'] SYSTEM_INFO_P = lltype.Ptr(SYSTEM_INFO) GetSystemInfo = winexternal('GetSystemInfo', [SYSTEM_INFO_P], lltype.Void) GetFileSize = winexternal('GetFileSize', [HANDLE, LPDWORD], DWORD) - GetCurrentProcess = winexternal('GetCurrentProcess', [], INT) - DuplicateHandle = winexternal('DuplicateHandle', [INT, INT, INT, INT_P, DWORD, BOOL, DWORD], BOOL) - CreateFileMapping = winexternal('CreateFileMappingA', [INT, PTR, INT, INT, INT, LPCTSTR], INT) - MapViewOfFile = winexternal('MapViewOfFile', [INT, DWORD, DWORD, DWORD, DWORD], PTR) - CloseHandle = winexternal('CloseHandle', [INT], BOOL) + GetCurrentProcess = winexternal('GetCurrentProcess', [], HANDLE) + DuplicateHandle = winexternal('DuplicateHandle', [HANDLE, HANDLE, HANDLE, LPHANDLE, DWORD, BOOL, DWORD], BOOL) + CreateFileMapping = winexternal('CreateFileMappingA', [HANDLE, LPSECURITY_ATTRIBUTES, DWORD, DWORD, DWORD, LPCTSTR], HANDLE) + MapViewOfFile = winexternal('MapViewOfFile', [HANDLE, DWORD, DWORD, DWORD, SIZE_T], LPCTSTR)##!!LPVOID) + CloseHandle = winexternal('CloseHandle', [HANDLE], BOOL) UnmapViewOfFile = winexternal('UnmapViewOfFile', [LPCVOID], BOOL) - FlushViewOfFile = winexternal('FlushViewOfFile', [LPCVOID, INT], BOOL) - SetFilePointer = winexternal('SetFilePointer', [INT, INT, INT_P, INT], DWORD) - SetEndOfFile = winexternal('SetEndOfFile', [INT], INT) - _get_osfhandle = winexternal('_get_osfhandle', [INT], INT) + FlushViewOfFile = winexternal('FlushViewOfFile', [LPCVOID, SIZE_T], BOOL) + SetFilePointer = winexternal('SetFilePointer', [HANDLE, LONG, PLONG, DWORD], DWORD) + SetEndOfFile = winexternal('SetEndOfFile', [HANDLE], BOOL) + ##_get_osfhandle = winexternal('_get_osfhandle', [INT], LONG) + # casting from int to handle did not work, so I changed this + # but it should not be so! + _get_osfhandle = winexternal('_get_osfhandle', [INT], HANDLE) GetLastError = winexternal('GetLastError', [], DWORD) @@ -209,7 +219,8 @@ PAGESIZE = _get_page_size() NULL = lltype.nullptr(PTR.TO) NODATA = lltype.nullptr(PTR.TO) -INVALID_INT_VALUE = -1 +NULL_HANDLE = rffi.cast(HANDLE, 0) +INVALID_HANDLE = rffi.cast(HANDLE, -1) class MMap(object): def __init__(self, access): @@ -218,8 +229,8 @@ self.access = access if _MS_WINDOWS: - self.map_handle = 0 - self.file_handle = 0 + self.map_handle = NULL_HANDLE + self.file_handle = NULL_HANDLE self.tagname = "" elif _POSIX: self.fd = -1 @@ -227,7 +238,7 @@ def check_valid(self): if _MS_WINDOWS: - to_close = self.map_handle == INVALID_INT_VALUE + to_close = self.map_handle == INVALID_HANDLE elif _POSIX: to_close = self.closed @@ -253,12 +264,12 @@ if self.size > 0: self.unmapview() self.setdata(NODATA, 0) - if self.map_handle != INVALID_INT_VALUE: + if self.map_handle != INVALID_HANDLE: CloseHandle(self.map_handle) - self.map_handle = INVALID_INT_VALUE - if self.file_handle != INVALID_INT_VALUE: + self.map_handle = INVALID_HANDLE + if self.file_handle != INVALID_HANDLE: CloseHandle(self.file_handle) - self.file_handle = INVALID_INT_VALUE + self.file_handle = INVALID_HANDLE elif _POSIX: self.closed = True if self.fd != -1: @@ -361,7 +372,7 @@ size = self.size if _MS_WINDOWS: - if self.file_handle != INVALID_INT_VALUE: + if self.file_handle != INVALID_HANDLE: low, high = _get_file_size(self.file_handle) if not high and low <= sys.maxint: return low @@ -474,13 +485,12 @@ if _64BIT: newsize_high = newsize >> 32 newsize_low = newsize & 0xFFFFFFFF - high_ref = lltype.malloc(DWORD_PTR.TO, 1, flavor='raw') else: newsize_high = 0 newsize_low = newsize - high_ref = lltype.malloc(INT_P.TO, 1, flavor='raw') FILE_BEGIN = 0 + high_ref = lltype.malloc(PLONG.TO, 1, flavor='raw') try: high_ref[0] = newsize_high SetFilePointer(self.file_handle, newsize_low, high_ref, @@ -499,7 +509,9 @@ data = MapViewOfFile(self.map_handle, FILE_MAP_WRITE, 0, 0, 0) if data: - self.setdata(data, newsize) + # XXX we should have a real LPVOID which must always be casted + charp = rffi.cast(LPCTSTR, data) + self.setdata(charp, newsize) return else: dwErrCode = GetLastError() @@ -613,7 +625,7 @@ flProtect = 0 dwDesiredAccess = 0 - fh = 0 + fh = NULL_HANDLE if access == ACCESS_READ: flProtect = PAGE_READONLY @@ -631,19 +643,21 @@ # to 'anonymously' map memory. if fileno != -1 and fileno != 0: fh = _get_osfhandle(fileno) - if fh == -1: + # parts of the C library use HANDLE, others just ints + # XXX hack - made _get_osfhandle compatible + if fh == INVALID_HANDLE: raise REnvironmentError(_get_error_msg()) # Win9x appears to need us seeked to zero # SEEK_SET = 0 # libc._lseek(fileno, 0, SEEK_SET) m = MMap(access) - m.file_handle = INVALID_INT_VALUE - m.map_handle = INVALID_INT_VALUE + m.file_handle = INVALID_HANDLE + m.map_handle = INVALID_HANDLE if fh: # it is necessary to duplicate the handle, so the # Python code can close it on us - handle_ref = lltype.malloc(INT_P.TO, 1, flavor='raw') + handle_ref = lltype.malloc(LPHANDLE.TO, 1, flavor='raw') handle_ref[0] = m.file_handle try: res = DuplicateHandle(GetCurrentProcess(), # source process handle @@ -688,7 +702,9 @@ res = MapViewOfFile(m.map_handle, dwDesiredAccess, 0, 0, 0) if res: - m.setdata(res, map_size) + # XXX we should have a real LPVOID which must always be casted + charp = rffi.cast(LPCTSTR, res) + m.setdata(charp, map_size) return m else: dwErr = GetLastError() From fijal at codespeak.net Mon Oct 15 16:58:32 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 15 Oct 2007 16:58:32 +0200 (CEST) Subject: [pypy-svn] r47466 - in pypy/dist/pypy/module/_ffi: . test Message-ID: <20071015145832.02B2D811B@code0.codespeak.net> Author: fijal Date: Mon Oct 15 16:58:31 2007 New Revision: 47466 Modified: pypy/dist/pypy/module/_ffi/__init__.py pypy/dist/pypy/module/_ffi/app_ffi.py pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: Very strange implementation of structure (only flat ones, no nested structures). Needs rethinking, but at least I've got better idea now. Modified: pypy/dist/pypy/module/_ffi/__init__.py ============================================================================== --- pypy/dist/pypy/module/_ffi/__init__.py (original) +++ pypy/dist/pypy/module/_ffi/__init__.py Mon Oct 15 16:58:31 2007 @@ -10,9 +10,11 @@ applevelname = '_ffi' interpleveldefs = { - 'CDLL' : 'interp_ffi.W_CDLL', - 'FuncPtr' : 'interp_ffi.W_FuncPtr', + 'CDLL' : 'interp_ffi.W_CDLL', + 'FuncPtr' : 'interp_ffi.W_FuncPtr', } appleveldefs = { + 'Structure' : 'app_ffi.Structure', + 'StructureInstance' : 'app_ffi.StructureInstance', } Modified: pypy/dist/pypy/module/_ffi/app_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/app_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/app_ffi.py Mon Oct 15 16:58:31 2007 @@ -0,0 +1,26 @@ +# NOT_RPYTHON + +class Structure(object): + def __init__(self, fields): + self.fields = fields + + def __call__(self, **kwds): + from _ffi import StructureInstance + return StructureInstance(self.fields, kwds) + +class StructureInstance(object): + def __init__(self, shape, **kwds): + self.shape = shape + self.format = "".join([i[1] for i in shape]) + for kwd, value in kwds.items(): + setattr(self, kwd, value) + + def pack(self): + args = [getattr(self, i[0], 0) for i in self.shape] + return struct.pack(self.format, *args) + + def unpack(self, s): + values = struct.unpack(self.format, s) + for (name, _), value in zip(self.shape, values): + setattr(self, name, value) + Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Mon Oct 15 16:58:31 2007 @@ -5,14 +5,48 @@ from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.rlib.libffi import CDLL, RTLD_LOCAL, RTLD_GLOBAL,\ - ffi_type_sint, ffi_type_double, ffi_type_slong +from pypy.rlib.libffi import * from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib.unroll import unrolling_iterable -typemap = { +TYPEMAP = { + # XXX A mess with unsigned/signed/normal chars :-/ + 'c' : ffi_type_uchar, + 'b' : ffi_type_schar, + 'B' : ffi_type_uchar, + 'h' : ffi_type_sshort, + 'H' : ffi_type_ushort, 'i' : ffi_type_sint, + 'I' : ffi_type_uint, 'l' : ffi_type_slong, - 'd' : ffi_type_double + 'L' : ffi_type_ulong, + # XXX I'm not sure what is long long here, let's assume it's 64 bit :-/ + 'q' : ffi_type_sint64, + 'Q' : ffi_type_uint64, + 'f' : ffi_type_float, + 'd' : ffi_type_double, + 's' : ffi_type_pointer, + 'p' : ffi_type_pointer, + 'P' : ffi_type_pointer, +} + +LL_TYPEMAP = { + 'c' : rffi.CHAR, + 'b' : rffi.UCHAR, + 'B' : rffi.CHAR, + 'h' : rffi.SHORT, + 'H' : rffi.USHORT, + 'i' : rffi.INT, + 'I' : rffi.UINT, + 'l' : rffi.LONG, + 'L' : rffi.ULONG, + 'q' : rffi.LONGLONG, + 'Q' : rffi.ULONGLONG, + 'f' : rffi.FLOAT, + 'd' : rffi.DOUBLE, + 's' : rffi.CCHARP, + 'p' : rffi.CCHARP, + 'P' : rffi.VOIDP, } class W_CDLL(Wrappable): @@ -25,7 +59,7 @@ def get_type(self, key): space = self.space try: - return typemap[key] + return TYPEMAP[key] except KeyError: raise OperationError(space.w_ValueError, space.wrap( "Uknown type letter %s" % key)) @@ -68,16 +102,67 @@ W_CDLL.typedef = TypeDef( 'CDLL', __new__ = interp2app(descr_new_cdll), - ptr = interp2app(W_CDLL.ptr) + ptr = interp2app(W_CDLL.ptr), + __doc__ = """ C Dynamically loaded library +use CDLL(libname) to create handle to C library (argument is processed the +same way as dlopen processes it). On such library call: +lib.ptr(func_name, argtype_list, restype) + +where argtype_list is a list of single characters and restype is a single +character. Character meanings are more or less the same as in struct module, +except that s has trailing \x00 added, while p is considered a raw buffer. +""" ) -def push_arg(space, ptr, argtype, w_arg): - if argtype == "i": - ptr.push_arg(space.int_w(w_arg)) - elif argtype == "d": +def pack_pointer(space, w_arg, ptr): + arg = space.str_w(w_arg) + ll_str = lltype.malloc(rffi.CCHARP.TO, len(arg), flavor='raw') + for i in range(len(arg)): + ll_str[i] = arg[i] + ptr.push_arg(ll_str) + return ll_str + +def push_arg(space, ptr, argnum, argtype, w_arg, to_free, to_reconsider): + w = space.wrap + # XXX how to handle LONGLONG here? + # they're probably long, so we'll not get them through int_w + if argtype == "d" or argtype == "f": ptr.push_arg(space.float_w(w_arg)) + elif argtype == "s": + ll_str = rffi.str2charp(space.str_w(w_arg)) + ptr.push_arg(ll_str) + to_free.append(ll_str) + elif argtype == "p": + # check for NULL ptr + if space.is_w(w_arg, space.w_None): + ptr.push_arg(lltype.nullptr(rffi.VOIDP.TO)) + elif space.is_true(space.isinstance(w_arg, space.w_basestring)): + to_free.append(pack_pointer(space, w_arg, ptr)) + else: + mod = space.getbuiltinmodule('_ffi') + w_StructureInstance = space.getattr(mod, w('StructureInstance')) + if space.is_true(space.isinstance(w_arg, w_StructureInstance)): + w_pack = space.getattr(w_arg, w('pack')) + w_res = space.call(w_pack, space.newtuple([])) + size = space.int_w(space.len(w_res)) + ll_ptr = pack_pointer(space, w_res, ptr) + to_reconsider.append((argnum, ll_ptr, size)) + else: + raise OperationError(space.w_TypeError, w( + "Expected structure, array or simple type")) + elif argtype == "c" or argtype == "b" or argtype == "B": + ptr.push_arg(space.str_w(w_arg)) else: - raise TypeError("Stuff changed in between?") + assert argtype in ["iIhHlLqQ"] + ptr.push_arg(space.int_w(w_arg)) + +ll_typemap_iter = unrolling_iterable(LL_TYPEMAP.items()) + +def repack_struct(space, w_struct, value, size): + w_unpack = space.getattr(w_struct, space.wrap('unpack')) + # ARGH! too much copying! + x = "".join([value[i] for i in range(size)]) + space.call(w_unpack, space.newtuple([space.wrap(x)])) class W_FuncPtr(Wrappable): def __init__(self, ptr, argtypes, restype): @@ -91,14 +176,30 @@ if kwds_w: raise OperationError(space.w_TypeError, space.wrap( "Provided keyword arguments for C function call")) + to_free = [] + to_reconsider = [] + i = 0 for argtype, w_arg in zip(self.argtypes, args_w): - push_arg(space, self.ptr, argtype, w_arg) - # a bit of specialcasing, rpython trick instead? - if self.restype == "i": - return space.wrap(self.ptr.call(rffi.INT)) - elif self.restype == "d": - return space.wrap(self.ptr.call(rffi.DOUBLE)) - raise TypeError("Stuff changed in between?") + push_arg(space, self.ptr, i, argtype, w_arg, to_free, to_reconsider) + i += 1 + try: + for c, ll_type in ll_typemap_iter: + if self.restype == c: + if c == 's' or c == 'p': + return space.wrap(rffi.charp2str(self.ptr.call(rffi.CCHARP))) + elif c == 'P': + res = self.ptr.call(rffi.VOIDP) + return space.wrap(lltype.cast_ptr_to_int(res)) + elif c == 'q' or c == 'Q' or c == 'L': + return space.newlong(self.ptr.call(ll_type)) + else: + return space.wrap(self.ptr.call(ll_type)) + finally: + for elem in to_free: + lltype.free(elem, flavor='raw') + for num, value, size in to_reconsider: + repack_struct(space, args_w[num], value, size) + lltype.free(value, flavor='raw') call.unwrap_spec = ['self', ObjSpace, Arguments] W_FuncPtr.typedef = TypeDef( Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Mon Oct 15 16:58:31 2007 @@ -10,7 +10,7 @@ class AppTestCTypes: def setup_class(cls): - cls.space = gettestobjspace(usemodules=('_ffi',)) + cls.space = gettestobjspace(usemodules=('_ffi','struct')) def test_libload(self): import _ffi @@ -46,4 +46,31 @@ assert pow(2, 2) == 4.0 raises(TypeError, "pow('x', 2.0)") - + def test_strlen(self): + import _ffi + libc = _ffi.CDLL('libc.so.6') + strlen = libc.ptr('strlen', ['s'], 'i') + assert strlen("dupa") == 4 + assert strlen("zupa") == 4 + strlen = libc.ptr('strlen', ['p'], 'i') + assert strlen("ddd\x00") == 3 + strdup = libc.ptr('strdup', ['s'], 's') + assert strdup("xxx") == "xxx" + + def test_time(self): + import _ffi + libc = _ffi.CDLL('libc.so.6') + time = libc.ptr('time', ['p'], 'l') + assert time(None) != 0 + + def test_gettimeofday(self): + import _ffi + struct_type = _ffi.Structure([('tv_sec', 'l'), ('tv_usec', 'l')]) + structure = struct_type() + libc = _ffi.CDLL('libc.so.6') + gettimeofday = libc.ptr('gettimeofday', ['p', 'p'], 'i') + assert gettimeofday(structure, None) == 0 + struct2 = struct_type() + assert gettimeofday(struct2, None) == 0 + assert structure.tv_usec != struct2.tv_usec + assert structure.tv_sec == struct2.tv_sec or structure.tv_sec == struct2.tv_sec - 1 From niko at codespeak.net Mon Oct 15 17:33:19 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Mon, 15 Oct 2007 17:33:19 +0200 (CEST) Subject: [pypy-svn] r47467 - in pypy/dist/pypy/translator/jvm: . test Message-ID: <20071015153319.620CD8117@code0.codespeak.net> Author: niko Date: Mon Oct 15 17:33:17 2007 New Revision: 47467 Modified: pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/test/test_op.py Log: fix test compare_big_ullong: bug was that we were doing math with rpython rulonglong objects when we wanted normal python longs Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Mon Oct 15 17:33:17 2007 @@ -23,29 +23,6 @@ def _isinf(v): return v!=0 and (v == v*2) -def _unsigned_to_signed_32(val): - """ In the JVM, we store unsigned integers in a signed integer slot - (since JVM has no signed integers). This function converts an - unsigned value Python integer (possibly a long) into its corresponding - Python signed integer. """ - if val <= 0x7FFFFFFF: - return int(val) - return int(_two_comp_32(val)) - -def _unsigned_to_signed_64(val): - """ Same as _unsigned_to_signed_32, but for longs. """ - if val <= 0x7FFFFFFFFFFFFFFF: - return val - return _two_comp_64(val) - -def _two_comp_32(val): - """ Returns the 32 bit two's complement. """ - return -((val ^ 0xFFFFFFFF)+1) - -def _two_comp_64(val): - """ Returns the 64 bit two's complement. """ - return -((val ^ 0xFFFFFFFFFFFFFFFF)+1) - # ___________________________________________________________________________ # JVM Opcodes: # @@ -1094,6 +1071,7 @@ '0 /* we are not jitted here */': 0} def push_primitive_constant(self, TYPE, value): + if TYPE is ootype.Void: return elif isinstance(value, CDefinedIntSymbolic): @@ -1101,15 +1079,19 @@ elif TYPE in (ootype.Bool, ootype.Signed): self.emit(ICONST, int(value)) elif TYPE is ootype.Unsigned: - # Converts the unsigned int into its corresponding signed value - # and emits it using ICONST. - self.emit(ICONST, _unsigned_to_signed_32(value)) + # Converts the unsigned int into its corresponding signed value: + if value > 0x7FFFFFFF: + value = -((int(val) ^ 0xFFFFFFFF)+1) + self.emit(ICONST, value) elif TYPE is ootype.Char or TYPE is ootype.UniChar: self.emit(ICONST, ord(value)) elif TYPE is ootype.SignedLongLong: self._push_long_constant(long(value)) elif TYPE is ootype.UnsignedLongLong: - self._push_long_constant(_unsigned_to_signed_64(value)) + # Converts the unsigned long into its corresponding signed value: + if value > 0x7FFFFFFFFFFFFFFF: + value = -((long(value) ^ 0xFFFFFFFFFFFFFFFF)+1) + self._push_long_constant(value) elif TYPE is ootype.Float: self._push_double_constant(float(value)) elif TYPE is ootype.String: Modified: pypy/dist/pypy/translator/jvm/test/test_op.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_op.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_op.py Mon Oct 15 17:33:17 2007 @@ -5,5 +5,4 @@ # ====> ../../oosupport/test_template/operations.py class TestOperations(JvmTest, BaseTestOperations): - def test_compare_big_ullongs(self): - py.test.skip('fixme!') + pass From tismer at codespeak.net Mon Oct 15 18:56:06 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Mon, 15 Oct 2007 18:56:06 +0200 (CEST) Subject: [pypy-svn] r47470 - in pypy/dist/pypy: rlib rpython/tool Message-ID: <20071015165606.E91E2810A@code0.codespeak.net> Author: tismer Date: Mon Oct 15 18:56:06 2007 New Revision: 47470 Modified: pypy/dist/pypy/rlib/rmmap.py pypy/dist/pypy/rpython/tool/rffi_platform.py Log: fixed rffi_platform.Has to check against Linkerror as well. Fixed rmmap.py to build after windows changes Modified: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/rmmap.py (original) +++ pypy/dist/pypy/rlib/rmmap.py Mon Oct 15 18:56:06 2007 @@ -216,11 +216,12 @@ errno = rffi.cast(lltype.Signed, GetLastError()) return os.strerror(errno) + NULL_HANDLE = rffi.cast(HANDLE, 0) + INVALID_HANDLE = rffi.cast(HANDLE, -1) + PAGESIZE = _get_page_size() NULL = lltype.nullptr(PTR.TO) NODATA = lltype.nullptr(PTR.TO) -NULL_HANDLE = rffi.cast(HANDLE, 0) -INVALID_HANDLE = rffi.cast(HANDLE, -1) class MMap(object): def __init__(self, access): Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Mon Oct 15 18:56:06 2007 @@ -412,7 +412,8 @@ try: ask_gcc(self.name + ';') return True - except distutils.errors.CompileError: + except (distutils.errors.CompileError, + distutils.errors.LinkError): return False # ____________________________________________________________ From tismer at codespeak.net Mon Oct 15 19:02:42 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Mon, 15 Oct 2007 19:02:42 +0200 (CEST) Subject: [pypy-svn] r47472 - pypy/dist/pypy/rlib Message-ID: <20071015170242.74C3A80FE@code0.codespeak.net> Author: tismer Date: Mon Oct 15 19:02:42 2007 New Revision: 47472 Removed: pypy/dist/pypy/rlib/rmmap2.py Log: removed obsolete file From fijal at codespeak.net Mon Oct 15 19:12:45 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 15 Oct 2007 19:12:45 +0200 (CEST) Subject: [pypy-svn] r47473 - in pypy/dist/pypy/module/_ffi: . test Message-ID: <20071015171245.B4926810E@code0.codespeak.net> Author: fijal Date: Mon Oct 15 19:12:44 2007 New Revision: 47473 Added: pypy/dist/pypy/module/_ffi/structure.py (contents, props changed) pypy/dist/pypy/module/_ffi/test/test_struct.py (contents, props changed) Modified: pypy/dist/pypy/module/_ffi/__init__.py pypy/dist/pypy/module/_ffi/app_ffi.py pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: Another approach to structures. This should scale to nested structures, not rpython by now (will fix it later) Modified: pypy/dist/pypy/module/_ffi/__init__.py ============================================================================== --- pypy/dist/pypy/module/_ffi/__init__.py (original) +++ pypy/dist/pypy/module/_ffi/__init__.py Mon Oct 15 19:12:44 2007 @@ -10,11 +10,11 @@ applevelname = '_ffi' interpleveldefs = { - 'CDLL' : 'interp_ffi.W_CDLL', - 'FuncPtr' : 'interp_ffi.W_FuncPtr', + 'CDLL' : 'interp_ffi.W_CDLL', + 'FuncPtr' : 'interp_ffi.W_FuncPtr', + 'StructureInstance' : 'structure.W_StructureInstance', } appleveldefs = { 'Structure' : 'app_ffi.Structure', - 'StructureInstance' : 'app_ffi.StructureInstance', } Modified: pypy/dist/pypy/module/_ffi/app_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/app_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/app_ffi.py Mon Oct 15 19:12:44 2007 @@ -6,21 +6,4 @@ def __call__(self, **kwds): from _ffi import StructureInstance - return StructureInstance(self.fields, kwds) - -class StructureInstance(object): - def __init__(self, shape, **kwds): - self.shape = shape - self.format = "".join([i[1] for i in shape]) - for kwd, value in kwds.items(): - setattr(self, kwd, value) - - def pack(self): - args = [getattr(self, i[0], 0) for i in self.shape] - return struct.pack(self.format, *args) - - def unpack(self, s): - values = struct.unpack(self.format, s) - for (name, _), value in zip(self.shape, values): - setattr(self, name, value) - + return StructureInstance(self, kwds) Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Mon Oct 15 19:12:44 2007 @@ -122,7 +122,7 @@ ptr.push_arg(ll_str) return ll_str -def push_arg(space, ptr, argnum, argtype, w_arg, to_free, to_reconsider): +def push_arg(space, ptr, argnum, argtype, w_arg, to_free): w = space.wrap # XXX how to handle LONGLONG here? # they're probably long, so we'll not get them through int_w @@ -142,11 +142,8 @@ mod = space.getbuiltinmodule('_ffi') w_StructureInstance = space.getattr(mod, w('StructureInstance')) if space.is_true(space.isinstance(w_arg, w_StructureInstance)): - w_pack = space.getattr(w_arg, w('pack')) - w_res = space.call(w_pack, space.newtuple([])) - size = space.int_w(space.len(w_res)) - ll_ptr = pack_pointer(space, w_res, ptr) - to_reconsider.append((argnum, ll_ptr, size)) + #ptr.push_arg(lltype.cast_int_to_ptr(rffi.VOIDP, space.int_w(space.getattr(w_arg, w('buffer'))))) + ptr.push_arg(w_arg.ll_buffer) else: raise OperationError(space.w_TypeError, w( "Expected structure, array or simple type")) @@ -158,11 +155,20 @@ ll_typemap_iter = unrolling_iterable(LL_TYPEMAP.items()) -def repack_struct(space, w_struct, value, size): - w_unpack = space.getattr(w_struct, space.wrap('unpack')) - # ARGH! too much copying! - x = "".join([value[i] for i in range(size)]) - space.call(w_unpack, space.newtuple([space.wrap(x)])) +def wrap_result(space, restype, func): + for c, ll_type in ll_typemap_iter: + if restype == c: + if c == 's' or c == 'p': + return space.wrap(rffi.charp2str(func(rffi.CCHARP))) + elif c == 'P': + res = func(rffi.VOIDP) + return space.wrap(lltype.cast_ptr_to_int(res)) + elif c == 'q' or c == 'Q' or c == 'L': + return space.newlong(func(ll_type)) + else: + return space.wrap(func(ll_type)) + return space.w_None +wrap_result._annspecialcase_ = 'specialize:arg(2)' class W_FuncPtr(Wrappable): def __init__(self, ptr, argtypes, restype): @@ -177,29 +183,15 @@ raise OperationError(space.w_TypeError, space.wrap( "Provided keyword arguments for C function call")) to_free = [] - to_reconsider = [] i = 0 for argtype, w_arg in zip(self.argtypes, args_w): - push_arg(space, self.ptr, i, argtype, w_arg, to_free, to_reconsider) + push_arg(space, self.ptr, i, argtype, w_arg, to_free) i += 1 try: - for c, ll_type in ll_typemap_iter: - if self.restype == c: - if c == 's' or c == 'p': - return space.wrap(rffi.charp2str(self.ptr.call(rffi.CCHARP))) - elif c == 'P': - res = self.ptr.call(rffi.VOIDP) - return space.wrap(lltype.cast_ptr_to_int(res)) - elif c == 'q' or c == 'Q' or c == 'L': - return space.newlong(self.ptr.call(ll_type)) - else: - return space.wrap(self.ptr.call(ll_type)) + return wrap_result(space, self.restype, self.ptr.call) finally: for elem in to_free: lltype.free(elem, flavor='raw') - for num, value, size in to_reconsider: - repack_struct(space, args_w[num], value, size) - lltype.free(value, flavor='raw') call.unwrap_spec = ['self', ObjSpace, Arguments] W_FuncPtr.typedef = TypeDef( Added: pypy/dist/pypy/module/_ffi/structure.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/_ffi/structure.py Mon Oct 15 19:12:44 2007 @@ -0,0 +1,83 @@ + +""" Interpreter-level implementation of structure, exposing ll-structure +to app-level with apropriate interface +""" + +from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable,\ + Arguments +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.interpreter.error import OperationError, wrap_oserror +# XXX we've got the very same info in two places - one is native_fmttable +# the other one is in rlib/libffi, we should refactor it to reuse the same +# logic, I'll not touch it by now, and refactor it later +from pypy.module.struct.nativefmttable import native_fmttable +from pypy.module._ffi.interp_ffi import wrap_result + +def unpack_fields(space, w_fields): + fields_w = space.unpackiterable(w_fields) + fields = [] + for w_tup in fields_w: + l_w = space.unpackiterable(w_tup) + if not len(l_w) == 2: + raise OperationError(space.w_ValueError, space.wrap( + "Expected list of 2-size tuples")) + fields.append((space.str_w(l_w[0]), space.str_w(l_w[1]))) + return fields + +def size_and_pos(fields): + size = native_fmttable[fields[0][1]]['size'] + pos = [0] + for i in range(1, len(fields)): + field_desc = native_fmttable[fields[i][1]] + missing = size % field_desc.get('alignment', 1) + if missing: + size += field_desc['alignment'] - missing + pos.append(size) + size += field_desc['size'] + return size, pos + +class W_StructureInstance(Wrappable): + def __init__(self, space, w_shape, w_fieldinits): + if space.is_true(w_fieldinits): + raise OperationError(space.w_ValueError, space.wrap( + "Fields should be not initialized with values by now")) + w_fields = space.getattr(w_shape, space.wrap('fields')) + fields = unpack_fields(space, w_fields) + size, pos = size_and_pos(fields) + self.fields = fields + self.ll_buffer = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', + zero=True) + self.ll_positions = pos + self.next_pos = 0 + + def cast_pos(self, ll_t): + i = self.next_pos + pos = rffi.ptradd(self.ll_buffer, self.ll_positions[i]) + TP = rffi.CArrayPtr(ll_t) + return rffi.cast(TP, pos)[0] + cast_pos._annspecialcase_ = 'specialize:arg(1)' + + def getattr(self, space, attr): + for i in range(len(self.fields)): + name, c = self.fields[i] + if name == attr: + # XXX RPython-trick for passing lambda around + self.next_pos = i + return wrap_result(space, c, self.cast_pos) + raise OperationError(space.w_AttributeError, space.wrap( + "C Structure has no attribute %s" % name)) + getattr.unwrap_spec = ['self', ObjSpace, str] + + def __del__(self): + lltype.free(self.ll_buffer, flavor='raw') + +def descr_new_structure_instance(space, w_type, w_shape, w_fieldinits): + return W_StructureInstance(space, w_shape, w_fieldinits) + +W_StructureInstance.typedef = TypeDef( + 'StructureInstance', + __new__ = interp2app(descr_new_structure_instance), + __getattr__ = interp2app(W_StructureInstance.getattr), +) Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Mon Oct 15 19:12:44 2007 @@ -73,4 +73,7 @@ struct2 = struct_type() assert gettimeofday(struct2, None) == 0 assert structure.tv_usec != struct2.tv_usec - assert structure.tv_sec == struct2.tv_sec or structure.tv_sec == struct2.tv_sec - 1 + assert (structure.tv_sec == struct2.tv_sec) or (structure.tv_sec == struct2.tv_sec - 1) + raises(AttributeError, "structure.xxx") + + Added: pypy/dist/pypy/module/_ffi/test/test_struct.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/_ffi/test/test_struct.py Mon Oct 15 19:12:44 2007 @@ -0,0 +1,15 @@ + +from pypy.module._ffi.structure import sizeof + +def unpack(desc): + return [('x', i) for i in desc] + +def test_sizeof(): + s_c = sizeof(unpack('c')) + s_l = sizeof(unpack('l')) + s_q = sizeof(unpack('q')) + assert sizeof(unpack('cl')) == 2*s_l + assert sizeof(unpack('cq')) == s_q + s_l + assert sizeof(unpack('ccq')) == s_q + s_l + assert sizeof(unpack('ccccq')) == 4 * s_c + s_q + From fijal at codespeak.net Mon Oct 15 19:23:49 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 15 Oct 2007 19:23:49 +0200 (CEST) Subject: [pypy-svn] r47474 - in pypy/dist/pypy/rlib: . test Message-ID: <20071015172349.6FE82810A@code0.codespeak.net> Author: fijal Date: Mon Oct 15 19:23:48 2007 New Revision: 47474 Modified: pypy/dist/pypy/rlib/libffi.py pypy/dist/pypy/rlib/test/test_libffi.py Log: Support for pointers. Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Mon Oct 15 19:23:48 2007 @@ -49,7 +49,10 @@ return l base_names = ['double', 'uchar', 'schar', 'sshort', 'ushort', 'uint', 'sint', - 'ulong', 'slong', 'float', 'pointer', 'void'] + 'ulong', 'slong', 'float', 'pointer', 'void', + # by size + 'sint8', 'uint8', 'sint16', 'uint16', 'sint32', 'uint32', + 'sint64', 'uint64'] type_names = ['ffi_type_%s' % name for name in base_names] for i in type_names: add_simple_type(i) @@ -149,6 +152,9 @@ buf[0] = arg push_arg_as_ffiptr._annspecialcase_ = 'specialize:argtype(1)' +def check_pointer_type(TP): + pass + class FuncPtr(object): def __init__(self, name, argtypes, restype, funcsym): self.name = name @@ -175,15 +181,24 @@ self.ll_args[i] = lltype.malloc(rffi.VOIDP.TO, intmask(argtypes[i].c_size), flavor='raw') - self.ll_result = lltype.malloc(rffi.VOIDP.TO, intmask(restype.c_size), - flavor='raw') + if restype != ffi_type_void: + self.ll_result = lltype.malloc(rffi.VOIDP.TO, + intmask(restype.c_size), + flavor='raw') - # XXX some rpython trick to get rid of TP here? def push_arg(self, value): if self.pushed_args == self.argnum: raise TypeError("Too much arguments, eats %d, pushed %d" % (self.argnum, self.argnum + 1)) TP = lltype.typeOf(value) + if isinstance(TP, lltype.Ptr): + if TP.TO._gckind != 'raw': + raise ValueError("Can only push raw values to C, not 'gc'") + # XXX probably we should recursively check for struct fields + # here, lets just ignore that for now + if isinstance(TP.TO, lltype.Array) and not \ + TP.TO._hints.get('nolength', None): + raise ValueError("Can only push to C arrays without length info") push_arg_as_ffiptr(self.argtypes[self.pushed_args], TP, value, self.ll_args[self.pushed_args]) self.pushed_args += 1 @@ -214,7 +229,8 @@ for i in range(argnum): lltype.free(self.ll_args[i], flavor='raw') lltype.free(self.ll_args, flavor='raw') - lltype.free(self.ll_result, flavor='raw') + if self.restype != ffi_type_void: + lltype.free(self.ll_result, flavor='raw') lltype.free(self.ll_cif, flavor='raw') lltype.free(self.ll_argtypes, flavor='raw') Modified: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_libffi.py (original) +++ pypy/dist/pypy/rlib/test/test_libffi.py Mon Oct 15 19:23:48 2007 @@ -9,6 +9,7 @@ from pypy.rpython.lltypesystem import rffi, lltype import os, sys import py +import time def setup_module(mod): if not sys.platform.startswith('linux'): @@ -18,10 +19,6 @@ def setup_method(self, meth): ALLOCATED.clear() - def teardown_method(self, meth): - pass - #assert not ALLOCATED, not yet - def test_dlopen(self): py.test.raises(OSError, "dlopen(rffi.str2charp('xxxxxxxxxxxx'))") assert dlopen(rffi.str2charp('/lib/libc.so.6')) @@ -32,24 +29,30 @@ def test_library_open(self): lib = self.get_libc() del lib + assert not ALLOCATED def test_library_get_func(self): lib = self.get_libc() ptr = lib.getpointer('time', [], ffi_type_void) py.test.raises(KeyError, lib.getpointer, 'xxxxxxxxxxxxxxx', [], ffi_type_void) + del ptr del lib + assert len(ALLOCATED) == 1 def test_library_func_call(self): lib = self.get_libc() ptr = lib.getpointer('rand', [], ffi_type_sint) zeroes = 0 + first = ptr.call(rffi.INT) for i in range(100): res = ptr.call(rffi.INT) - print res - if not res: + if res == first: zeroes += 1 - assert not zeroes + assert zeroes < 90 # not very hard check, but something :] + del ptr + del lib + assert len(ALLOCATED) == 1 # ffi_type_sint get allocated def test_call_args(self): libm = CDLL('libm.so') @@ -63,6 +66,42 @@ pow.push_arg(3.0) res = pow.call(rffi.DOUBLE) assert res == 27.0 + del pow + del libm + assert len(ALLOCATED) == 1 + + def test_wrong_args(self): + libc = CDLL('libc.so.6') + # XXX assume time_t is long + ctime = libc.getpointer('time', [ffi_type_pointer], ffi_type_ulong) + x = lltype.malloc(lltype.GcStruct('xxx')) + y = lltype.malloc(lltype.GcArray(rffi.LONG), 3) + z = lltype.malloc(lltype.Array(rffi.LONG), 4, flavor='raw') + py.test.raises(ValueError, "ctime.push_arg(x)") + py.test.raises(ValueError, "ctime.push_arg(y)") + py.test.raises(ValueError, "ctime.push_arg(z)") + del ctime + del libc + lltype.free(z, flavor='raw') + # allocation check makes no sense, since we've got GcStructs around + + def test_call_time(self): + libc = CDLL('libc.so.6') + # XXX assume time_t is long + ctime = libc.getpointer('time', [ffi_type_pointer], ffi_type_ulong) + ctime.push_arg(lltype.nullptr(rffi.CArray(rffi.LONG))) + t0 = ctime.call(rffi.LONG) + time.sleep(2) + ctime.push_arg(lltype.nullptr(rffi.CArray(rffi.LONG))) + t1 = ctime.call(rffi.LONG) + assert t1 > t0 + l_t = lltype.malloc(rffi.CArray(rffi.LONG), 1, flavor='raw') + ctime.push_arg(l_t) + t1 = ctime.call(rffi.LONG) + assert l_t[0] == t1 + lltype.free(l_t, flavor='raw') + del ctime + assert len(ALLOCATED) == 1 def test_compile(self): # XXX cannot run it on top of llinterp, some problems From cfbolz at codespeak.net Mon Oct 15 21:39:06 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 15 Oct 2007 21:39:06 +0200 (CEST) Subject: [pypy-svn] r47477 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071015193906.6FE4980E6@code0.codespeak.net> Author: cfbolz Date: Mon Oct 15 21:39:03 2007 New Revision: 47477 Added: pypy/dist/pypy/rpython/memory/gctransform/log.py Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: write out some statistics about number of typeids, roots and push/pops Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Mon Oct 15 21:39:03 2007 @@ -14,6 +14,7 @@ from pypy.rpython.rbuiltin import gen_cast from pypy.rpython.memory.gctypelayout import ll_weakref_deref, WEAKREF from pypy.rpython.memory.gctypelayout import convert_weakref_to, WEAKREFPTR +from pypy.rpython.memory.gctransform.log import log from pypy.tool.sourcetools import func_with_new_name import sys @@ -119,6 +120,7 @@ StackRootIterator = self.build_stack_root_iterator() gcdata.gc = GCClass(AddressLinkedList, get_roots=StackRootIterator, **GC_PARAMS) + self.num_pushs = 0 def frameworkgc_setup(): # run-time initialization code @@ -364,6 +366,9 @@ def finish_tables(self): table = self.layoutbuilder.flatten_table() + log.info("assigned %s typeids" % (len(table), )) + log.info("added %s push/pop stack root instructions" % ( + self.num_pushs, )) # replace the type_info_table pointer in gcdata -- at this point, # the database is in principle complete, so it has already seen @@ -383,6 +388,7 @@ #self.gcdata.type_info_table = table addresses_of_static_ptrs = self.layoutbuilder.addresses_of_static_ptrs + log.info("found %s static roots" % (len(addresses_of_static_ptrs), )) ll_static_roots_inside = lltype.malloc(lltype.Array(llmemory.Address), len(addresses_of_static_ptrs), immortal=True) @@ -565,6 +571,7 @@ if self.incr_stack_ptr is None: return livevars = [var for var in self.livevars if not var_ispyobj(var)] + self.num_pushs += len(livevars) if not livevars: return c_len = rmodel.inputconst(lltype.Signed, len(livevars) ) Added: pypy/dist/pypy/rpython/memory/gctransform/log.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/memory/gctransform/log.py Mon Oct 15 21:39:03 2007 @@ -0,0 +1,4 @@ +import py +from pypy.tool.ansi_print import ansi_log +log = py.log.Producer("gctransform") +py.log.setconsumer("gctransform", ansi_log) From niko at codespeak.net Mon Oct 15 23:08:25 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Mon, 15 Oct 2007 23:08:25 +0200 (CEST) Subject: [pypy-svn] r47478 - pypy/dist/pypy/translator/jvm Message-ID: <20071015210825.521B080F2@code0.codespeak.net> Author: niko Date: Mon Oct 15 23:08:24 2007 New Revision: 47478 Modified: pypy/dist/pypy/translator/jvm/generator.py Log: fix stupid typo: s/int(val)/int(value)/ Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Mon Oct 15 23:08:24 2007 @@ -1081,7 +1081,7 @@ elif TYPE is ootype.Unsigned: # Converts the unsigned int into its corresponding signed value: if value > 0x7FFFFFFF: - value = -((int(val) ^ 0xFFFFFFFF)+1) + value = -((int(value) ^ 0xFFFFFFFF)+1) self.emit(ICONST, value) elif TYPE is ootype.Char or TYPE is ootype.UniChar: self.emit(ICONST, ord(value)) From cfbolz at codespeak.net Mon Oct 15 23:15:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 15 Oct 2007 23:15:27 +0200 (CEST) Subject: [pypy-svn] r47479 - pypy/dist/pypy/translator/backendopt Message-ID: <20071015211527.2F38880EC@code0.codespeak.net> Author: cfbolz Date: Mon Oct 15 23:15:26 2007 New Revision: 47479 Modified: pypy/dist/pypy/translator/backendopt/inline.py Log: don't insert keepalives any more when inlining. it's no longer necessary and produces more gc pointers around, which is bad for the framework GCs. Modified: pypy/dist/pypy/translator/backendopt/inline.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/inline.py (original) +++ pypy/dist/pypy/translator/backendopt/inline.py Mon Oct 15 23:15:26 2007 @@ -14,6 +14,7 @@ from pypy.tool.algo import sparsemat from pypy.translator.backendopt import removenoops from pypy.translator.backendopt.support import log, split_block_with_keepalive +from pypy.translator.unsimplify import split_block from pypy.translator.backendopt.support import find_backedges, find_loop_blocks from pypy.translator.backendopt.canraise import RaiseAnalyzer @@ -440,7 +441,7 @@ copiedexceptblock.operations += self.generate_keepalive(linkargs) def do_inline(self, block, index_operation): - splitlink = split_block_with_keepalive(block, index_operation) + splitlink = split_block(None, block, index_operation) afterblock = splitlink.target # these variables have to be passed along all the links in the inlined # graph because the original function needs them in the blocks after From cfbolz at codespeak.net Mon Oct 15 23:20:57 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 15 Oct 2007 23:20:57 +0200 (CEST) Subject: [pypy-svn] r47480 - pypy/dist/pypy/translator Message-ID: <20071015212057.1FC7B80EC@code0.codespeak.net> Author: cfbolz Date: Mon Oct 15 23:20:56 2007 New Revision: 47480 Modified: pypy/dist/pypy/translator/exceptiontransform.py Log: also don't insert keepalives in the exception transformer Modified: pypy/dist/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/dist/pypy/translator/exceptiontransform.py (original) +++ pypy/dist/pypy/translator/exceptiontransform.py Mon Oct 15 23:20:56 2007 @@ -1,6 +1,6 @@ from pypy.translator.simplify import join_blocks, cleanup_graph from pypy.translator.unsimplify import copyvar, varoftype -from pypy.translator.unsimplify import insert_empty_block +from pypy.translator.unsimplify import insert_empty_block, split_block from pypy.translator.backendopt import canraise, inline, support, removenoops from pypy.objspace.flow.model import Block, Constant, Variable, Link, \ c_last_exception, SpaceOperation, checkgraph, FunctionGraph @@ -179,7 +179,7 @@ if not self.raise_analyzer.can_raise(op): continue - splitlink = support.split_block_with_keepalive(block, i+1, False) + splitlink = split_block(None, block, i+1) afterblock = splitlink.target if lastblock is block: lastblock = afterblock From fijal at codespeak.net Tue Oct 16 09:38:42 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Oct 2007 09:38:42 +0200 (CEST) Subject: [pypy-svn] r47484 - pypy/dist/pypy/module/_ffi Message-ID: <20071016073842.6B50780DE@code0.codespeak.net> Author: fijal Date: Tue Oct 16 09:38:41 2007 New Revision: 47484 Added: pypy/dist/pypy/module/_ffi/TODO Log: a todo for _ffi module Added: pypy/dist/pypy/module/_ffi/TODO ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/_ffi/TODO Tue Oct 16 09:38:41 2007 @@ -0,0 +1,11 @@ + +* long support with all messy corners (when to create long integer and + such) + +* add arrays + +* setattr on structures (finish) + +* keyword arguments for structure creation + +* make it all rpython From fijal at codespeak.net Tue Oct 16 09:41:28 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Oct 2007 09:41:28 +0200 (CEST) Subject: [pypy-svn] r47485 - in pypy/dist/pypy/module/_ffi: . test Message-ID: <20071016074128.9829780E0@code0.codespeak.net> Author: fijal Date: Tue Oct 16 09:41:28 2007 New Revision: 47485 Modified: pypy/dist/pypy/module/_ffi/app_ffi.py pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/structure.py pypy/dist/pypy/module/_ffi/test/test__ffi.py pypy/dist/pypy/module/_ffi/test/test_struct.py Log: * 'P' for pointer, 'p' for pascal string (not supported) * add a dummy __setattr__ (working only for int) on structure * add a test and few fixes for returning structure pointer Modified: pypy/dist/pypy/module/_ffi/app_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/app_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/app_ffi.py Tue Oct 16 09:41:28 2007 @@ -4,6 +4,12 @@ def __init__(self, fields): self.fields = fields - def __call__(self, **kwds): + def __call__(self, *args, **kwds): from _ffi import StructureInstance - return StructureInstance(self, kwds) + if args: + if len(args) > 1: + raise TypeError("Can give at most one non-keyword argument") + if kwds: + raise TypeError("Keyword arguments not allowed when passing address argument") + return StructureInstance(self, args[0], None) + return StructureInstance(self, None, kwds) Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Tue Oct 16 09:41:28 2007 @@ -26,7 +26,6 @@ 'f' : ffi_type_float, 'd' : ffi_type_double, 's' : ffi_type_pointer, - 'p' : ffi_type_pointer, 'P' : ffi_type_pointer, } @@ -45,7 +44,6 @@ 'f' : rffi.FLOAT, 'd' : rffi.DOUBLE, 's' : rffi.CCHARP, - 'p' : rffi.CCHARP, 'P' : rffi.VOIDP, } @@ -132,7 +130,7 @@ ll_str = rffi.str2charp(space.str_w(w_arg)) ptr.push_arg(ll_str) to_free.append(ll_str) - elif argtype == "p": + elif argtype == "P": # check for NULL ptr if space.is_w(w_arg, space.w_None): ptr.push_arg(lltype.nullptr(rffi.VOIDP.TO)) @@ -158,11 +156,11 @@ def wrap_result(space, restype, func): for c, ll_type in ll_typemap_iter: if restype == c: - if c == 's' or c == 'p': + if c == 's': return space.wrap(rffi.charp2str(func(rffi.CCHARP))) elif c == 'P': res = func(rffi.VOIDP) - return space.wrap(lltype.cast_ptr_to_int(res)) + return space.wrap(rffi.cast(rffi.INT, res)) elif c == 'q' or c == 'Q' or c == 'L': return space.newlong(func(ll_type)) else: Modified: pypy/dist/pypy/module/_ffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_ffi/structure.py (original) +++ pypy/dist/pypy/module/_ffi/structure.py Tue Oct 16 09:41:28 2007 @@ -39,7 +39,7 @@ return size, pos class W_StructureInstance(Wrappable): - def __init__(self, space, w_shape, w_fieldinits): + def __init__(self, space, w_shape, w_address, w_fieldinits): if space.is_true(w_fieldinits): raise OperationError(space.w_ValueError, space.wrap( "Fields should be not initialized with values by now")) @@ -47,8 +47,13 @@ fields = unpack_fields(space, w_fields) size, pos = size_and_pos(fields) self.fields = fields - self.ll_buffer = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', - zero=True) + if space.is_true(w_address): + self.free_afterwards = False + self.ll_buffer = rffi.cast(rffi.VOIDP, space.int_w(w_address)) + else: + self.free_afterwards = True + self.ll_buffer = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', + zero=True) self.ll_positions = pos self.next_pos = 0 @@ -60,6 +65,8 @@ cast_pos._annspecialcase_ = 'specialize:arg(1)' def getattr(self, space, attr): + if attr.startswith('tm'): + pass for i in range(len(self.fields)): name, c = self.fields[i] if name == attr: @@ -70,14 +77,27 @@ "C Structure has no attribute %s" % name)) getattr.unwrap_spec = ['self', ObjSpace, str] + def setattr(self, space, attr, value): + # XXX value is now always int, needs fixing + for i in range(len(self.fields)): + name, c = self.fields[i] + if name == attr: + pos = rffi.ptradd(self.ll_buffer, self.ll_positions[i]) + TP = rffi.CArrayPtr(rffi.INT) + rffi.cast(TP, pos)[0] = value + return + setattr.unwrap_spec = ['self', ObjSpace, str, int] + def __del__(self): - lltype.free(self.ll_buffer, flavor='raw') + if self.free_afterwards: + lltype.free(self.ll_buffer, flavor='raw') -def descr_new_structure_instance(space, w_type, w_shape, w_fieldinits): - return W_StructureInstance(space, w_shape, w_fieldinits) +def descr_new_structure_instance(space, w_type, w_shape, w_adr, w_fieldinits): + return W_StructureInstance(space, w_shape, w_adr, w_fieldinits) W_StructureInstance.typedef = TypeDef( 'StructureInstance', __new__ = interp2app(descr_new_structure_instance), __getattr__ = interp2app(W_StructureInstance.getattr), + __setattr__ = interp2app(W_StructureInstance.setattr), ) Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Tue Oct 16 09:41:28 2007 @@ -9,8 +9,20 @@ py.test.skip("Linux only tests by now") class AppTestCTypes: + def prepare_c_example(): + from pypy.tool.udir import udir + udir.join("xlib.c").write(py.code.Source(""" + typedef struct x { + char x1; + long x2; + struct x *x3; + } + """)) + prepare_c_example = staticmethod(prepare_c_example) + def setup_class(cls): cls.space = gettestobjspace(usemodules=('_ffi','struct')) + cls.prepare_c_example() def test_libload(self): import _ffi @@ -52,7 +64,7 @@ strlen = libc.ptr('strlen', ['s'], 'i') assert strlen("dupa") == 4 assert strlen("zupa") == 4 - strlen = libc.ptr('strlen', ['p'], 'i') + strlen = libc.ptr('strlen', ['P'], 'i') assert strlen("ddd\x00") == 3 strdup = libc.ptr('strdup', ['s'], 's') assert strdup("xxx") == "xxx" @@ -60,7 +72,7 @@ def test_time(self): import _ffi libc = _ffi.CDLL('libc.so.6') - time = libc.ptr('time', ['p'], 'l') + time = libc.ptr('time', ['P'], 'l') assert time(None) != 0 def test_gettimeofday(self): @@ -68,7 +80,7 @@ struct_type = _ffi.Structure([('tv_sec', 'l'), ('tv_usec', 'l')]) structure = struct_type() libc = _ffi.CDLL('libc.so.6') - gettimeofday = libc.ptr('gettimeofday', ['p', 'p'], 'i') + gettimeofday = libc.ptr('gettimeofday', ['P', 'P'], 'i') assert gettimeofday(structure, None) == 0 struct2 = struct_type() assert gettimeofday(struct2, None) == 0 @@ -76,4 +88,27 @@ assert (structure.tv_sec == struct2.tv_sec) or (structure.tv_sec == struct2.tv_sec - 1) raises(AttributeError, "structure.xxx") - + def test_structreturn(self): + import _ffi + X = _ffi.Structure([('x', 'l')]) + x = X() + x.x = 121 + Tm = _ffi.Structure([('tm_sec', 'i'), + ('tm_min', 'i'), + ('tm_hour', 'i'), + ("tm_mday", 'i'), + ("tm_mon", 'i'), + ("tm_year", 'i'), + ("tm_wday", 'i'), + ("tm_yday", 'i'), + ("tm_isdst", 'i')]) + libc = _ffi.CDLL('libc.so.6') + gmtime = libc.ptr('gmtime', ['P'], 'P') + t = Tm(gmtime(x)) + assert t.tm_year == 70 + assert t.tm_sec == 1 + assert t.tm_min == 2 + + + #def test_nested_structures(self): + # Modified: pypy/dist/pypy/module/_ffi/test/test_struct.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test_struct.py (original) +++ pypy/dist/pypy/module/_ffi/test/test_struct.py Tue Oct 16 09:41:28 2007 @@ -1,5 +1,7 @@ -from pypy.module._ffi.structure import sizeof +from pypy.module._ffi.structure import size_and_pos + +sizeof = lambda x : size_and_pos(x)[0] def unpack(desc): return [('x', i) for i in desc] From fijal at codespeak.net Tue Oct 16 11:20:23 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Oct 2007 11:20:23 +0200 (CEST) Subject: [pypy-svn] r47486 - pypy/dist/pypy/module/_ffi Message-ID: <20071016092023.9530180E4@code0.codespeak.net> Author: fijal Date: Tue Oct 16 11:20:22 2007 New Revision: 47486 Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/structure.py Log: Make this thing RPython. Also fix setattr. Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Tue Oct 16 11:20:22 2007 @@ -112,44 +112,48 @@ """ ) -def pack_pointer(space, w_arg, ptr): +def pack_pointer(space, argdesc, w_arg, push_func): arg = space.str_w(w_arg) ll_str = lltype.malloc(rffi.CCHARP.TO, len(arg), flavor='raw') for i in range(len(arg)): ll_str[i] = arg[i] - ptr.push_arg(ll_str) + push_func(argdesc, ll_str) return ll_str -def push_arg(space, ptr, argnum, argtype, w_arg, to_free): +def unwrap_arg(space, push_func, argdesc, argtype, w_arg, to_free): w = space.wrap # XXX how to handle LONGLONG here? # they're probably long, so we'll not get them through int_w if argtype == "d" or argtype == "f": - ptr.push_arg(space.float_w(w_arg)) + push_func(argdesc, space.float_w(w_arg)) elif argtype == "s": ll_str = rffi.str2charp(space.str_w(w_arg)) - ptr.push_arg(ll_str) - to_free.append(ll_str) + if to_free is not None: + to_free.append(ll_str) + push_func(argdesc, ll_str) elif argtype == "P": # check for NULL ptr if space.is_w(w_arg, space.w_None): - ptr.push_arg(lltype.nullptr(rffi.VOIDP.TO)) + push_func(argdesc, lltype.nullptr(rffi.VOIDP.TO)) elif space.is_true(space.isinstance(w_arg, space.w_basestring)): - to_free.append(pack_pointer(space, w_arg, ptr)) + if to_free is not None: + to_free.append(pack_pointer(space, argdesc, w_arg, push_func)) else: mod = space.getbuiltinmodule('_ffi') w_StructureInstance = space.getattr(mod, w('StructureInstance')) if space.is_true(space.isinstance(w_arg, w_StructureInstance)): #ptr.push_arg(lltype.cast_int_to_ptr(rffi.VOIDP, space.int_w(space.getattr(w_arg, w('buffer'))))) - ptr.push_arg(w_arg.ll_buffer) + push_func(argdesc, w_arg.ll_buffer) else: raise OperationError(space.w_TypeError, w( "Expected structure, array or simple type")) elif argtype == "c" or argtype == "b" or argtype == "B": - ptr.push_arg(space.str_w(w_arg)) + push_func(argdesc, space.str_w(w_arg)) else: - assert argtype in ["iIhHlLqQ"] - ptr.push_arg(space.int_w(w_arg)) + assert argtype in "iIhHlLqQ" + push_func(argdesc, space.int_w(w_arg)) +unwrap_arg._annspecialcase_ = 'specialize:arg(1)' +# we should have also here specialize:argtype(5) :-/ ll_typemap_iter = unrolling_iterable(LL_TYPEMAP.items()) @@ -174,6 +178,9 @@ self.restype = restype self.argtypes = argtypes + def push(self, argdesc, value): + self.ptr.push_arg(value) + def call(self, space, arguments): args_w, kwds_w = arguments.unpack() # C has no keyword arguments @@ -183,7 +190,7 @@ to_free = [] i = 0 for argtype, w_arg in zip(self.argtypes, args_w): - push_arg(space, self.ptr, i, argtype, w_arg, to_free) + unwrap_arg(space, self.push, i, argtype, w_arg, to_free) i += 1 try: return wrap_result(space, self.restype, self.ptr.call) Modified: pypy/dist/pypy/module/_ffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_ffi/structure.py (original) +++ pypy/dist/pypy/module/_ffi/structure.py Tue Oct 16 11:20:22 2007 @@ -12,8 +12,13 @@ # XXX we've got the very same info in two places - one is native_fmttable # the other one is in rlib/libffi, we should refactor it to reuse the same # logic, I'll not touch it by now, and refactor it later -from pypy.module.struct.nativefmttable import native_fmttable -from pypy.module._ffi.interp_ffi import wrap_result +from pypy.module.struct.nativefmttable import native_fmttable as struct_native_fmttable +from pypy.module._ffi.interp_ffi import wrap_result, unwrap_arg + +native_fmttable = {} +for key, value in struct_native_fmttable.items(): + native_fmttable[key] = {'size': value['size'], + 'alignment': value.get('alignment', value['size'])} def unpack_fields(space, w_fields): fields_w = space.unpackiterable(w_fields) @@ -77,16 +82,20 @@ "C Structure has no attribute %s" % name)) getattr.unwrap_spec = ['self', ObjSpace, str] - def setattr(self, space, attr, value): - # XXX value is now always int, needs fixing + def push_field(self, num, value): + ptr = rffi.ptradd(self.ll_buffer, self.ll_positions[num]) + TP = lltype.typeOf(value) + T = rffi.CArrayPtr(TP) + rffi.cast(T, ptr)[0] = value + push_field._annspecialcase_ = 'specialize:argtype(2)' + + def setattr(self, space, attr, w_value): for i in range(len(self.fields)): name, c = self.fields[i] if name == attr: - pos = rffi.ptradd(self.ll_buffer, self.ll_positions[i]) - TP = rffi.CArrayPtr(rffi.INT) - rffi.cast(TP, pos)[0] = value + unwrap_arg(space, self.push_field, i, c, w_value, None) return - setattr.unwrap_spec = ['self', ObjSpace, str, int] + setattr.unwrap_spec = ['self', ObjSpace, str, W_Root] def __del__(self): if self.free_afterwards: From cfbolz at codespeak.net Tue Oct 16 11:21:17 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 16 Oct 2007 11:21:17 +0200 (CEST) Subject: [pypy-svn] r47487 - pypy/dist/pypy/translator/c/test Message-ID: <20071016092117.8B35080EE@code0.codespeak.net> Author: cfbolz Date: Tue Oct 16 11:21:17 2007 New Revision: 47487 Modified: pypy/dist/pypy/translator/c/test/test_newgc.py Log: huh? fix this test which was waiting to fail: the keepalive list needs to be global, or it can be collected very early itself. Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Tue Oct 16 11:21:17 2007 @@ -576,9 +576,9 @@ class A: pass + keepalive = [] def fn(): n = 7000 - keepalive = [] weakrefs = [] a = None for i in range(n): From fijal at codespeak.net Tue Oct 16 11:28:17 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Oct 2007 11:28:17 +0200 (CEST) Subject: [pypy-svn] r47488 - pypy/dist/pypy/module/_ffi Message-ID: <20071016092817.6470680DC@code0.codespeak.net> Author: fijal Date: Tue Oct 16 11:28:17 2007 New Revision: 47488 Modified: pypy/dist/pypy/module/_ffi/TODO Log: Implemented Modified: pypy/dist/pypy/module/_ffi/TODO ============================================================================== --- pypy/dist/pypy/module/_ffi/TODO (original) +++ pypy/dist/pypy/module/_ffi/TODO Tue Oct 16 11:28:17 2007 @@ -4,8 +4,5 @@ * add arrays -* setattr on structures (finish) - * keyword arguments for structure creation -* make it all rpython From antocuni at codespeak.net Tue Oct 16 11:32:20 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 16 Oct 2007 11:32:20 +0200 (CEST) Subject: [pypy-svn] r47489 - pypy/dist/pypy/bin Message-ID: <20071016093220.E1A3080DC@code0.codespeak.net> Author: antocuni Date: Tue Oct 16 11:32:20 2007 New Revision: 47489 Modified: pypy/dist/pypy/bin/checkmodule.py Log: add support for -h and --help options Modified: pypy/dist/pypy/bin/checkmodule.py ============================================================================== --- pypy/dist/pypy/bin/checkmodule.py (original) +++ pypy/dist/pypy/bin/checkmodule.py Tue Oct 16 11:32:20 2007 @@ -24,6 +24,9 @@ if len(argv) == 2: backend = 'cli' modname = argv[1] + if modname in ('-h', '--help'): + print >> sys.stderr, __doc__ + sys.exit(0) else: _, b, backend, modname = argv assert b == '-b' From fijal at codespeak.net Tue Oct 16 11:34:03 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Oct 2007 11:34:03 +0200 (CEST) Subject: [pypy-svn] r47490 - pypy/dist/pypy/bin Message-ID: <20071016093403.4488E80E4@code0.codespeak.net> Author: fijal Date: Tue Oct 16 11:34:02 2007 New Revision: 47490 Modified: pypy/dist/pypy/bin/checkmodule.py Log: Support for -unknown option Modified: pypy/dist/pypy/bin/checkmodule.py ============================================================================== --- pypy/dist/pypy/bin/checkmodule.py (original) +++ pypy/dist/pypy/bin/checkmodule.py Tue Oct 16 11:34:02 2007 @@ -27,6 +27,10 @@ if modname in ('-h', '--help'): print >> sys.stderr, __doc__ sys.exit(0) + if modname.startswith('-'): + print >> sys.stderr, "Bad command line" + print >> sys.stderr, __doc__ + sys.exit(1) else: _, b, backend, modname = argv assert b == '-b' From fijal at codespeak.net Tue Oct 16 11:35:08 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Oct 2007 11:35:08 +0200 (CEST) Subject: [pypy-svn] r47491 - pypy/dist/pypy/module/_ffi Message-ID: <20071016093508.9D3E380DC@code0.codespeak.net> Author: fijal Date: Tue Oct 16 11:35:08 2007 New Revision: 47491 Modified: pypy/dist/pypy/module/_ffi/structure.py Log: Fix rpythonism Modified: pypy/dist/pypy/module/_ffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_ffi/structure.py (original) +++ pypy/dist/pypy/module/_ffi/structure.py Tue Oct 16 11:35:08 2007 @@ -79,7 +79,7 @@ self.next_pos = i return wrap_result(space, c, self.cast_pos) raise OperationError(space.w_AttributeError, space.wrap( - "C Structure has no attribute %s" % name)) + "C Structure has no attribute %s" % attr)) getattr.unwrap_spec = ['self', ObjSpace, str] def push_field(self, num, value): From fijal at codespeak.net Tue Oct 16 12:15:31 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Oct 2007 12:15:31 +0200 (CEST) Subject: [pypy-svn] r47492 - pypy/dist/pypy/doc/config Message-ID: <20071016101531.C63D980EF@code0.codespeak.net> Author: fijal Date: Tue Oct 16 12:15:30 2007 New Revision: 47492 Added: pypy/dist/pypy/doc/config/objspace.usemodules._ffi.txt (contents, props changed) Log: Document modules/_ffi Added: pypy/dist/pypy/doc/config/objspace.usemodules._ffi.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/config/objspace.usemodules._ffi.txt Tue Oct 16 12:15:30 2007 @@ -0,0 +1,3 @@ +An experimental module providing low-level interface to +C-level libraries, for use when implementing ctypes, not +indended for a direct use. \ No newline at end of file From cfbolz at codespeak.net Tue Oct 16 12:21:11 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 16 Oct 2007 12:21:11 +0200 (CEST) Subject: [pypy-svn] r47493 - pypy/dist/pypy/translator/backendopt/test Message-ID: <20071016102111.D716980F6@code0.codespeak.net> Author: cfbolz Date: Tue Oct 16 12:21:11 2007 New Revision: 47493 Modified: pypy/dist/pypy/translator/backendopt/test/test_escape.py pypy/dist/pypy/translator/backendopt/test/test_malloc.py Log: skip some tests that fail due to inability of dealing with the new substructs handling. unclear whether we want to support that. Modified: pypy/dist/pypy/translator/backendopt/test/test_escape.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_escape.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_escape.py Tue Oct 16 12:21:11 2007 @@ -401,6 +401,7 @@ assert state.does_change() def test_extfunc_resultonheap(): + py.test.skip("not a valid test anymore") import os def f(i): s = str(i) Modified: pypy/dist/pypy/translator/backendopt/test/test_malloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_malloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_malloc.py Tue Oct 16 12:21:11 2007 @@ -194,6 +194,7 @@ self.check(fn7, [int], [10], 55, must_be_removed=False) def test_getsubstruct(self): + py.test.skip("fails because of the interior structure changes") SMALL = lltype.Struct('SMALL', ('x', lltype.Signed)) BIG = lltype.GcStruct('BIG', ('z', lltype.Signed), ('s', SMALL)) @@ -206,6 +207,7 @@ self.check(fn, [int, int], [100, 58], 42) def test_fixedsizearray(self): + py.test.skip("fails because of the interior structure changes") A = lltype.FixedSizeArray(lltype.Signed, 3) S = lltype.GcStruct('S', ('a', A)) @@ -259,6 +261,7 @@ self.check(fn, [], [], 42) def test_getarraysubstruct(self): + py.test.skip("fails because of the interior structure changes") U = lltype.Struct('U', ('n', lltype.Signed)) for length in [1, 2]: S = lltype.GcStruct('S', ('a', lltype.FixedSizeArray(U, length))) @@ -288,6 +291,7 @@ self.check(fn, [], [], 12) def test_union(self): + py.test.skip("fails because of the interior structure changes") UNION = lltype.Struct('UNION', ('a', lltype.Signed), ('b', lltype.Signed), hints = {'union': True}) BIG = lltype.GcStruct('BIG', ('u1', UNION), ('u2', UNION)) From fijal at codespeak.net Tue Oct 16 12:23:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Oct 2007 12:23:04 +0200 (CEST) Subject: [pypy-svn] r47494 - pypy/dist/pypy/module/_ffi Message-ID: <20071016102304.772CE80F5@code0.codespeak.net> Author: fijal Date: Tue Oct 16 12:23:04 2007 New Revision: 47494 Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/structure.py Log: Make it less-readable-more-rpythonic Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Tue Oct 16 12:23:04 2007 @@ -112,46 +112,46 @@ """ ) -def pack_pointer(space, argdesc, w_arg, push_func): +def pack_pointer(space, add_arg, argdesc, w_arg, push_func): arg = space.str_w(w_arg) ll_str = lltype.malloc(rffi.CCHARP.TO, len(arg), flavor='raw') for i in range(len(arg)): ll_str[i] = arg[i] - push_func(argdesc, ll_str) + push_func(add_arg, argdesc, ll_str) return ll_str -def unwrap_arg(space, push_func, argdesc, argtype, w_arg, to_free): +def unwrap_arg(space, push_func, add_arg, argdesc, argtype, w_arg, to_free): w = space.wrap # XXX how to handle LONGLONG here? # they're probably long, so we'll not get them through int_w if argtype == "d" or argtype == "f": - push_func(argdesc, space.float_w(w_arg)) + push_func(add_arg, argdesc, space.float_w(w_arg)) elif argtype == "s": ll_str = rffi.str2charp(space.str_w(w_arg)) if to_free is not None: to_free.append(ll_str) - push_func(argdesc, ll_str) + push_func(add_arg, argdesc, ll_str) elif argtype == "P": # check for NULL ptr if space.is_w(w_arg, space.w_None): - push_func(argdesc, lltype.nullptr(rffi.VOIDP.TO)) + push_func(add_arg, argdesc, lltype.nullptr(rffi.VOIDP.TO)) elif space.is_true(space.isinstance(w_arg, space.w_basestring)): if to_free is not None: - to_free.append(pack_pointer(space, argdesc, w_arg, push_func)) + to_free.append(pack_pointer(space, add_arg, argdesc, w_arg, push_func)) else: mod = space.getbuiltinmodule('_ffi') w_StructureInstance = space.getattr(mod, w('StructureInstance')) if space.is_true(space.isinstance(w_arg, w_StructureInstance)): #ptr.push_arg(lltype.cast_int_to_ptr(rffi.VOIDP, space.int_w(space.getattr(w_arg, w('buffer'))))) - push_func(argdesc, w_arg.ll_buffer) + push_func(add_arg, argdesc, w_arg.ll_buffer) else: raise OperationError(space.w_TypeError, w( "Expected structure, array or simple type")) elif argtype == "c" or argtype == "b" or argtype == "B": - push_func(argdesc, space.str_w(w_arg)) + push_func(add_arg, argdesc, space.str_w(w_arg)) else: assert argtype in "iIhHlLqQ" - push_func(argdesc, space.int_w(w_arg)) + push_func(add_arg, argdesc, space.int_w(w_arg)) unwrap_arg._annspecialcase_ = 'specialize:arg(1)' # we should have also here specialize:argtype(5) :-/ @@ -178,8 +178,9 @@ self.restype = restype self.argtypes = argtypes - def push(self, argdesc, value): - self.ptr.push_arg(value) + def push(ptr, argdesc, value): + ptr.push_arg(value) + push = staticmethod(push) def call(self, space, arguments): args_w, kwds_w = arguments.unpack() @@ -190,7 +191,7 @@ to_free = [] i = 0 for argtype, w_arg in zip(self.argtypes, args_w): - unwrap_arg(space, self.push, i, argtype, w_arg, to_free) + unwrap_arg(space, self.push, self.ptr, i, argtype, w_arg, to_free) i += 1 try: return wrap_result(space, self.restype, self.ptr.call) Modified: pypy/dist/pypy/module/_ffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_ffi/structure.py (original) +++ pypy/dist/pypy/module/_ffi/structure.py Tue Oct 16 12:23:04 2007 @@ -43,6 +43,13 @@ size += field_desc['size'] return size, pos +def push_field(self, num, value): + ptr = rffi.ptradd(self.ll_buffer, self.ll_positions[num]) + TP = lltype.typeOf(value) + T = rffi.CArrayPtr(TP) + rffi.cast(T, ptr)[0] = value +push_field._annspecialcase_ = 'specialize:argtype(2)' + class W_StructureInstance(Wrappable): def __init__(self, space, w_shape, w_address, w_fieldinits): if space.is_true(w_fieldinits): @@ -82,18 +89,11 @@ "C Structure has no attribute %s" % attr)) getattr.unwrap_spec = ['self', ObjSpace, str] - def push_field(self, num, value): - ptr = rffi.ptradd(self.ll_buffer, self.ll_positions[num]) - TP = lltype.typeOf(value) - T = rffi.CArrayPtr(TP) - rffi.cast(T, ptr)[0] = value - push_field._annspecialcase_ = 'specialize:argtype(2)' - def setattr(self, space, attr, w_value): for i in range(len(self.fields)): name, c = self.fields[i] if name == attr: - unwrap_arg(space, self.push_field, i, c, w_value, None) + unwrap_arg(space, push_field, self, i, c, w_value, None) return setattr.unwrap_spec = ['self', ObjSpace, str, W_Root] From fijal at codespeak.net Tue Oct 16 12:33:31 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Oct 2007 12:33:31 +0200 (CEST) Subject: [pypy-svn] r47495 - pypy/dist/pypy/module/_ffi Message-ID: <20071016103331.44AB880DE@code0.codespeak.net> Author: fijal Date: Tue Oct 16 12:33:30 2007 New Revision: 47495 Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/structure.py Log: Mess++ RPythonism++ Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Tue Oct 16 12:33:30 2007 @@ -157,20 +157,24 @@ ll_typemap_iter = unrolling_iterable(LL_TYPEMAP.items()) -def wrap_result(space, restype, func): +def wrap_result(space, restype, arg, func): for c, ll_type in ll_typemap_iter: if restype == c: if c == 's': - return space.wrap(rffi.charp2str(func(rffi.CCHARP))) + return space.wrap(rffi.charp2str(func(arg, rffi.CCHARP))) elif c == 'P': - res = func(rffi.VOIDP) + res = func(arg, rffi.VOIDP) return space.wrap(rffi.cast(rffi.INT, res)) elif c == 'q' or c == 'Q' or c == 'L': - return space.newlong(func(ll_type)) + return space.newlong(func(arg, ll_type)) else: - return space.wrap(func(ll_type)) + return space.wrap(func(arg, ll_type)) return space.w_None -wrap_result._annspecialcase_ = 'specialize:arg(2)' +wrap_result._annspecialcase_ = 'specialize:arg(3)' + +def ptr_call(ptr, ll_type): + return ptr.call(ll_type) +ptr_call._annspecialcase_ = 'specialize:arg(1)' class W_FuncPtr(Wrappable): def __init__(self, ptr, argtypes, restype): @@ -194,7 +198,7 @@ unwrap_arg(space, self.push, self.ptr, i, argtype, w_arg, to_free) i += 1 try: - return wrap_result(space, self.restype, self.ptr.call) + return wrap_result(space, self.restype, self.ptr, ptr_call) finally: for elem in to_free: lltype.free(elem, flavor='raw') Modified: pypy/dist/pypy/module/_ffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_ffi/structure.py (original) +++ pypy/dist/pypy/module/_ffi/structure.py Tue Oct 16 12:33:30 2007 @@ -50,6 +50,13 @@ rffi.cast(T, ptr)[0] = value push_field._annspecialcase_ = 'specialize:argtype(2)' +def cast_pos(self, ll_t): + i = self.next_pos + pos = rffi.ptradd(self.ll_buffer, self.ll_positions[i]) + TP = rffi.CArrayPtr(ll_t) + return rffi.cast(TP, pos)[0] +cast_pos._annspecialcase_ = 'specialize:arg(1)' + class W_StructureInstance(Wrappable): def __init__(self, space, w_shape, w_address, w_fieldinits): if space.is_true(w_fieldinits): @@ -69,13 +76,6 @@ self.ll_positions = pos self.next_pos = 0 - def cast_pos(self, ll_t): - i = self.next_pos - pos = rffi.ptradd(self.ll_buffer, self.ll_positions[i]) - TP = rffi.CArrayPtr(ll_t) - return rffi.cast(TP, pos)[0] - cast_pos._annspecialcase_ = 'specialize:arg(1)' - def getattr(self, space, attr): if attr.startswith('tm'): pass @@ -84,7 +84,7 @@ if name == attr: # XXX RPython-trick for passing lambda around self.next_pos = i - return wrap_result(space, c, self.cast_pos) + return wrap_result(space, c, self, cast_pos) raise OperationError(space.w_AttributeError, space.wrap( "C Structure has no attribute %s" % attr)) getattr.unwrap_spec = ['self', ObjSpace, str] From fijal at codespeak.net Tue Oct 16 13:45:06 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Oct 2007 13:45:06 +0200 (CEST) Subject: [pypy-svn] r47496 - pypy/dist/pypy/translator/js/examples/bnb Message-ID: <20071016114506.BAF9B80E6@code0.codespeak.net> Author: fijal Date: Tue Oct 16 13:45:05 2007 New Revision: 47496 Modified: pypy/dist/pypy/translator/js/examples/bnb/bnb.py Log: Stupid fix. This file should be updated and tested or removed, but is so nice... :-( Modified: pypy/dist/pypy/translator/js/examples/bnb/bnb.py ============================================================================== --- pypy/dist/pypy/translator/js/examples/bnb/bnb.py (original) +++ pypy/dist/pypy/translator/js/examples/bnb/bnb.py Tue Oct 16 13:45:05 2007 @@ -75,10 +75,10 @@ if hasattr(self, '_port'): return self._port try: - port = re.findall('value=".*"', urllib.urlopen('http://%s:8000' % host).read())[0] + port = re.findall('value=".*"', urllib.urlopen('http://%s:8000' % self.host).read())[0] port = int(port[7:-1]) except IOError: - log("ERROR: Can't connect to BnB server on %s:8000" % host) + log("ERROR: Can't connect to BnB server on %s:8000" % self.host) raise IOError except IndexError: log("ERROR: Connected to BnB server but unable to detect a running game") From cfbolz at codespeak.net Tue Oct 16 14:50:34 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 16 Oct 2007 14:50:34 +0200 (CEST) Subject: [pypy-svn] r47497 - in pypy/dist/pypy/rpython/memory/gctransform: . test Message-ID: <20071016125034.0D76180D4@code0.codespeak.net> Author: cfbolz Date: Tue Oct 16 14:50:33 2007 New Revision: 47497 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py pypy/dist/pypy/rpython/memory/gctransform/transform.py Log: significantly reduce the number of pushing and popping the framework GCs do: variables are only saved across a call if the variable is needed afterwards. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Tue Oct 16 14:50:33 2007 @@ -416,9 +416,9 @@ def gct_direct_call(self, hop): if self.collect_analyzer.analyze(hop.spaceop): - self.push_roots(hop) + livevars = self.push_roots(hop) self.default(hop) - self.pop_roots(hop) + self.pop_roots(hop, livevars) else: self.default(hop) @@ -462,20 +462,20 @@ args = [self.c_const_gc, c_type_id, v_length, c_size, c_varitemsize, c_ofstolength, c_can_collect, c_has_finalizer] - self.push_roots(hop) + livevars = self.push_roots(hop) v_result = hop.genop("direct_call", [malloc_ptr] + args, resulttype=llmemory.GCREF) - self.pop_roots(hop) + self.pop_roots(hop, livevars) return v_result gct_fv_gc_malloc_varsize = gct_fv_gc_malloc def gct_gc__collect(self, hop): op = hop.spaceop - self.push_roots(hop) + livevars = self.push_roots(hop) hop.genop("direct_call", [self.collect_ptr, self.c_const_gc], resultvar=op.result) - self.pop_roots(hop) + self.pop_roots(hop, livevars) def gct_gc_x_swap_pool(self, hop): op = hop.spaceop @@ -502,11 +502,11 @@ def gct_gc_x_become(self, hop): op = hop.spaceop [v_target, v_source] = op.args - self.push_roots(hop) + livevars = self.push_roots(hop) hop.genop("direct_call", [self.x_become_ptr, self.c_const_gc, v_target, v_source], resultvar=op.result) - self.pop_roots(hop) + self.pop_roots(hop, livevars) def gct_zero_gc_pointers_inside(self, hop): v_ob = hop.spaceop.args[0] @@ -530,12 +530,12 @@ v_instance, = op.args v_addr = hop.genop("cast_ptr_to_adr", [v_instance], resulttype=llmemory.Address) - self.push_roots(hop) + livevars = self.push_roots(hop) v_result = hop.genop("direct_call", [malloc_ptr] + args, resulttype=llmemory.GCREF) v_result = hop.genop("cast_opaque_ptr", [v_result], resulttype=WEAKREFPTR) - self.pop_roots(hop) + self.pop_roots(hop, livevars) hop.genop("bare_setfield", [v_result, rmodel.inputconst(lltype.Void, "weakptr"), v_addr]) v_weakref = hop.genop("cast_ptr_to_weakrefptr", [v_result], @@ -551,13 +551,13 @@ def gct_gc_id(self, hop): if self.id_ptr is not None: - self.push_roots(hop) + livevars = self.push_roots(hop) [v_ptr] = hop.spaceop.args v_ptr = hop.genop("cast_opaque_ptr", [v_ptr], resulttype=llmemory.GCREF) hop.genop("direct_call", [self.id_ptr, self.c_const_gc, v_ptr], resultvar=hop.spaceop.result) - self.pop_roots(hop) + self.pop_roots(hop, livevars) else: hop.rename('cast_ptr_to_int') # works nicely for non-moving GCs @@ -570,10 +570,17 @@ def push_roots(self, hop): if self.incr_stack_ptr is None: return - livevars = [var for var in self.livevars if not var_ispyobj(var)] + if self.gcdata.gc.moving_gc: + # moving GCs don't borrow, so the caller does not need to keep + # the arguments alive + livevars = [var for var in self.livevars_after_op + if not var_ispyobj(var)] + else: + livevars = self.livevars_after_op + self.current_op_keeps_alive + livevars = [var for var in livevars if not var_ispyobj(var)] self.num_pushs += len(livevars) if not livevars: - return + return [] c_len = rmodel.inputconst(lltype.Signed, len(livevars) ) base_addr = hop.genop("direct_call", [self.incr_stack_ptr, c_len ], resulttype=llmemory.Address) @@ -588,11 +595,11 @@ resulttype=WEAKREFPTR) v_adr = gen_cast(hop.llops, llmemory.Address, var) hop.genop("raw_store", [base_addr, c_type, c_k, v_adr]) + return livevars - def pop_roots(self, hop): + def pop_roots(self, hop, livevars): if self.decr_stack_ptr is None: return - livevars = [var for var in self.livevars if not var_ispyobj(var)] if not livevars: return c_len = rmodel.inputconst(lltype.Signed, len(livevars) ) Modified: pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py Tue Oct 16 14:50:33 2007 @@ -79,7 +79,7 @@ [varoftype(PTR_TYPE), Constant('x', lltype.Void)], varoftype(lltype.Void)) transformer = WriteBarrierTransformer(t) - hop = GcHighLevelOp(transformer, spaceop, llops) + hop = GcHighLevelOp(transformer, spaceop, 0, llops) hop.dispatch() found = False for op in llops: Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Tue Oct 16 14:50:33 2007 @@ -33,10 +33,17 @@ PyObjPtr = lltype.Ptr(lltype.PyObject) class GcHighLevelOp(object): - def __init__(self, gctransformer, op, llops): - self.gctransformer = gctransformer + def __init__(self, gct, op, index, llops): + self.gctransformer = gct self.spaceop = op + self.index = index self.llops = llops + gct.livevars_after_op = [ + var for var in gct.livevars + if gct.var_last_needed_in[var] > self.index] + gct.current_op_keeps_alive = [ + var for var in self.spaceop.args + if gct.var_last_needed_in.get(var) == self.index] def dispatch(self): gct = self.gctransformer @@ -56,6 +63,8 @@ gct.push_alive(v_result) elif opname not in ('direct_call', 'indirect_call'): gct.push_alive(v_result) + + def rename(self, newopname): self.llops.append( @@ -152,8 +161,21 @@ #self.curr_block = block self.livevars = [var for var in block.inputargs if var_needsgc(var) and not is_borrowed(var)] - for op in block.operations: - hop = GcHighLevelOp(self, op, self.llops) + allvars = [var for var in block.getvariables() if var_needsgc(var)] + self.var_last_needed_in = dict.fromkeys(allvars, 0) + for i, op in enumerate(block.operations): + for var in op.args: + if not var_needsgc(var): + continue + self.var_last_needed_in[var] = i + for link in block.exits: + for var in link.args: + if not var_needsgc(var): + continue + self.var_last_needed_in[var] = len(block.operations) + 1 + + for i, op in enumerate(block.operations): + hop = GcHighLevelOp(self, op, i, self.llops) hop.dispatch() if len(block.exits) != 0: # i.e not the return block @@ -182,6 +204,7 @@ block.operations[:] = self.llops self.llops = None self.livevars = None + self.var_last_needed_in = None def transform_graph(self, graph): if graph in self.minimal_transform: From cfbolz at codespeak.net Tue Oct 16 15:38:59 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 16 Oct 2007 15:38:59 +0200 (CEST) Subject: [pypy-svn] r47498 - pypy/dist/pypy/rpython/test Message-ID: <20071016133859.22D8F80EB@code0.codespeak.net> Author: cfbolz Date: Tue Oct 16 15:38:57 2007 New Revision: 47498 Modified: pypy/dist/pypy/rpython/test/test_rclass.py Log: (passing) test to test the behaviour of _immutable_ in the presence of inheritance Modified: pypy/dist/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rclass.py (original) +++ pypy/dist/pypy/rpython/test/test_rclass.py Tue Oct 16 15:38:57 2007 @@ -667,6 +667,24 @@ t, typer, graph = self.gengraph(f, [], backendopt=True) assert summary(graph) == {} + def test_immutable_inheritance(self): + class I(object): + def __init__(self, v): + self.v = v + + class J(I): + def __init__(self, v, w): + self.w = w + I.__init__(self, v) + + j = J(3, 4) + def f(): + return j.v + j.w + + t, typer, graph = self.gengraph(f, [], backendopt=True) + assert summary(graph) == {} + + def test_instance_repr(self): class FooBar(object): pass From cfbolz at codespeak.net Tue Oct 16 15:53:10 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 16 Oct 2007 15:53:10 +0200 (CEST) Subject: [pypy-svn] r47499 - pypy/dist/pypy/rpython/test Message-ID: <20071016135310.A50C280ED@code0.codespeak.net> Author: cfbolz Date: Tue Oct 16 15:53:08 2007 New Revision: 47499 Modified: pypy/dist/pypy/rpython/test/test_llinterp.py Log: add a view graph Modified: pypy/dist/pypy/rpython/test/test_llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_llinterp.py (original) +++ pypy/dist/pypy/rpython/test/test_llinterp.py Tue Oct 16 15:53:08 2007 @@ -55,6 +55,8 @@ from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(t) timelog("checking graphs", t.checkgraphs) + if viewbefore: + t.view() desc = t.annotator.bookkeeper.getdesc(func) graph = desc.specialize(argtypes) return t, typer, graph From cfbolz at codespeak.net Tue Oct 16 16:16:08 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 16 Oct 2007 16:16:08 +0200 (CEST) Subject: [pypy-svn] r47500 - pypy/dist/pypy/rpython/test Message-ID: <20071016141608.5DF0E80F2@code0.codespeak.net> Author: cfbolz Date: Tue Oct 16 16:16:06 2007 New Revision: 47500 Modified: pypy/dist/pypy/rpython/test/test_rclass.py Log: make test actually test what was intended Modified: pypy/dist/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rclass.py (original) +++ pypy/dist/pypy/rpython/test/test_rclass.py Tue Oct 16 16:16:06 2007 @@ -673,18 +673,20 @@ self.v = v class J(I): + _immutable_ = True def __init__(self, v, w): self.w = w I.__init__(self, v) j = J(3, 4) def f(): + j.v = j.v * 1 # make the annotator think it is mutated + j.w = j.w * 1 # make the annotator think it is mutated return j.v + j.w t, typer, graph = self.gengraph(f, [], backendopt=True) - assert summary(graph) == {} + assert summary(graph) == {"setfield": 2} - def test_instance_repr(self): class FooBar(object): pass From cfbolz at codespeak.net Tue Oct 16 16:18:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 16 Oct 2007 16:18:47 +0200 (CEST) Subject: [pypy-svn] r47501 - pypy/dist/pypy/objspace/std Message-ID: <20071016141847.47D4B80F2@code0.codespeak.net> Author: cfbolz Date: Tue Oct 16 16:18:46 2007 New Revision: 47501 Modified: pypy/dist/pypy/objspace/std/dictobject.py pypy/dist/pypy/objspace/std/stringobject.py Log: drop two immutable-hints Modified: pypy/dist/pypy/objspace/std/dictobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictobject.py (original) +++ pypy/dist/pypy/objspace/std/dictobject.py Tue Oct 16 16:18:46 2007 @@ -7,6 +7,7 @@ class W_DictObject(W_Object): from pypy.objspace.std.dicttype import dict_typedef as typedef + _immutable_ = True def __init__(w_self, space, w_otherdict=None): if w_otherdict is None: w_self.content = r_dict(space.eq_w, space.hash_w) Modified: pypy/dist/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/stringobject.py (original) +++ pypy/dist/pypy/objspace/std/stringobject.py Tue Oct 16 16:18:46 2007 @@ -19,6 +19,7 @@ class W_StringObject(W_Object): from pypy.objspace.std.stringtype import str_typedef as typedef + _immutable_ = True def __init__(w_self, str): w_self._value = str From cfbolz at codespeak.net Tue Oct 16 16:20:45 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 16 Oct 2007 16:20:45 +0200 (CEST) Subject: [pypy-svn] r47502 - pypy/dist/pypy/interpreter Message-ID: <20071016142045.4763180EE@code0.codespeak.net> Author: cfbolz Date: Tue Oct 16 16:20:44 2007 New Revision: 47502 Modified: pypy/dist/pypy/interpreter/gateway.py Log: and another immutable-hint Modified: pypy/dist/pypy/interpreter/gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/gateway.py (original) +++ pypy/dist/pypy/interpreter/gateway.py Tue Oct 16 16:20:44 2007 @@ -367,6 +367,7 @@ class BuiltinCode(eval.Code): "The code object implementing a built-in (interpreter-level) hook." + _immutable_ = True hidden_applevel = True descrmismatch_op = None descr_reqcls = None From fijal at codespeak.net Tue Oct 16 17:37:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Oct 2007 17:37:27 +0200 (CEST) Subject: [pypy-svn] r47503 - pypy/dist/pypy/module/_ffi Message-ID: <20071016153727.67F0280ED@code0.codespeak.net> Author: fijal Date: Tue Oct 16 17:37:26 2007 New Revision: 47503 Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/structure.py Log: * Do the right thing to access buffer * Do the right think with characters (test coming) Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Tue Oct 16 17:37:26 2007 @@ -142,13 +142,18 @@ mod = space.getbuiltinmodule('_ffi') w_StructureInstance = space.getattr(mod, w('StructureInstance')) if space.is_true(space.isinstance(w_arg, w_StructureInstance)): - #ptr.push_arg(lltype.cast_int_to_ptr(rffi.VOIDP, space.int_w(space.getattr(w_arg, w('buffer'))))) - push_func(add_arg, argdesc, w_arg.ll_buffer) + ptr = rffi.cast(rffi.VOIDP, space.int_w(space.getattr(w_arg, w('buffer')))) + push_func(add_arg, argdesc, ptr) else: raise OperationError(space.w_TypeError, w( "Expected structure, array or simple type")) elif argtype == "c" or argtype == "b" or argtype == "B": - push_func(add_arg, argdesc, space.str_w(w_arg)) + s = space.str_w(w_arg) + if len(s) != 1: + raise OperationError(space.w_ValueError, w( + "Expected string of length one as character")) + s = s[0] + push_func(add_arg, argdesc, s) else: assert argtype in "iIhHlLqQ" push_func(add_arg, argdesc, space.int_w(w_arg)) Modified: pypy/dist/pypy/module/_ffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_ffi/structure.py (original) +++ pypy/dist/pypy/module/_ffi/structure.py Tue Oct 16 17:37:26 2007 @@ -46,14 +46,14 @@ def push_field(self, num, value): ptr = rffi.ptradd(self.ll_buffer, self.ll_positions[num]) TP = lltype.typeOf(value) - T = rffi.CArrayPtr(TP) + T = lltype.Ptr(rffi.CArray(TP)) rffi.cast(T, ptr)[0] = value push_field._annspecialcase_ = 'specialize:argtype(2)' def cast_pos(self, ll_t): i = self.next_pos pos = rffi.ptradd(self.ll_buffer, self.ll_positions[i]) - TP = rffi.CArrayPtr(ll_t) + TP = lltype.Ptr(rffi.CArray(ll_t)) return rffi.cast(TP, pos)[0] cast_pos._annspecialcase_ = 'specialize:arg(1)' @@ -101,6 +101,9 @@ if self.free_afterwards: lltype.free(self.ll_buffer, flavor='raw') + def getbuffer(space, self): + return space.wrap(rffi.cast(rffi.INT, self.ll_buffer)) + def descr_new_structure_instance(space, w_type, w_shape, w_adr, w_fieldinits): return W_StructureInstance(space, w_shape, w_adr, w_fieldinits) @@ -109,4 +112,5 @@ __new__ = interp2app(descr_new_structure_instance), __getattr__ = interp2app(W_StructureInstance.getattr), __setattr__ = interp2app(W_StructureInstance.setattr), + buffer = GetSetProperty(W_StructureInstance.getbuffer), ) From cfbolz at codespeak.net Tue Oct 16 18:32:39 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 16 Oct 2007 18:32:39 +0200 (CEST) Subject: [pypy-svn] r47504 - in pypy/dist/pypy: config translator/c translator/c/test Message-ID: <20071016163239.8727C80FA@code0.codespeak.net> Author: cfbolz Date: Tue Oct 16 18:32:38 2007 New Revision: 47504 Modified: pypy/dist/pypy/config/translationoption.py pypy/dist/pypy/translator/c/gc.py pypy/dist/pypy/translator/c/test/test_boehm.py Log: remove exact boehm stuff, since it never worked (mostly due to boehm crashing) Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Tue Oct 16 18:32:38 2007 @@ -41,7 +41,7 @@ default=False, cmdline="--llvm-via-c", requires=[("translation.backend", "llvm")]), ChoiceOption("gc", "Garbage Collection Strategy", - ["boehm", "ref", "framework", "none", "exact_boehm"], + ["boehm", "ref", "framework", "none"], "ref", requires={ "ref": [("translation.rweakref", False)], # XXX "none": [("translation.rweakref", False)], # XXX Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Tue Oct 16 18:32:38 2007 @@ -169,8 +169,6 @@ class BoehmInfo: finalizer = None - # for MoreExactBoehmGcPolicy - malloc_exact = False class BoehmGcPolicy(BasicGcPolicy): transformerclass = boehm.BoehmGCTransformer @@ -260,59 +258,6 @@ nodekind = 'framework rtti' -class MoreExactBoehmGcPolicy(BoehmGcPolicy): - """ policy to experiment with giving some layout information to boehm. Use - new class to prevent breakage. """ - - def __init__(self, db, thread_enabled=False): - super(MoreExactBoehmGcPolicy, self).__init__(db, thread_enabled) - self.exactly_typed_structs = {} - - def get_descr_name(self, defnode): - # XXX somewhat illegal way of introducing a name - return '%s__gc_descr__' % (defnode.name, ) - - def pre_pre_gc_code(self): - for line in super(MoreExactBoehmGcPolicy, self).pre_pre_gc_code(): - yield line - yield "#include " - - def struct_setup(self, structdefnode, rtti): - T = structdefnode.STRUCT - if T._is_atomic(): - malloc_exact = False - else: - if T._is_varsize(): - malloc_exact = T._flds[T._arrayfld]._is_atomic() - else: - malloc_exact = True - if malloc_exact: - if structdefnode.gcinfo is None: - structdefnode.gcinfo = BoehmInfo() - structdefnode.gcinfo.malloc_exact = True - self.exactly_typed_structs[structdefnode.STRUCT] = structdefnode - - def struct_after_definition(self, defnode): - if defnode.gcinfo and defnode.gcinfo.malloc_exact: - yield 'GC_descr %s;' % (self.get_descr_name(defnode), ) - - def gc_startup_code(self): - for line in super(MoreExactBoehmGcPolicy, self).gc_startup_code(): - yield line - for TYPE, defnode in self.exactly_typed_structs.iteritems(): - T = defnode.gettype().replace("@", "") - yield "{" - yield "GC_word T_bitmap[GC_BITMAP_SIZE(%s)] = {0};" % (T, ) - for field in TYPE._flds: - if getattr(TYPE, field) == lltype.Void: - continue - yield "GC_set_bit(T_bitmap, GC_WORD_OFFSET(%s, %s));" % ( - T, defnode.c_struct_field_name(field)) - yield "%s = GC_make_descriptor(T_bitmap, GC_WORD_LEN(%s));" % ( - self.get_descr_name(defnode), T) - yield "}" - - # to get an idea how it looks like with no refcount/gc at all class NoneGcPolicy(BoehmGcPolicy): @@ -381,7 +326,6 @@ name_to_gcpolicy = { 'boehm': BoehmGcPolicy, - 'exact_boehm': MoreExactBoehmGcPolicy, 'ref': RefcountingGcPolicy, 'none': NoneGcPolicy, 'framework': FrameworkGcPolicy, Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Tue Oct 16 18:32:38 2007 @@ -335,7 +335,3 @@ c_fn(100) -class TestUsingExactBoehm(TestUsingBoehm): - gcpolicy = "exact_boehm" - - From fijal at codespeak.net Tue Oct 16 22:56:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Oct 2007 22:56:51 +0200 (CEST) Subject: [pypy-svn] r47505 - in pypy/dist/pypy: module/_ffi rlib rlib/test Message-ID: <20071016205651.0497380CC@code0.codespeak.net> Author: fijal Date: Tue Oct 16 22:56:50 2007 New Revision: 47505 Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/rlib/libffi.py pypy/dist/pypy/rlib/test/test_libffi.py Log: * Some more rpythonism. * Disable things that are known to be broken (uchar, ushort and friends) Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Tue Oct 16 22:56:50 2007 @@ -11,11 +11,11 @@ TYPEMAP = { # XXX A mess with unsigned/signed/normal chars :-/ - 'c' : ffi_type_uchar, - 'b' : ffi_type_schar, - 'B' : ffi_type_uchar, - 'h' : ffi_type_sshort, - 'H' : ffi_type_ushort, + #'c' : ffi_type_uchar, + #'b' : ffi_type_schar, + #'B' : ffi_type_uchar, + #'h' : ffi_type_sshort, + #'H' : ffi_type_ushort, 'i' : ffi_type_sint, 'I' : ffi_type_uint, 'l' : ffi_type_slong, @@ -30,11 +30,11 @@ } LL_TYPEMAP = { - 'c' : rffi.CHAR, - 'b' : rffi.UCHAR, - 'B' : rffi.CHAR, - 'h' : rffi.SHORT, - 'H' : rffi.USHORT, + #'c' : rffi.CHAR, + #'b' : rffi.UCHAR, + #'B' : rffi.CHAR, + #'h' : rffi.SHORT, + #'H' : rffi.USHORT, 'i' : rffi.INT, 'I' : rffi.UINT, 'l' : rffi.LONG, @@ -155,7 +155,7 @@ s = s[0] push_func(add_arg, argdesc, s) else: - assert argtype in "iIhHlLqQ" + #assert argtype "iIhHlLqQ" push_func(add_arg, argdesc, space.int_w(w_arg)) unwrap_arg._annspecialcase_ = 'specialize:arg(1)' # we should have also here specialize:argtype(5) :-/ @@ -170,10 +170,14 @@ elif c == 'P': res = func(arg, rffi.VOIDP) return space.wrap(rffi.cast(rffi.INT, res)) - elif c == 'q' or c == 'Q' or c == 'L': - return space.newlong(func(arg, ll_type)) - else: + #elif c == 'q' or c == 'Q' or c == 'L': + # return space.newlong(func(arg, ll_type)) + elif c == 'f' or c == 'd': + return space.wrap(float(func(arg, ll_type))) + elif c == 'c' or c == 'b' or c == 'B': return space.wrap(func(arg, ll_type)) + else: + return space.wrap(intmask(func(arg, ll_type))) return space.w_None wrap_result._annspecialcase_ = 'specialize:arg(3)' @@ -181,16 +185,16 @@ return ptr.call(ll_type) ptr_call._annspecialcase_ = 'specialize:arg(1)' +def push(ptr, argdesc, value): + ptr.push_arg(value) +push._annspecialcase_ = 'specialize:argtype(2)' + class W_FuncPtr(Wrappable): def __init__(self, ptr, argtypes, restype): self.ptr = ptr self.restype = restype self.argtypes = argtypes - def push(ptr, argdesc, value): - ptr.push_arg(value) - push = staticmethod(push) - def call(self, space, arguments): args_w, kwds_w = arguments.unpack() # C has no keyword arguments @@ -199,8 +203,10 @@ "Provided keyword arguments for C function call")) to_free = [] i = 0 - for argtype, w_arg in zip(self.argtypes, args_w): - unwrap_arg(space, self.push, self.ptr, i, argtype, w_arg, to_free) + for i in range(len(self.argtypes)): + argtype = self.argtypes[i] + w_arg = args_w[i] + unwrap_arg(space, push, self.ptr, i, argtype, w_arg, to_free) i += 1 try: return wrap_result(space, self.restype, self.ptr, ptr_call) Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Tue Oct 16 22:56:50 2007 @@ -6,6 +6,7 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import intmask +from pypy.rlib.objectmodel import we_are_translated includes = ['dlfcn.h', 'ffi.h'] include_dirs = ['/usr/include/libffi'] @@ -144,10 +145,11 @@ return TYPE_MAP[tp] cast_type_to_ffitype._annspecialcase_ = 'specialize:memo' -def push_arg_as_ffiptr(ffitp, TP, arg, ll_buf): +def push_arg_as_ffiptr(ffitp, arg, ll_buf): # this is for primitive types. For structures and arrays # would be something different (more dynamic) - TP_P = rffi.CArrayPtr(TP) + TP = lltype.typeOf(arg) + TP_P = lltype.Ptr(rffi.CArray(TP)) buf = rffi.cast(TP_P, ll_buf) buf[0] = arg push_arg_as_ffiptr._annspecialcase_ = 'specialize:argtype(1)' @@ -190,18 +192,22 @@ if self.pushed_args == self.argnum: raise TypeError("Too much arguments, eats %d, pushed %d" % (self.argnum, self.argnum + 1)) - TP = lltype.typeOf(value) - if isinstance(TP, lltype.Ptr): - if TP.TO._gckind != 'raw': - raise ValueError("Can only push raw values to C, not 'gc'") - # XXX probably we should recursively check for struct fields - # here, lets just ignore that for now - if isinstance(TP.TO, lltype.Array) and not \ - TP.TO._hints.get('nolength', None): - raise ValueError("Can only push to C arrays without length info") - push_arg_as_ffiptr(self.argtypes[self.pushed_args], TP, value, + if not we_are_translated(): + TP = lltype.typeOf(value) + if isinstance(TP, lltype.Ptr): + if TP.TO._gckind != 'raw': + raise ValueError("Can only push raw values to C, not 'gc'") + # XXX probably we should recursively check for struct fields + # here, lets just ignore that for now + if isinstance(TP.TO, lltype.Array): + try: + TP.TO._hints['nolength'] + except KeyError: + raise ValueError("Can only push to C arrays without length info") + push_arg_as_ffiptr(self.argtypes[self.pushed_args], value, self.ll_args[self.pushed_args]) self.pushed_args += 1 + push_arg._annspecialcase_ = 'specialize:argtype(1)' def _check_args(self): if self.pushed_args < self.argnum: @@ -215,14 +221,14 @@ c_ffi_call(self.ll_cif, self.funcsym, rffi.cast(rffi.VOIDP, self.ll_result), rffi.cast(VOIDPP, self.ll_args)) - if self.restype != ffi_type_void: - TP = rffi.CArrayPtr(RES_TP) + if RES_TP is not lltype.Void: + TP = lltype.Ptr(rffi.CArray(RES_TP)) res = rffi.cast(TP, self.ll_result)[0] else: res = None self._clean_args() return res - call._annspecialcase_ = 'specialize:argtype(1)' + call._annspecialcase_ = 'specialize:arg(1)' def __del__(self): argnum = len(self.argtypes) Modified: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_libffi.py (original) +++ pypy/dist/pypy/rlib/test/test_libffi.py Tue Oct 16 22:56:50 2007 @@ -112,7 +112,8 @@ c_pow = libm.getpointer('pow', [ffi_type_double, ffi_type_double], ffi_type_double) c_pow.push_arg(x) c_pow.push_arg(y) - return c_pow.call(rffi.DOUBLE) + res = c_pow.call(rffi.DOUBLE) + return res fn = compile(f, [float, float]) res = fn(2.0, 4.0) From fijal at codespeak.net Wed Oct 17 00:15:16 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Oct 2007 00:15:16 +0200 (CEST) Subject: [pypy-svn] r47507 - in pypy/dist/pypy/module/_ffi: . test Message-ID: <20071016221516.21C8880DD@code0.codespeak.net> Author: fijal Date: Wed Oct 17 00:15:14 2007 New Revision: 47507 Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/structure.py pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: * Increase testibility * Few fixes * Make it non-rpythonic again Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Wed Oct 17 00:15:14 2007 @@ -11,11 +11,11 @@ TYPEMAP = { # XXX A mess with unsigned/signed/normal chars :-/ - #'c' : ffi_type_uchar, - #'b' : ffi_type_schar, - #'B' : ffi_type_uchar, - #'h' : ffi_type_sshort, - #'H' : ffi_type_ushort, + 'c' : ffi_type_uchar, + 'b' : ffi_type_schar, + 'B' : ffi_type_uchar, + 'h' : ffi_type_sshort, + 'H' : ffi_type_ushort, 'i' : ffi_type_sint, 'I' : ffi_type_uint, 'l' : ffi_type_slong, @@ -30,11 +30,11 @@ } LL_TYPEMAP = { - #'c' : rffi.CHAR, - #'b' : rffi.UCHAR, - #'B' : rffi.CHAR, - #'h' : rffi.SHORT, - #'H' : rffi.USHORT, + 'c' : rffi.CHAR, + 'b' : rffi.UCHAR, + 'B' : rffi.CHAR, + 'h' : rffi.SHORT, + 'H' : rffi.USHORT, 'i' : rffi.INT, 'I' : rffi.UINT, 'l' : rffi.LONG, @@ -166,7 +166,10 @@ for c, ll_type in ll_typemap_iter: if restype == c: if c == 's': - return space.wrap(rffi.charp2str(func(arg, rffi.CCHARP))) + ptr = func(arg, rffi.CCHARP) + if not ptr: + return space.w_None + return space.wrap(rffi.charp2str(ptr)) elif c == 'P': res = func(arg, rffi.VOIDP) return space.wrap(rffi.cast(rffi.INT, res)) Modified: pypy/dist/pypy/module/_ffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_ffi/structure.py (original) +++ pypy/dist/pypy/module/_ffi/structure.py Wed Oct 17 00:15:14 2007 @@ -59,15 +59,12 @@ class W_StructureInstance(Wrappable): def __init__(self, space, w_shape, w_address, w_fieldinits): - if space.is_true(w_fieldinits): - raise OperationError(space.w_ValueError, space.wrap( - "Fields should be not initialized with values by now")) + self.free_afterwards = False w_fields = space.getattr(w_shape, space.wrap('fields')) fields = unpack_fields(space, w_fields) size, pos = size_and_pos(fields) self.fields = fields if space.is_true(w_address): - self.free_afterwards = False self.ll_buffer = rffi.cast(rffi.VOIDP, space.int_w(w_address)) else: self.free_afterwards = True @@ -75,6 +72,10 @@ zero=True) self.ll_positions = pos self.next_pos = 0 + if space.is_true(w_fieldinits): + for w_field in space.unpackiterable(w_fieldinits): + w_value = space.getitem(w_fieldinits, w_field) + self.setattr(space, space.str_w(w_field), w_value) def getattr(self, space, attr): if attr.startswith('tm'): Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Wed Oct 17 00:15:14 2007 @@ -1,6 +1,7 @@ from pypy.conftest import gettestobjspace +from pypy.translator.tool.cbuild import compile_c_module import os, sys, py @@ -11,18 +12,62 @@ class AppTestCTypes: def prepare_c_example(): from pypy.tool.udir import udir - udir.join("xlib.c").write(py.code.Source(""" - typedef struct x { - char x1; - long x2; - struct x *x3; + c_file = udir.join("xlib.c") + c_file.write(py.code.Source(''' + #include + #include + + struct x + { + int x1; + short x2; + char x3; + struct x* next; + }; + + char inner_struct_elem(struct x *x1) + { + return x1->next->x3; } - """)) + + struct x* create_double_struct() + { + struct x* x1, *x2; + + x1 = (struct x*)malloc(sizeof(struct x)); + x2 = (struct x*)malloc(sizeof(struct x)); + x1->next = x2; + x2->x2 = 3; + return x1; + } + + const char *static_str = "xxxxxx"; + + unsigned short add_shorts(short one, short two) + { + return one + two; + } + + char get_char(char* s, unsigned short num) + { + return s[num]; + } + + char *char_check(char x, char y) + { + if (y == static_str[0]) + return static_str; + return NULL; + } + ''')) + compile_c_module([c_file], 'x') + return str(udir.join('x.so')) prepare_c_example = staticmethod(prepare_c_example) def setup_class(cls): - cls.space = gettestobjspace(usemodules=('_ffi','struct')) - cls.prepare_c_example() + space = gettestobjspace(usemodules=('_ffi','struct')) + cls.space = space + cls.w_lib_name = space.wrap(cls.prepare_c_example()) def test_libload(self): import _ffi @@ -37,6 +82,28 @@ assert isinstance(func, _ffi.FuncPtr) raises(AttributeError, "libc.xxxxxxxxxxxxxx") + def test_getchar(self): + import _ffi + lib = _ffi.CDLL(self.lib_name) + get_char = lib.ptr('get_char', ['s', 'H'], 'c') + assert get_char('dupa', 2) == 'p' + assert get_char('dupa', 1) == 'u' + skip("this module does not do overflow checking by now") + raises(OverflowError, "get_char('xxx', 2 ** 17)") + + def test_returning_str(self): + import _ffi + lib = _ffi.CDLL(self.lib_name) + char_check = lib.ptr('char_check', ['c', 'c'], 's') + assert char_check('y', 'x') == 'xxxxxx' + assert char_check('x', 'y') is None + + def test_short_addition(self): + import _ffi + lib = _ffi.CDLL(self.lib_name) + short_add = lib.ptr('add_shorts', ['h', 'h'], 'H') + assert short_add(1, 2) == 3 + def test_rand(self): import _ffi libc = _ffi.CDLL('libc.so.6') @@ -107,8 +174,20 @@ t = Tm(gmtime(x)) assert t.tm_year == 70 assert t.tm_sec == 1 - assert t.tm_min == 2 - + assert t.tm_min == 2 + + def test_nested_structures(self): + import _ffi + lib = _ffi.CDLL(self.lib_name) + inner = lib.ptr("inner_struct_elem", ['P'], 'c') + X = _ffi.Structure([('x1', 'i'), ('x2', 'h'), ('x3', 'c'), ('next', 'P')]) + x = X(next=X(next=None, x3='x'), x1=1, x2=2, x3='x') + assert X(x.next).x3 == 'x' + assert inner(x) == 'x' + create_double_struct = lib.ptr("create_double_struct", [], 'P') + x = create_double_struct() + assert X(X(x).next).x2 == 3 + + def test_longs_ulongs(self): + skip("Not implemented yet") - #def test_nested_structures(self): - # From fijal at codespeak.net Wed Oct 17 01:19:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Oct 2007 01:19:51 +0200 (CEST) Subject: [pypy-svn] r47508 - in pypy/dist/pypy/module/_ffi: . test Message-ID: <20071016231951.6ADD880DC@code0.codespeak.net> Author: fijal Date: Wed Oct 17 01:19:49 2007 New Revision: 47508 Modified: pypy/dist/pypy/module/_ffi/TODO pypy/dist/pypy/module/_ffi/__init__.py pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: * A skipped test * Make it rpython once again Modified: pypy/dist/pypy/module/_ffi/TODO ============================================================================== --- pypy/dist/pypy/module/_ffi/TODO (original) +++ pypy/dist/pypy/module/_ffi/TODO Wed Oct 17 01:19:49 2007 @@ -1,8 +1,7 @@ * long support with all messy corners (when to create long integer and - such) + such), also short overflowing * add arrays -* keyword arguments for structure creation Modified: pypy/dist/pypy/module/_ffi/__init__.py ============================================================================== --- pypy/dist/pypy/module/_ffi/__init__.py (original) +++ pypy/dist/pypy/module/_ffi/__init__.py Wed Oct 17 01:19:49 2007 @@ -10,11 +10,12 @@ applevelname = '_ffi' interpleveldefs = { - 'CDLL' : 'interp_ffi.W_CDLL', - 'FuncPtr' : 'interp_ffi.W_FuncPtr', + 'CDLL' : 'interp_ffi.W_CDLL', + 'FuncPtr' : 'interp_ffi.W_FuncPtr', 'StructureInstance' : 'structure.W_StructureInstance', } appleveldefs = { 'Structure' : 'app_ffi.Structure', + #'StructureInstance' : 'app_ffi.StructureInstance', } Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Wed Oct 17 01:19:49 2007 @@ -178,7 +178,9 @@ elif c == 'f' or c == 'd': return space.wrap(float(func(arg, ll_type))) elif c == 'c' or c == 'b' or c == 'B': - return space.wrap(func(arg, ll_type)) + return space.wrap(chr(rffi.cast(rffi.INT, func(arg, ll_type)))) + elif c == 'h' or c == 'H': + return space.wrap(rffi.cast(rffi.INT, func(arg, ll_type))) else: return space.wrap(intmask(func(arg, ll_type))) return space.w_None Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Wed Oct 17 01:19:49 2007 @@ -188,6 +188,20 @@ x = create_double_struct() assert X(X(x).next).x2 == 3 + def test_implicit_structure(self): + skip("Does not work yet") + import _ffi + lib = _ffi.CDLL(self.lib_name) + X = _ffi.Structure([('x1', 'i'), ('x2', 'h'), ('x3', 'c'), ('next', 'self')]) + inner = lib.ptr("inner_struct_elem", [X], 'c') + x = X(next=X(next=None, x3='x'), x1=1, x2=2, x3='x') + assert x.next.x3 == 'x' + assert inner(x) == 'x' + create_double_struct = lib.ptr("create_double_struct", [], X) + x = create_double_struct() + assert x.next.x2 == 3 + + def test_longs_ulongs(self): skip("Not implemented yet") From fijal at codespeak.net Wed Oct 17 12:33:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Oct 2007 12:33:51 +0200 (CEST) Subject: [pypy-svn] r47509 - in pypy/dist/pypy/module/_ffi: . test Message-ID: <20071017103351.ED5F480D5@code0.codespeak.net> Author: fijal Date: Wed Oct 17 12:33:50 2007 New Revision: 47509 Modified: pypy/dist/pypy/module/_ffi/TODO pypy/dist/pypy/module/_ffi/__init__.py pypy/dist/pypy/module/_ffi/app_ffi.py pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/structure.py pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: Add arrays, working almost out of the box. Modified: pypy/dist/pypy/module/_ffi/TODO ============================================================================== --- pypy/dist/pypy/module/_ffi/TODO (original) +++ pypy/dist/pypy/module/_ffi/TODO Wed Oct 17 12:33:50 2007 @@ -2,6 +2,6 @@ * long support with all messy corners (when to create long integer and such), also short overflowing -* add arrays - +* add faulty parameters checking (they end up with fatal rpython error KeyErro + or sth like that by now) Modified: pypy/dist/pypy/module/_ffi/__init__.py ============================================================================== --- pypy/dist/pypy/module/_ffi/__init__.py (original) +++ pypy/dist/pypy/module/_ffi/__init__.py Wed Oct 17 12:33:50 2007 @@ -12,10 +12,12 @@ interpleveldefs = { 'CDLL' : 'interp_ffi.W_CDLL', 'FuncPtr' : 'interp_ffi.W_FuncPtr', - 'StructureInstance' : 'structure.W_StructureInstance', + 'StructureInstance' : 'structure.W_StructureInstance', + 'ArrayInstance' : 'array.W_ArrayInstance', } appleveldefs = { 'Structure' : 'app_ffi.Structure', + 'Array' : 'app_ffi.Array', #'StructureInstance' : 'app_ffi.StructureInstance', } Modified: pypy/dist/pypy/module/_ffi/app_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/app_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/app_ffi.py Wed Oct 17 12:33:50 2007 @@ -13,3 +13,11 @@ raise TypeError("Keyword arguments not allowed when passing address argument") return StructureInstance(self, args[0], None) return StructureInstance(self, None, kwds) + +class Array(object): + def __init__(self, of): + self.of = of + + def __call__(self, size): + from _ffi import ArrayInstance + return ArrayInstance(self.of, size) Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Wed Oct 17 12:33:50 2007 @@ -141,7 +141,9 @@ else: mod = space.getbuiltinmodule('_ffi') w_StructureInstance = space.getattr(mod, w('StructureInstance')) - if space.is_true(space.isinstance(w_arg, w_StructureInstance)): + w_ArrayInstance = space.getattr(mod, w('ArrayInstance')) + if space.is_true(space.isinstance(w_arg, w_StructureInstance)) or\ + space.is_true(space.isinstance(w_arg, w_ArrayInstance)): ptr = rffi.cast(rffi.VOIDP, space.int_w(space.getattr(w_arg, w('buffer')))) push_func(add_arg, argdesc, ptr) else: @@ -162,33 +164,37 @@ ll_typemap_iter = unrolling_iterable(LL_TYPEMAP.items()) -def wrap_result(space, restype, arg, func): +def wrap_result(space, restype, arg, argdesc, func): for c, ll_type in ll_typemap_iter: if restype == c: if c == 's': - ptr = func(arg, rffi.CCHARP) + ptr = func(arg, argdesc, rffi.CCHARP) if not ptr: return space.w_None return space.wrap(rffi.charp2str(ptr)) elif c == 'P': - res = func(arg, rffi.VOIDP) + res = func(arg, argdesc, rffi.VOIDP) + if not res: + return space.w_None return space.wrap(rffi.cast(rffi.INT, res)) #elif c == 'q' or c == 'Q' or c == 'L': # return space.newlong(func(arg, ll_type)) elif c == 'f' or c == 'd': - return space.wrap(float(func(arg, ll_type))) + return space.wrap(float(func(arg, argdesc, ll_type))) elif c == 'c' or c == 'b' or c == 'B': - return space.wrap(chr(rffi.cast(rffi.INT, func(arg, ll_type)))) + return space.wrap(chr(rffi.cast(rffi.INT, func(arg, argdesc, + ll_type)))) elif c == 'h' or c == 'H': - return space.wrap(rffi.cast(rffi.INT, func(arg, ll_type))) + return space.wrap(rffi.cast(rffi.INT, func(arg, argdesc, + ll_type))) else: - return space.wrap(intmask(func(arg, ll_type))) + return space.wrap(intmask(func(arg, argdesc, ll_type))) return space.w_None wrap_result._annspecialcase_ = 'specialize:arg(3)' -def ptr_call(ptr, ll_type): +def ptr_call(ptr, some_arg, ll_type): return ptr.call(ll_type) -ptr_call._annspecialcase_ = 'specialize:arg(1)' +ptr_call._annspecialcase_ = 'specialize:arg(2)' def push(ptr, argdesc, value): ptr.push_arg(value) @@ -214,7 +220,7 @@ unwrap_arg(space, push, self.ptr, i, argtype, w_arg, to_free) i += 1 try: - return wrap_result(space, self.restype, self.ptr, ptr_call) + return wrap_result(space, self.restype, self.ptr, None, ptr_call) finally: for elem in to_free: lltype.free(elem, flavor='raw') Modified: pypy/dist/pypy/module/_ffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_ffi/structure.py (original) +++ pypy/dist/pypy/module/_ffi/structure.py Wed Oct 17 12:33:50 2007 @@ -50,12 +50,11 @@ rffi.cast(T, ptr)[0] = value push_field._annspecialcase_ = 'specialize:argtype(2)' -def cast_pos(self, ll_t): - i = self.next_pos +def cast_pos(self, i, ll_t): pos = rffi.ptradd(self.ll_buffer, self.ll_positions[i]) TP = lltype.Ptr(rffi.CArray(ll_t)) return rffi.cast(TP, pos)[0] -cast_pos._annspecialcase_ = 'specialize:arg(1)' +cast_pos._annspecialcase_ = 'specialize:arg(2)' class W_StructureInstance(Wrappable): def __init__(self, space, w_shape, w_address, w_fieldinits): @@ -71,7 +70,6 @@ self.ll_buffer = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', zero=True) self.ll_positions = pos - self.next_pos = 0 if space.is_true(w_fieldinits): for w_field in space.unpackiterable(w_fieldinits): w_value = space.getitem(w_fieldinits, w_field) @@ -83,9 +81,7 @@ for i in range(len(self.fields)): name, c = self.fields[i] if name == attr: - # XXX RPython-trick for passing lambda around - self.next_pos = i - return wrap_result(space, c, self, cast_pos) + return wrap_result(space, c, self, i, cast_pos) raise OperationError(space.w_AttributeError, space.wrap( "C Structure has no attribute %s" % attr)) getattr.unwrap_spec = ['self', ObjSpace, str] Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Wed Oct 17 12:33:50 2007 @@ -59,6 +59,16 @@ return static_str; return NULL; } + + int get_array_elem(int* stuff, int num) + { + return stuff[num]; + } + + struct x* get_array_elem_s(struct x** array, int num) + { + return array[num]; + } ''')) compile_c_module([c_file], 'x') return str(udir.join('x.so')) @@ -188,6 +198,34 @@ x = create_double_struct() assert X(X(x).next).x2 == 3 + def test_array(self): + import _ffi + lib = _ffi.CDLL(self.lib_name) + A = _ffi.Array('i') + get_array_elem = lib.ptr('get_array_elem', ['P', 'i'], 'i') + a = A(10) + a[8] = 3 + a[7] = 1 + a[6] = 2 + assert get_array_elem(a, 9) == 0 + assert get_array_elem(a, 8) == 3 + assert get_array_elem(a, 7) == 1 + assert get_array_elem(a, 6) == 2 + assert a[3] == 0 + + def test_array_of_structure(self): + import _ffi + lib = _ffi.CDLL(self.lib_name) + A = _ffi.Array('P') + X = _ffi.Structure([('x1', 'i'), ('x2', 'h'), ('x3', 'c'), ('next', 'P')]) + x = X(x2=3) + a = A(3) + a[1] = x + get_array_elem_s = lib.ptr('get_array_elem_s', ['P', 'i'], 'P') + ptr1 = get_array_elem_s(a, 0) + assert ptr1 is None + assert X(get_array_elem_s(a, 1)).x2 == 3 + def test_implicit_structure(self): skip("Does not work yet") import _ffi From fijal at codespeak.net Wed Oct 17 12:43:25 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Oct 2007 12:43:25 +0200 (CEST) Subject: [pypy-svn] r47510 - pypy/dist/pypy/module/_ffi Message-ID: <20071017104325.0778380D7@code0.codespeak.net> Author: fijal Date: Wed Oct 17 12:43:25 2007 New Revision: 47510 Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py Log: RPyhton fix. Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Wed Oct 17 12:43:25 2007 @@ -190,7 +190,7 @@ else: return space.wrap(intmask(func(arg, argdesc, ll_type))) return space.w_None -wrap_result._annspecialcase_ = 'specialize:arg(3)' +wrap_result._annspecialcase_ = 'specialize:arg(4)' def ptr_call(ptr, some_arg, ll_type): return ptr.call(ll_type) From fijal at codespeak.net Wed Oct 17 13:16:11 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Oct 2007 13:16:11 +0200 (CEST) Subject: [pypy-svn] r47511 - pypy/dist/pypy/module/_ffi Message-ID: <20071017111611.8705880D1@code0.codespeak.net> Author: fijal Date: Wed Oct 17 13:16:09 2007 New Revision: 47511 Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/structure.py Log: Name & args consistency Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Wed Oct 17 13:16:09 2007 @@ -120,18 +120,18 @@ push_func(add_arg, argdesc, ll_str) return ll_str -def unwrap_arg(space, push_func, add_arg, argdesc, argtype, w_arg, to_free): +def unwrap_value(space, push_func, add_arg, argdesc, tp, w_arg, to_free): w = space.wrap # XXX how to handle LONGLONG here? # they're probably long, so we'll not get them through int_w - if argtype == "d" or argtype == "f": + if tp == "d" or tp == "f": push_func(add_arg, argdesc, space.float_w(w_arg)) - elif argtype == "s": + elif tp == "s": ll_str = rffi.str2charp(space.str_w(w_arg)) if to_free is not None: to_free.append(ll_str) push_func(add_arg, argdesc, ll_str) - elif argtype == "P": + elif tp == "P": # check for NULL ptr if space.is_w(w_arg, space.w_None): push_func(add_arg, argdesc, lltype.nullptr(rffi.VOIDP.TO)) @@ -149,7 +149,7 @@ else: raise OperationError(space.w_TypeError, w( "Expected structure, array or simple type")) - elif argtype == "c" or argtype == "b" or argtype == "B": + elif tp == "c" or tp == "b" or tp == "B": s = space.str_w(w_arg) if len(s) != 1: raise OperationError(space.w_ValueError, w( @@ -157,40 +157,39 @@ s = s[0] push_func(add_arg, argdesc, s) else: - #assert argtype "iIhHlLqQ" + #assert tp "iIhHlLqQ" push_func(add_arg, argdesc, space.int_w(w_arg)) -unwrap_arg._annspecialcase_ = 'specialize:arg(1)' -# we should have also here specialize:argtype(5) :-/ +unwrap_value._annspecialcase_ = 'specialize:arg(1)' ll_typemap_iter = unrolling_iterable(LL_TYPEMAP.items()) -def wrap_result(space, restype, arg, argdesc, func): +def wrap_value(space, func, add_arg, argdesc, tp): for c, ll_type in ll_typemap_iter: - if restype == c: + if tp == c: if c == 's': - ptr = func(arg, argdesc, rffi.CCHARP) + ptr = func(add_arg, argdesc, rffi.CCHARP) if not ptr: return space.w_None return space.wrap(rffi.charp2str(ptr)) elif c == 'P': - res = func(arg, argdesc, rffi.VOIDP) + res = func(add_arg, argdesc, rffi.VOIDP) if not res: return space.w_None return space.wrap(rffi.cast(rffi.INT, res)) #elif c == 'q' or c == 'Q' or c == 'L': # return space.newlong(func(arg, ll_type)) elif c == 'f' or c == 'd': - return space.wrap(float(func(arg, argdesc, ll_type))) + return space.wrap(float(func(add_arg, argdesc, ll_type))) elif c == 'c' or c == 'b' or c == 'B': - return space.wrap(chr(rffi.cast(rffi.INT, func(arg, argdesc, + return space.wrap(chr(rffi.cast(rffi.INT, func(add_arg, argdesc, ll_type)))) elif c == 'h' or c == 'H': - return space.wrap(rffi.cast(rffi.INT, func(arg, argdesc, + return space.wrap(rffi.cast(rffi.INT, func(add_arg, argdesc, ll_type))) else: - return space.wrap(intmask(func(arg, argdesc, ll_type))) + return space.wrap(intmask(func(add_arg, argdesc, ll_type))) return space.w_None -wrap_result._annspecialcase_ = 'specialize:arg(4)' +wrap_value._annspecialcase_ = 'specialize:arg(1)' def ptr_call(ptr, some_arg, ll_type): return ptr.call(ll_type) @@ -217,10 +216,10 @@ for i in range(len(self.argtypes)): argtype = self.argtypes[i] w_arg = args_w[i] - unwrap_arg(space, push, self.ptr, i, argtype, w_arg, to_free) + unwrap_value(space, push, self.ptr, i, argtype, w_arg, to_free) i += 1 try: - return wrap_result(space, self.restype, self.ptr, None, ptr_call) + return wrap_value(space, ptr_call, self.ptr, None, self.restype) finally: for elem in to_free: lltype.free(elem, flavor='raw') Modified: pypy/dist/pypy/module/_ffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_ffi/structure.py (original) +++ pypy/dist/pypy/module/_ffi/structure.py Wed Oct 17 13:16:09 2007 @@ -13,7 +13,7 @@ # the other one is in rlib/libffi, we should refactor it to reuse the same # logic, I'll not touch it by now, and refactor it later from pypy.module.struct.nativefmttable import native_fmttable as struct_native_fmttable -from pypy.module._ffi.interp_ffi import wrap_result, unwrap_arg +from pypy.module._ffi.interp_ffi import wrap_value, unwrap_value native_fmttable = {} for key, value in struct_native_fmttable.items(): @@ -81,7 +81,7 @@ for i in range(len(self.fields)): name, c = self.fields[i] if name == attr: - return wrap_result(space, c, self, i, cast_pos) + return wrap_value(space, cast_pos, self, i, c) raise OperationError(space.w_AttributeError, space.wrap( "C Structure has no attribute %s" % attr)) getattr.unwrap_spec = ['self', ObjSpace, str] @@ -90,7 +90,7 @@ for i in range(len(self.fields)): name, c = self.fields[i] if name == attr: - unwrap_arg(space, push_field, self, i, c, w_value, None) + unwrap_value(space, push_field, self, i, c, w_value, None) return setattr.unwrap_spec = ['self', ObjSpace, str, W_Root] From fijal at codespeak.net Wed Oct 17 13:16:32 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Oct 2007 13:16:32 +0200 (CEST) Subject: [pypy-svn] r47512 - pypy/dist/pypy/module/_ffi Message-ID: <20071017111632.56EAE80D1@code0.codespeak.net> Author: fijal Date: Wed Oct 17 13:16:31 2007 New Revision: 47512 Added: pypy/dist/pypy/module/_ffi/array.py (contents, props changed) Log: Ooops. Add array file Added: pypy/dist/pypy/module/_ffi/array.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/_ffi/array.py Wed Oct 17 13:16:31 2007 @@ -0,0 +1,70 @@ + +""" Interpreter-level implementation of array, exposing ll-structure +to app-level with apropriate interface +""" + +from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable,\ + Arguments +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.module._ffi.structure import native_fmttable +from pypy.module._ffi.interp_ffi import unwrap_value, wrap_value + +def push_elem(ll_array, pos, value): + TP = lltype.typeOf(value) + ll_array = rffi.cast(lltype.Ptr(rffi.CArray(TP)), ll_array) + ll_array[pos] = value +push_elem._annspecialcase_ = 'specialize:argtype(2)' + +def get_elem(ll_array, pos, ll_t): + ll_array = rffi.cast(lltype.Ptr(rffi.CArray(ll_t)), ll_array) + return ll_array[pos] +get_elem._annspecialcase_ = 'specialize:arg(2)' + +class W_ArrayInstance(Wrappable): + def __init__(self, space, of, length): + self.alloced = False + self.of = of + self.length = length + size = native_fmttable[of]['size'] * length + self.ll_array = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', + zero=True) + self.alloced = True + + # XXX don't allow negative indexes, nor slices + + def setitem(self, space, num, w_value): + if num >= self.length or num < 0: + raise OperationError(space.w_ValueError, space.wrap( + "Setting element %d of array sized %d" % (num, self.length))) + unwrap_value(space, push_elem, self.ll_array, num, self.of, w_value, + None) + setitem.unwrap_spec = ['self', ObjSpace, int, W_Root] + + def getitem(self, space, num): + if num >= self.length or num < 0: + raise OperationError(space.w_ValueError, space.wrap( + "Getting element %d of array sized %d" % (num, self.length))) + return wrap_value(space, get_elem, self.ll_array, num, self.of) + getitem.unwrap_spec = ['self', ObjSpace, int] + + def getbuffer(space, self): + return space.wrap(rffi.cast(rffi.INT, self.ll_array)) + + def __del__(self): + if self.alloced: + lltype.free(self.ll_array, flavor='raw') + +def descr_new_array_instance(space, w_type, of, size): + return space.wrap(W_ArrayInstance(space, of, size)) +descr_new_array_instance.unwrap_spec = [ObjSpace, W_Root, str, int] + +W_ArrayInstance.typedef = TypeDef( + 'ArrayInstance', + __new__ = interp2app(descr_new_array_instance), + __setitem__ = interp2app(W_ArrayInstance.setitem), + __getitem__ = interp2app(W_ArrayInstance.getitem), + buffer = GetSetProperty(W_ArrayInstance.getbuffer), +) From fijal at codespeak.net Wed Oct 17 13:43:20 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Oct 2007 13:43:20 +0200 (CEST) Subject: [pypy-svn] r47513 - pypy/dist/pypy/rlib Message-ID: <20071017114320.477E180D7@code0.codespeak.net> Author: fijal Date: Wed Oct 17 13:43:19 2007 New Revision: 47513 Modified: pypy/dist/pypy/rlib/libffi.py Log: Don't rely on obscure old-style classes details Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Wed Oct 17 13:43:19 2007 @@ -61,7 +61,8 @@ class cConfig: pass -cConfig.__dict__.update(rffi_platform.configure(CConfig)) +for k, v in rffi_platform.configure(CConfig).items(): + setattr(cConfig, k, v) FFI_TYPE_P.TO.become(cConfig.ffi_type) size_t = cConfig.size_t From fijal at codespeak.net Wed Oct 17 15:54:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Oct 2007 15:54:04 +0200 (CEST) Subject: [pypy-svn] r47514 - in pypy/dist/pypy/module: bz2 fcntl rctime Message-ID: <20071017135404.1A40A80EB@code0.codespeak.net> Author: fijal Date: Wed Oct 17 15:54:01 2007 New Revision: 47514 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/fcntl/interp_fcntl.py pypy/dist/pypy/module/rctime/interp_time.py Log: Kill dependency on old-style classes (very obscure though) Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Wed Oct 17 15:54:01 2007 @@ -44,9 +44,10 @@ for name in constant_names: setattr(CConfig, name, platform.DefinedConstantInteger(name)) -class cConfig: +class cConfig(object): pass -cConfig.__dict__.update(platform.configure(CConfig)) +for k, v in platform.configure(CConfig).items(): + setattr(cConfig, k, v) for name in constant_names: value = getattr(cConfig, name) Modified: pypy/dist/pypy/module/fcntl/interp_fcntl.py ============================================================================== --- pypy/dist/pypy/module/fcntl/interp_fcntl.py (original) +++ pypy/dist/pypy/module/fcntl/interp_fcntl.py Wed Oct 17 15:54:01 2007 @@ -31,10 +31,11 @@ for name in constant_names: setattr(CConfig, name, platform.DefinedConstantInteger(name)) -class cConfig: +class cConfig(object): pass -cConfig.__dict__.update(platform.configure(CConfig)) +for k, v in platform.configure(CConfig).items(): + setattr(cConfig, k, v) cConfig.flock.__name__ = "_flock" if "linux" in sys.platform: Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Wed Oct 17 15:54:01 2007 @@ -43,7 +43,9 @@ class cConfig: pass -cConfig.__dict__.update(platform.configure(CConfig)) + +for k, v in platform.configure(CConfig).items(): + setattr(cConfig, k, v) cConfig.tm.__name__ = "_tm" def external(name, args, result): From fijal at codespeak.net Wed Oct 17 16:13:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Oct 2007 16:13:27 +0200 (CEST) Subject: [pypy-svn] r47515 - pypy/dist/pypy/module/fcntl/test Message-ID: <20071017141327.6488380F2@code0.codespeak.net> Author: fijal Date: Wed Oct 17 16:13:25 2007 New Revision: 47515 Modified: pypy/dist/pypy/module/fcntl/test/test_fcntl.py Log: (exarkun) fix the test Modified: pypy/dist/pypy/module/fcntl/test/test_fcntl.py ============================================================================== --- pypy/dist/pypy/module/fcntl/test/test_fcntl.py (original) +++ pypy/dist/pypy/module/fcntl/test/test_fcntl.py Wed Oct 17 16:13:25 2007 @@ -97,8 +97,8 @@ if "linux" in sys.platform: # test managing signals assert fcntl.fcntl(f, fcntl.F_GETOWN) == 0 - fcntl.fcntl(f, fcntl.F_SETOWN, 20) - assert fcntl.fcntl(f, fcntl.F_GETOWN) == 20 + fcntl.fcntl(f, fcntl.F_SETOWN, os.getpid()) + assert fcntl.fcntl(f, fcntl.F_GETOWN) == os.getpid() assert fcntl.fcntl(f, fcntl.F_GETSIG) == 0 fcntl.fcntl(f, fcntl.F_SETSIG, 20) assert fcntl.fcntl(f, fcntl.F_GETSIG) == 20 From arigo at codespeak.net Wed Oct 17 17:19:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 17 Oct 2007 17:19:24 +0200 (CEST) Subject: [pypy-svn] r47516 - pypy/dist/pypy/module/zlib Message-ID: <20071017151924.92B9780F6@code0.codespeak.net> Author: arigo Date: Wed Oct 17 17:19:22 2007 New Revision: 47516 Modified: pypy/dist/pypy/module/zlib/interp_zlib.py Log: Increase the probability to be safe against __del__ being called while self.stream is still in use. Modified: pypy/dist/pypy/module/zlib/interp_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/interp_zlib.py (original) +++ pypy/dist/pypy/module/zlib/interp_zlib.py Wed Oct 17 17:19:22 2007 @@ -4,6 +4,7 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty from pypy.interpreter.error import OperationError from pypy.rlib.rarithmetic import intmask +from pypy.rlib.objectmodel import keepalive_until_here from pypy.rlib import rzlib @@ -95,19 +96,39 @@ decompress.unwrap_spec = [ObjSpace, str, int, int] -class Compress(Wrappable): +class ZLibObject(Wrappable): """ - Wrapper around zlib's z_stream structure which provides convenient - compression functionality. + Common base class for Compress and Decompress. """ stream = rzlib.null_stream + def __init__(self, space): + self.space = space + self._lock = space.allocate_lock() + + def lock(self): + """To call before using self.stream.""" + self._lock.acquire(True) + + def unlock(self): + """To call after using self.stream.""" + self._lock.release() + keepalive_until_here(self) + # subtle: we have to make sure that 'self' is not garbage-collected + # while we are still using 'self.stream' - hence the keepalive. + + +class Compress(ZLibObject): + """ + Wrapper around zlib's z_stream structure which provides convenient + compression functionality. + """ def __init__(self, space, level=rzlib.Z_DEFAULT_COMPRESSION, method=rzlib.Z_DEFLATED, # \ wbits=rzlib.MAX_WBITS, # \ undocumented memLevel=rzlib.DEF_MEM_LEVEL, # / parameters strategy=rzlib.Z_DEFAULT_STRATEGY): # / - self.space = space + ZLibObject.__init__(self, space) try: self.stream = rzlib.deflateInit(level, method, wbits, memLevel, strategy) @@ -116,7 +137,6 @@ except ValueError: raise OperationError(space.w_ValueError, space.wrap("Invalid initialization option")) - self.lock = space.allocate_lock() def __del__(self): """Automatically free the resources used by the stream.""" @@ -135,15 +155,14 @@ Call the flush() method to clear these buffers. """ try: - lock = self.lock - lock.acquire(True) + self.lock() try: if not self.stream: raise zlib_error(self.space, "compressor object already flushed") result = rzlib.compress(self.stream, data) finally: - lock.release() + self.unlock() except rzlib.RZlibError, e: raise zlib_error(self.space, e.msg) return self.space.wrap(result) @@ -163,20 +182,19 @@ compressed. """ try: - lock = self.lock - lock.acquire(True) + self.lock() try: if not self.stream: raise zlib_error(self.space, "compressor object already flushed") result = rzlib.compress(self.stream, '', mode) + if mode == rzlib.Z_FINISH: # release the data structures now + rzlib.deflateEnd(self.stream) + self.stream = rzlib.null_stream finally: - lock.release() + self.unlock() except rzlib.RZlibError, e: raise zlib_error(self.space, e.msg) - if mode == rzlib.Z_FINISH: # release the data structures now - rzlib.deflateEnd(self.stream) - self.stream = rzlib.null_stream return self.space.wrap(result) flush.unwrap_spec = ['self', int] @@ -208,13 +226,11 @@ """) -class Decompress(Wrappable): +class Decompress(ZLibObject): """ Wrapper around zlib's z_stream structure which provides convenient decompression functionality. """ - stream = rzlib.null_stream - def __init__(self, space, wbits=rzlib.MAX_WBITS): """ Initialize a new decompression object. @@ -224,7 +240,7 @@ and decompression. See the documentation for deflateInit2 and inflateInit2. """ - self.space = space + ZLibObject.__init__(self, space) self.unused_data = '' self.unconsumed_tail = '' try: @@ -234,7 +250,6 @@ except ValueError: raise OperationError(space.w_ValueError, space.wrap("Invalid initialization option")) - self.lock = space.allocate_lock() def __del__(self): """Automatically free the resources used by the stream.""" @@ -259,13 +274,12 @@ self.space.wrap("max_length must be " "greater than zero")) try: - lock = self.lock - lock.acquire(True) + self.lock() try: result = rzlib.decompress(self.stream, data, max_length = max_length) finally: - lock.release() + self.unlock() except rzlib.RZlibError, e: raise zlib_error(self.space, e.msg) From fijal at codespeak.net Wed Oct 17 18:33:03 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Oct 2007 18:33:03 +0200 (CEST) Subject: [pypy-svn] r47517 - in pypy/dist/pypy: module/mmap rlib rlib/test Message-ID: <20071017163303.463C580FD@code0.codespeak.net> Author: fijal Date: Wed Oct 17 18:33:02 2007 New Revision: 47517 Modified: pypy/dist/pypy/module/mmap/interp_mmap.py pypy/dist/pypy/rlib/rmmap.py pypy/dist/pypy/rlib/test/test_rmmap.py Log: * Move REnvironmentError -> OSError * A test for paranoid * Add implicit close Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Wed Oct 17 18:33:02 2007 @@ -1,11 +1,11 @@ from pypy.rpython.tool import rffi_platform from pypy.rpython.lltypesystem import rffi, lltype, llmemory -from pypy.interpreter.error import OperationError +from pypy.interpreter.error import OperationError, wrap_oserror from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app from pypy.rlib import rmmap -from pypy.rlib.rmmap import RValueError, RTypeError, REnvironmentError +from pypy.rlib.rmmap import RValueError, RTypeError import sys import os import platform @@ -212,8 +212,8 @@ try: return space.wrap(W_MMap(space, rmmap.mmap(fileno, length, flags, prot, access))) - except REnvironmentError, e: - raise OperationError(space.w_EnvironmentError, space.wrap(e.message)) + except OSError, e: + raise wrap_oserror(space, e, 'w_EnvironmentError') except RValueError, e: raise OperationError(space.w_ValueError, space.wrap(e.message)) except RTypeError, e: @@ -226,8 +226,8 @@ try: return space.wrap(W_MMap(space, rmmap.mmap(fileno, length, tagname, access))) - except REnvironmentError, e: - raise OperationError(space.w_EnvironmentError, space.wrap(e.message)) + except OSError, e: + raise wrap_oserror(space, e, 'w_EnvironmentError') except RValueError, e: raise OperationError(space.w_ValueError, space.wrap(e.message)) except RTypeError, e: Modified: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/rmmap.py (original) +++ pypy/dist/pypy/rlib/rmmap.py Wed Oct 17 18:33:02 2007 @@ -16,10 +16,6 @@ def __init__(self, message): self.message = message -class REnvironmentError(Exception): - def __init__(self, message): - self.message = message - class RTypeError(Exception): def __init__(self, message): self.message = message @@ -114,9 +110,8 @@ _get_page_size = external('getpagesize', [], rffi.INT) - def _get_error_msg(): - errno = rffi.get_errno() - return os.strerror(errno) + def _get_error_no(): + return rffi.get_errno() elif _MS_WINDOWS: @@ -207,14 +202,14 @@ dwErr = GetLastError() err = rffi.cast(lltype.Signed, dwErr) if low == INVALID_FILE_SIZE and err != NO_ERROR: - raise REnvironmentError(os.strerror(err)) + msg = os.strerror(err) + raise OSError(err, msg) return low, high finally: lltype.free(high_ref, flavor='raw') - def _get_error_msg(): - errno = rffi.cast(lltype.Signed, GetLastError()) - return os.strerror(errno) + def _get_error_no(): + return rffi.cast(lltype.Signed, GetLastError()) NULL_HANDLE = rffi.cast(HANDLE, 0) INVALID_HANDLE = rffi.cast(HANDLE, -1) @@ -280,6 +275,8 @@ c_munmap(self.getptr(0), self.size) self.setdata(NODATA, 0) + __del__ = close + def unmapview(self): UnmapViewOfFile(self.getptr(0)) @@ -439,7 +436,8 @@ ## new_size = size + value & (PAGESIZE - 1) res = c_msync(start, size, MS_SYNC) if res == -1: - raise REnvironmentError(_get_error_msg()) + errno = _get_error_no() + raise OSError(errno, os.strerror(errno)) return 0 @@ -464,14 +462,10 @@ if _POSIX: if not has_mremap: - msg = "mmap: resizing not available -- no mremap()" - raise REnvironmentError(msg) + raise OSError(-11111, "No mremap available") # resize the underlying file first - try: - os.ftruncate(self.fd, newsize) - except OSError, e: - raise REnvironmentError(os.strerror(e.errno)) + os.ftruncate(self.fd, newsize) # now resize the mmap newdata = c_mremap(self.getptr(0), self.size, newsize, @@ -519,7 +513,7 @@ else: dwErrCode = GetLastError() err = rffi.cast(lltype.Signed, dwErrCode) - raise REnvironmentError(os.strerror(err)) + raise OSError(err, os.strerror(err)) def len(self): self.check_valid() @@ -607,14 +601,12 @@ flags |= MAP_ANONYMOUS else: - try: - m.fd = os.dup(fd) - except OSError, e: - raise REnvironmentError(os.strerror(e.errno)) + m.fd = os.dup(fd) res = c_mmap(NULL, map_size, prot, flags, fd, 0) if res == rffi.cast(PTR, -1): - raise REnvironmentError(_get_error_msg()) + errno = _get_error_no() + raise OSError(errno, os.strerror(errno)) m.setdata(res, map_size) return m @@ -647,7 +639,8 @@ # parts of the C library use HANDLE, others just ints # XXX hack - made _get_osfhandle compatible if fh == INVALID_HANDLE: - raise REnvironmentError(_get_error_msg()) + errno = _get_error_no() + raise OSError(errno, os.strerror(errno)) # Win9x appears to need us seeked to zero # SEEK_SET = 0 # libc._lseek(fileno, 0, SEEK_SET) @@ -669,7 +662,8 @@ False, # inherited by child procs? DUPLICATE_SAME_ACCESS) # options if not res: - raise REnvironmentError(_get_error_msg()) + errno = _get_error_no() + raise OSError(errno, os.strerror(errno)) m.file_handle = handle_ref[0] finally: lltype.free(handle_ref, flavor='raw') @@ -712,7 +706,7 @@ else: dwErr = GetLastError() err = rffi.cast(lltype.Signed, dwErr) - raise REnvironmentError(os.strerror(err)) + raise OSError(err, os.strerror(err)) # register_external here? Modified: pypy/dist/pypy/rlib/test/test_rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rmmap.py (original) +++ pypy/dist/pypy/rlib/test/test_rmmap.py Wed Oct 17 18:33:02 2007 @@ -2,7 +2,7 @@ import os from pypy.rpython.test.test_llinterp import interpret from pypy.rlib import rmmap as mmap -from pypy.rlib.rmmap import RTypeError, RValueError, REnvironmentError +from pypy.rlib.rmmap import RTypeError, RValueError import sys class TestMMap: @@ -358,3 +358,16 @@ interpret(func, [f.fileno()]) f.close() + + def test_double_close(self): + f = open(self.tmpname + "s", "w+") + f.write("foobar") + f.flush() + + def func(no): + m = mmap.mmap(no, 6, access=mmap.ACCESS_WRITE) + m.close() + m.close() # didn't explode + + interpret(func, [f.fileno()]) + f.close() From fijal at codespeak.net Wed Oct 17 19:31:02 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Oct 2007 19:31:02 +0200 (CEST) Subject: [pypy-svn] r47518 - in pypy/dist/pypy/tool: . test Message-ID: <20071017173102.845F08109@code0.codespeak.net> Author: fijal Date: Wed Oct 17 19:31:01 2007 New Revision: 47518 Added: pypy/dist/pypy/tool/gcc_cache.py (contents, props changed) pypy/dist/pypy/tool/test/test_gcc_cache.py (contents, props changed) Log: Caching "mechanism" Added: pypy/dist/pypy/tool/gcc_cache.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/tool/gcc_cache.py Wed Oct 17 19:31:01 2007 @@ -0,0 +1,34 @@ + +from pypy.tool.autopath import pypydir +from pypy.translator.tool.cbuild import build_executable +import md5 +import py +import distutils + +cache_dir = py.path.local(pypydir).join('_cache', 'gcc') +cache_dir.ensure(dir=1) + +def build_executable_cache(c_files, *args, **kwds): + s = "\n\n".join([c_file.read() for c_file in c_files]) + hash = md5.md5(s).hexdigest() + try: + return cache_dir.join(hash).read() + except py.error.Error: + result = py.process.cmdexec(build_executable(c_files, *args, **kwds)) + cache_dir.join(hash).write(result) + return result + +def try_compile_cache(c_files, *args, **kwds): + s = "\n\n".join([c_file.read() for c_file in c_files]) + hash = md5.md5(s).hexdigest() + try: + return eval(cache_dir.join(hash).read()) + except py.error.Error: + try: + build_executable(c_files, *args, **kwds) + result = True + except (distutils.errors.CompileError, + distutils.errors.LinkError): + result = False + cache_dir.join(hash).write(repr(result)) + return bool(result) Added: pypy/dist/pypy/tool/test/test_gcc_cache.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/tool/test/test_gcc_cache.py Wed Oct 17 19:31:01 2007 @@ -0,0 +1,37 @@ + +from pypy.tool.gcc_cache import * +from pypy.tool.udir import udir +import md5 + +def test_gcc_exec(): + f = udir.join("x.c") + f.write(""" + #include + int main() + { + printf("3\\n"); + return 0; + } + """) + # remove cache + try: + cache_dir.join(md5.md5(f.read()).hexdigest()).remove() + except: + pass + assert build_executable_cache([f]) == "3\n" + assert build_executable_cache([f], compiler_exe="xxx") == "3\n" + +def test_gcc_ask(): + f = udir.join("y.c") + f.write(""" + int main() + { + return 0; + } + """) + try: + cache_dir.join(md5.md5(f.read()).hexdigest()).remove() + except: + pass + assert try_compile_cache([f]) + assert try_compile_cache([f], compiler_exe="xxx") From fijal at codespeak.net Wed Oct 17 19:33:56 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Oct 2007 19:33:56 +0200 (CEST) Subject: [pypy-svn] r47519 - pypy/dist/pypy/rpython/tool Message-ID: <20071017173356.A24A5810B@code0.codespeak.net> Author: fijal Date: Wed Oct 17 19:33:56 2007 New Revision: 47519 Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py Log: Use the caching mechanism Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Wed Oct 17 19:33:56 2007 @@ -4,7 +4,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import llmemory -from pypy.translator.tool.cbuild import build_executable +from pypy.tool.gcc_cache import build_executable_cache, try_compile_cache from pypy.tool.udir import udir import distutils @@ -114,8 +114,8 @@ self.close() include_dirs = getattr(self.config, '_include_dirs_', []) libraries = getattr(self.config, '_libraries_', []) - build_executable([self.path], include_dirs=include_dirs, - libraries=libraries) + return try_compile_cache([self.path], include_dirs=include_dirs, + libraries=libraries) def configure(CConfig): """Examine the local system by running the C compiler. @@ -409,12 +409,7 @@ self.name = name def question(self, ask_gcc): - try: - ask_gcc(self.name + ';') - return True - except (distutils.errors.CompileError, - distutils.errors.LinkError): - return False + return ask_gcc(self.name + ';') # ____________________________________________________________ # @@ -483,9 +478,8 @@ """ def run_example_code(filepath, include_dirs=[], libraries=[]): - executable = build_executable([filepath], include_dirs=include_dirs, - libraries=libraries) - output = py.process.cmdexec(executable) + output = build_executable_cache([filepath], include_dirs=include_dirs, + libraries=libraries) section = None for line in output.splitlines(): line = line.strip() From elmom at codespeak.net Wed Oct 17 21:44:11 2007 From: elmom at codespeak.net (elmom at codespeak.net) Date: Wed, 17 Oct 2007 21:44:11 +0200 (CEST) Subject: [pypy-svn] r47520 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20071017194411.9E7A88100@code0.codespeak.net> Author: elmom Date: Wed Oct 17 21:44:10 2007 New Revision: 47520 Modified: pypy/dist/pypy/objspace/std/listmultiobject.py pypy/dist/pypy/objspace/std/test/test_listmultiobject.py Log: * This commit adds framework to test that the different list implementations are really used. I was prompted to add this since for a big part, the implementations weren't really used, propably to such extent that speed figures are affected at least a bit. IOW, after the included fixes, the implementations are even slower :). * In this commit is also a little fix for a bug in the ChunkList impl. revealed by the added testing. * Next I'll commit tests for the BList implementation... Modified: pypy/dist/pypy/objspace/std/listmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/listmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/listmultiobject.py Wed Oct 17 21:44:10 2007 @@ -10,9 +10,9 @@ # ListImplementations -# An empty list always is an EmptyListImplementation. +# An empty list is always an EmptyListImplementation. # -# RDictImplementation -- standard implementation +# RListImplementation -- standard implementation # StrListImplementation -- lists consisting only of strings # ChunkedListImplementation -- when having set the withchunklist option # SmartResizableListImplementation -- when having set the @@ -87,7 +87,7 @@ for i in range(slicelength): res_w[i] = self.getitem(start) start += step - return RListImplementation(self.space, res_w) + return make_implementation(self.space, res_w) def delitem_slice_step(self, start, stop, step, slicelength): n = self.length() @@ -368,8 +368,8 @@ length = self._length self._grow() - for j in range(length - 1, 0, -1): - self.setitem(j + 1, self.getitem(j)) + for j in range(length, i, -1): + self.setitem(j, self.getitem(j-1)) self.setitem(i, w_item) return self @@ -399,13 +399,7 @@ class EmptyListImplementation(ListImplementation): def make_list_with_one_item(self, w_item): space = self.space - if space.config.objspace.std.withfastslice: - return SliceTrackingListImplementation(space, [w_item]) - w_type = space.type(w_item) - if space.is_w(w_type, space.w_str): - strlist = [space.str_w(w_item)] - return StrListImplementation(space, strlist) - return RListImplementation(space, [w_item]) + return make_implementation_with_one_item(space, w_item) def length(self): return 0 @@ -834,6 +828,23 @@ else: return RListImplementation(space, list_w) +def make_implementation_with_one_item(space, w_item): + if space.config.objspace.std.withfastslice: + return SliceTrackingListImplementation(space, [w_item]) + if space.config.objspace.std.withsmartresizablelist: + from pypy.objspace.std.smartresizablelist import \ + SmartResizableListImplementation + impl = SmartResizableListImplementation(space) + impl.append(w_item) + return impl + if space.config.objspace.std.withchunklist: + return ChunkedListImplementation(space, [w_item]) + w_type = space.type(w_item) + if space.is_w(w_type, space.w_str): + strlist = [space.str_w(w_item)] + return StrListImplementation(space, strlist) + return RListImplementation(space, [w_item]) + def convert_list_w(space, list_w): if not list_w: impl = space.fromcache(State).empty_impl @@ -884,7 +895,7 @@ if w_iterable is not EMPTY_LIST: list_w = space.unpackiterable(w_iterable) if list_w: - w_list.implementation = RListImplementation(space, list_w) + w_list.implementation = make_implementation(space, list_w) return w_list.implementation = space.fromcache(State).empty_impl @@ -1304,9 +1315,8 @@ sorterclass = CustomKeySort else: sorterclass = SimpleSort - impl = w_list.implementation.to_rlist() - w_list.implementation = impl - items = impl.list_w + impl=w_list.implementation + items = impl.get_list_w() sorter = sorterclass(items, impl.length()) sorter.space = space sorter.w_cmp = w_cmp @@ -1349,8 +1359,7 @@ mucked = w_list.implementation.length() > 0 # put the items back into the list - impl.list_w = sorter.list - w_list.implementation = impl + w_list.implementation = make_implementation(space, sorter.list) if mucked: raise OperationError(space.w_ValueError, Modified: pypy/dist/pypy/objspace/std/test/test_listmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_listmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_listmultiobject.py Wed Oct 17 21:44:10 2007 @@ -2,13 +2,51 @@ from pypy.objspace.std.listmultiobject import W_ListMultiObject, \ SliceTrackingListImplementation from pypy.conftest import gettestobjspace -from pypy.objspace.std.test import test_listobject +from pypy.objspace.std.test.test_listobject import AppTestW_ListObject from pypy.objspace.std.test.test_dictmultiobject import FakeSpace from pypy.objspace.std.test.test_rangeobject import AppTestRangeListObject -class AppTest_ListMultiObject(test_listobject.AppTestW_ListObject): +class BaseAppTest_ListMultiObject(AppTestW_ListObject): + def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withmultilist": True}) + self._setup_class(cls) + + @staticmethod + def _setup_class(cls, conf_switch='withmultilist', impl_tag=''): + cls.space = gettestobjspace(**{"objspace.std."+conf_switch: True}) + cls.w_impl_used = cls.space.appexec([cls.space.wrap(impl_tag)], + """(impl_tag): + import __pypy__ + def impl_used(obj, tag=''): + if not tag: + if impl_tag: + tag=impl_tag + else: + #import py.test + #py.test.skip('test not enabled (impl_tag not set)') + return True #XXX This hack because the above + # doesn't work... + return tag in __pypy__.internal_repr(obj) + return impl_used + """) + + def test_implementation_is_used(self): + l = ["1", "2", "3", "4", "5"] + assert self.impl_used(l) + l = list(["1", "2", "3", "4", "5"]) + assert self.impl_used(l) + l=[] + l.append('a') + assert self.impl_used(l) + l = ["6", "8", "3", "1", "5"] + l.sort() + assert self.impl_used(l) + # These few here ^ would fail before, but for good coverage, + # all the list methods etc. should be tested also... + +class AppTest_ListMultiObject(BaseAppTest_ListMultiObject): + def setup_class(cls): + BaseAppTest_ListMultiObject._setup_class(cls) def test_slice_with_step(self): l = range(20) @@ -17,9 +55,8 @@ assert l2 == range(1, 19, 2) def test_strlist_literal(self): - import __pypy__ l = ["1", "2", "3", "4", "5"] - assert "StrListImplementation" in __pypy__.internal_repr(l) + assert self.impl_used(l, "StrListImplementation") def test_strlist_delitem(self): l = ["1", "2"] @@ -27,14 +64,13 @@ assert l == ["2"] def test_strlist_append(self): - import __pypy__ l = [] l.append("a") - assert "StrListImplementation" in __pypy__.internal_repr(l) + assert self.impl_used(l, "StrListImplementation") l.extend(["b", "c", "d"]) l += ["e", "f"] assert l == ["a", "b", "c", "d", "e", "f"] - assert "StrListImplementation" in __pypy__.internal_repr(l) + assert self.impl_used(l, "StrListImplementation") class AppTestRangeImplementation(AppTestRangeListObject): @@ -52,39 +88,36 @@ pass # won't work with multilists -class AppTest_FastSlice(test_listobject.AppTestW_ListObject): +class AppTest_FastSlice(BaseAppTest_ListMultiObject): def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withfastslice": True}) + BaseAppTest_ListMultiObject._setup_class(cls, 'withfastslice') def test_lazy_slice(self): - import __pypy__ l = [i for i in range(100)] # force it to not be a range impl l2 = l[1:-1] - assert "SliceTrackingListImplementation" in __pypy__.internal_repr(l) - assert "SliceListImplementation" in __pypy__.internal_repr(l2) + assert self.impl_used(l, "SliceTrackingListImplementation") + assert self.impl_used(l2, "SliceListImplementation") result = 0 for i in l2: result += i # didn't force l2 - assert "SliceListImplementation" in __pypy__.internal_repr(l2) + assert self.impl_used(l2, "SliceListImplementation") # force l2: l2.append(10) assert l2 == range(1, 99) + [10] def test_append_extend_dont_force(self): - import __pypy__ l = [i for i in range(100)] # force it to not be a range impl l2 = l[1:-1] - assert "SliceTrackingListImplementation" in __pypy__.internal_repr(l) - assert "SliceListImplementation" in __pypy__.internal_repr(l2) + assert self.impl_used(l, "SliceTrackingListImplementation") + assert self.impl_used(l2, "SliceListImplementation") l.append(100) l.extend(range(101, 110)) assert l == range(110) - assert "SliceTrackingListImplementation" in __pypy__.internal_repr(l) - assert "SliceListImplementation" in __pypy__.internal_repr(l2) + assert self.impl_used(l, "SliceTrackingListImplementation") + assert self.impl_used(l2, "SliceListImplementation") def test_slice_of_slice(self): - import __pypy__ l = [i for i in range(100)] # force it to not be a range impl l2 = l[1:-1] l3 = l2[1:-1] @@ -92,17 +125,17 @@ assert l2 == range(1, 99) assert l3 == range(2, 98) assert l4 == range(3, 97) - assert "SliceListImplementation" in __pypy__.internal_repr(l4) + assert self.impl_used(l4, "SliceListImplementation") l2[3] = 4 - assert "SliceListImplementation" not in __pypy__.internal_repr(l2) - assert "SliceListImplementation" in __pypy__.internal_repr(l4) + assert not self.impl_used(l2, "SliceListImplementation") + assert self.impl_used(l4, "SliceListImplementation") def test_delitem_to_empty(self): import __pypy__ l = [i for i in range(100)] # force it to not be a range impl l1 = l[1:-1] del l1[:] - assert "EmptyListImplementation" in __pypy__.internal_repr(l1) + assert self.impl_used(l1, "EmptyListImplementation") class TestSliceListImplementation(object): def setup_method(self,method): @@ -119,10 +152,10 @@ impl2 = impl2.setitem(2, 5) assert impl2.getitem(2) == 5 -class AppTest_SmartListObject(test_listobject.AppTestW_ListObject): +class AppTest_SmartListObject(BaseAppTest_ListMultiObject): def setup_class(cls): - cls.space = gettestobjspace(**{ - "objspace.std.withsmartresizablelist": True}) + BaseAppTest_ListMultiObject._setup_class(cls, 'withsmartresizablelist', + 'SmartResizableList') def _set_chunk_size_bits(bits): @@ -135,10 +168,11 @@ return old_value return -1 -class AppTest_ChunkListObject(test_listobject.AppTestW_ListObject): +class AppTest_ChunkListObject(BaseAppTest_ListMultiObject): def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withchunklist": True}) + BaseAppTest_ListMultiObject._setup_class(cls, 'withchunklist', + 'ChunkedList') cls.chunk_size_bits = _set_chunk_size_bits(2) def teardown_class(cls): From arigo at codespeak.net Wed Oct 17 22:01:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 17 Oct 2007 22:01:32 +0200 (CEST) Subject: [pypy-svn] r47521 - in pypy/dist/pypy: rpython rpython/lltypesystem rpython/test translator/c/src translator/cli translator/jvm translator/oosupport/test_template Message-ID: <20071017200132.44C6780FB@code0.codespeak.net> Author: arigo Date: Wed Oct 17 22:01:30 2007 New Revision: 47521 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/rint.py pypy/dist/pypy/rpython/test/test_rint.py pypy/dist/pypy/translator/c/src/int.h pypy/dist/pypy/translator/cli/opcodes.py pypy/dist/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/oosupport/test_template/overflow.py Log: Experimental. An operation 'int_add_nonneg_ovf' which is like 'int_add_ovf' but can assume that the 2nd argument is nonneg. In genc we can test that much more simply, and somehow I expect this case to be relatively common in the ll helpers for RPython data structures. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Wed Oct 17 22:01:30 2007 @@ -969,6 +969,11 @@ _makefunc2('op_ullong_lshift_val', '<<', 'r_ulonglong') _makefunc2('op_ullong_rshift_val', '>>', 'r_ulonglong') + def op_int_add_nonneg_ovf(self, x, y): + if isinstance(y, int): + assert y >= 0 + return self.op_int_add_ovf(x, y) + def op_cast_float_to_int(self, f): assert type(f) is float try: Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Wed Oct 17 22:01:30 2007 @@ -186,6 +186,8 @@ 'int_xor': LLOp(canfold=True), 'int_add_ovf': LLOp(canraise=(OverflowError,), tryfold=True), + 'int_add_nonneg_ovf': LLOp(canraise=(OverflowError,), tryfold=True), + # ^^^ more efficient version when 2nd arg is nonneg 'int_sub_ovf': LLOp(canraise=(OverflowError,), tryfold=True), 'int_mul_ovf': LLOp(canraise=(OverflowError,), tryfold=True), 'int_floordiv_ovf': LLOp(canraise=(OverflowError,), tryfold=True), Modified: pypy/dist/pypy/rpython/rint.py ============================================================================== --- pypy/dist/pypy/rpython/rint.py (original) +++ pypy/dist/pypy/rpython/rint.py Wed Oct 17 22:01:30 2007 @@ -57,7 +57,15 @@ rtype_inplace_add = rtype_add def rtype_add_ovf(_, hop): - return _rtype_template(hop, 'add_ovf') + func = 'add_ovf' + if hop.r_result.opprefix == 'int_': + if hop.args_s[1].nonneg: + func = 'add_nonneg_ovf' + elif hop.args_s[0].nonneg: + hop = hop.copy() + hop.swap_fst_snd_args() + func = 'add_nonneg_ovf' + return _rtype_template(hop, func) def rtype_sub(_, hop): return _rtype_template(hop, 'sub') @@ -183,7 +191,7 @@ appendix = op_appendices[implicit_exc] func += '_' + appendix - r_result = hop.rtyper.makerepr(hop.s_result) + r_result = hop.r_result if r_result.lowleveltype == Bool: repr = signed_repr else: Modified: pypy/dist/pypy/rpython/test/test_rint.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rint.py (original) +++ pypy/dist/pypy/rpython/test/test_rint.py Wed Oct 17 22:01:30 2007 @@ -309,6 +309,24 @@ res2 = self.interpret(func, [x, y]) assert res1 == res2 + def test_int_add_nonneg_ovf(self): + def f(x): + try: + a = ovfcheck(x + 50) + except OverflowError: + return 0 + try: + a += ovfcheck(100 + x) + except OverflowError: + return 1 + return a + res = self.interpret(f, [-3]) + assert res == 144 + res = self.interpret(f, [sys.maxint-50]) + assert res == 1 + res = self.interpret(f, [sys.maxint]) + assert res == 0 + class TestLLtype(BaseTestRint, LLRtypeMixin): pass Modified: pypy/dist/pypy/translator/c/src/int.h ============================================================================== --- pypy/dist/pypy/translator/c/src/int.h (original) +++ pypy/dist/pypy/translator/c/src/int.h Wed Oct 17 22:01:30 2007 @@ -56,6 +56,11 @@ if ((r^(x)) >= 0 || (r^(y)) >= 0); \ else FAIL_OVF("integer addition") +#define OP_INT_ADD_NONNEG_OVF(x,y,r) /* y can be assumed >= 0 */ \ + OP_INT_ADD(x,y,r); \ + if (r >= (x)); \ + else FAIL_OVF("integer addition") + #define OP_INT_SUB(x,y,r) r = (x) - (y) #define OP_INT_SUB_OVF(x,y,r) \ Modified: pypy/dist/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/cli/opcodes.py (original) +++ pypy/dist/pypy/translator/cli/opcodes.py Wed Oct 17 22:01:30 2007 @@ -104,6 +104,7 @@ 'int_rshift': 'shr', 'int_xor': 'xor', 'int_add_ovf': _check_ovf('add.ovf'), + 'int_add_nonneg_ovf': _check_ovf('add.ovf'), 'int_sub_ovf': _check_ovf('sub.ovf'), 'int_mul_ovf': _check_ovf('mul.ovf'), 'int_floordiv_ovf': 'div', # these can't overflow! Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Wed Oct 17 22:01:30 2007 @@ -111,6 +111,7 @@ 'int_rshift': jvmgen.ISHR, 'int_xor': jvmgen.IXOR, 'int_add_ovf': jvmgen.IADDOVF, + 'int_add_nonneg_ovf': jvmgen.IADDOVF, 'int_sub_ovf': jvmgen.ISUBOVF, 'int_mul_ovf': jvmgen.IMULOVF, 'int_floordiv_ovf': jvmgen.IDIV, # these can't overflow! Modified: pypy/dist/pypy/translator/oosupport/test_template/overflow.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/overflow.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/overflow.py Wed Oct 17 22:01:30 2007 @@ -16,6 +16,14 @@ return 42 self.check(fn, [sys.maxint, 1]) + def test_add2(self): + def fn(x): + try: + return ovfcheck(x+1) # special 'int_add_nonneg_ovf' operation + except OverflowError: + return 42 + self.check(fn, [sys.maxint]) + def test_sub(self): def fn(x, y): try: From elmom at codespeak.net Wed Oct 17 23:50:43 2007 From: elmom at codespeak.net (elmom at codespeak.net) Date: Wed, 17 Oct 2007 23:50:43 +0200 (CEST) Subject: [pypy-svn] r47522 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20071017215043.386C4810B@code0.codespeak.net> Author: elmom Date: Wed Oct 17 23:50:43 2007 New Revision: 47522 Added: pypy/dist/pypy/objspace/std/test/test_blistimplementation.py (contents, props changed) Modified: pypy/dist/pypy/objspace/std/listmultiobject.py pypy/dist/pypy/objspace/std/test/test_listmultiobject.py Log: Tests for the coming BList and related minor fixes Modified: pypy/dist/pypy/objspace/std/listmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/listmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/listmultiobject.py Wed Oct 17 23:50:43 2007 @@ -816,6 +816,9 @@ return impl if space.config.objspace.std.withchunklist: return ChunkedListImplementation(space, list_w) + if space.config.objspace.std.withblist: + from pypy.objspace.std.blistimplementation import BListImplementation + return BListImplementation(space, list_w) elif space.config.objspace.std.withfastslice: return SliceTrackingListImplementation(space, list_w) else: @@ -829,21 +832,24 @@ return RListImplementation(space, list_w) def make_implementation_with_one_item(space, w_item): - if space.config.objspace.std.withfastslice: - return SliceTrackingListImplementation(space, [w_item]) - if space.config.objspace.std.withsmartresizablelist: - from pypy.objspace.std.smartresizablelist import \ - SmartResizableListImplementation - impl = SmartResizableListImplementation(space) - impl.append(w_item) - return impl - if space.config.objspace.std.withchunklist: - return ChunkedListImplementation(space, [w_item]) - w_type = space.type(w_item) - if space.is_w(w_type, space.w_str): - strlist = [space.str_w(w_item)] - return StrListImplementation(space, strlist) - return RListImplementation(space, [w_item]) + if space.config.objspace.std.withfastslice: + return SliceTrackingListImplementation(space, [w_item]) + if space.config.objspace.std.withsmartresizablelist: + from pypy.objspace.std.smartresizablelist import \ + SmartResizableListImplementation + impl = SmartResizableListImplementation(space) + impl.append(w_item) + return impl + if space.config.objspace.std.withchunklist: + return ChunkedListImplementation(space, [w_item]) + if space.config.objspace.std.withblist: + from pypy.objspace.std.blistimplementation import BListImplementation + return BListImplementation(space, [w_item]) + w_type = space.type(w_item) + if space.is_w(w_type, space.w_str): + strlist = [space.str_w(w_item)] + return StrListImplementation(space, strlist) + return RListImplementation(space, [w_item]) def convert_list_w(space, list_w): if not list_w: Added: pypy/dist/pypy/objspace/std/test/test_blistimplementation.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/std/test/test_blistimplementation.py Wed Oct 17 23:50:43 2007 @@ -0,0 +1,217 @@ +from pypy.objspace.std.test.test_dictmultiobject import FakeSpace +from pypy.objspace.std.test.test_listmultiobject import \ + BaseAppTest_ListMultiObject +from pypy.objspace.std.blistimplementation import BListImplementation + +import py.test +py.test.skip('Not implemented yet') + +## Most of these tests are from the reference implementation + +def _set_get_limit(limit=None): + from pypy.conftest import option + from pypy.objspace.std import blistimplementation + old_value = blistimplementation.limit + if limit: + if not option.runappdirect: + blistimplementation.limit=limit + cur_value=limit + else: + cur_value=old_value + else: + cur_value=old_value + + return old_value, cur_value + +class AppTest_BListObject(BaseAppTest_ListMultiObject): + def setup_class(cls): + BaseAppTest_ListMultiObject.setup_class(cls, 'withblist', 'BListImpl') + cls.old_limit, cls.limit = _set_get_limit(8) + cls.w_limit=cls.space.wrap(cls.limit) + + def teardown_class(cls): + _set_get_limit(cls.old_limit) + + def test_very_long_list(self): + little_list = [0] +# big_list = little_list * 2**512 #XXX App-level OverflowError + big_list = little_list * 2**30 + + def test_very_long_list_insert(self): + little_list = [0] + big_list = little_list * (2**30) + big_list = big_list * 2 + big_list.insert((2**31)-5, 1) + assert big_list[-10:]==[0,0,0,0,1,0,0,0,0,0] + + def test_blist_insert(self): + l1=list([2,4,6]) + l2=[] + l2.extend([2,4,6]) + l3=[2,4,6] + l4=[] + for i in (2,4,6): + l4.append(i) + l5=[] + l5.extend((2,4,6)) + l1.insert(2,'x') + l2.insert(2,'x') + l3.insert(2,'x') + l4.insert(2,'x') + l5.insert(2,'x') + + assert l2==[2,4,'x',6] + assert l5==l4==l3==l2==l1 + + del l1[:4] + assert l1==[] + + def test_from_reference_impl1(self): + limit=self.limit + n = 512 + + data = range(n) + import random + random.shuffle(data) + x = list(data) + x.sort() + + assert x == sorted(data), x + assert x[3] == 3 + assert x[100] == 100 + assert x[500] == 500 + + def test_from_reference_impl2(self): + limit=self.limit + n = 512 + + lst = [] + t = tuple(range(n)) + for i in range(n): + lst.append(i) + if tuple(lst) != t[:i+1]: + print i, tuple(lst), t[:i+1] + print lst.debug() + break + + x = lst[4:258] + assert tuple(x) == tuple(t[4:258]) + x.append(-1) + assert tuple(x) == tuple(t[4:258] + (-1,)) + assert tuple(lst) == t + + lst[200] = 6 + assert tuple(x) == tuple(t[4:258] + (-1,)) + assert tuple(lst) == tuple(t[0:200] + (6,) + t[201:]) + + del lst[200] + #print lst.debug() + assert tuple(lst) == tuple(t[0:200] + t[201:]) + + lst = list(range(n)) + lst.insert(200, 0) + assert tuple(lst) == (t[0:200] + (0,) + t[200:]) + del lst[200:] + assert tuple(lst) == tuple(range(200)) + + def test_from_reference_impl3(self): + limit=self.limit + n = 512 + + lst2 = list(range(limit+1)) + assert tuple(lst2) == tuple(range(limit+1)) + del lst2[1] + del lst2[-1] + assert tuple(lst2) == (0,) + tuple(range(2,limit)) +# assert lst2.leaf +# assert len(lst2.children) == limit-1 + + def test_from_reference_impl4(self): + limit=self.limit + n = 512 + + lst = [i for i in range(3)] + lst*3 + assert lst*3 == range(3)*3 + + a = [i for i in 'spam'] + a.extend('eggs') + assert a == list('spameggs') + + x = [0] + for i in range(290) + [1000, 10000, 100000, 1000000, 10000000]: + if len(x*i) != i: + print 'mul failure', i + print (x*i).debug() + break + +def test_interp_blist(): + old_limit, limit = _set_get_limit(8) + def BList(other=[]): + return BListImplementation(FakeSpace(), other) + + n = 512 + + data = range(n) + import random + random.shuffle(data) + x = BList(data) + #x.sort() # Not used... + + #assert list(x) == sorted(data), x + + lst = BList() + t = tuple(range(n)) + for i in range(n): + lst.append(i) + if tuple(lst) != t[:i+1]: + print i, tuple(lst), t[:i+1] + print lst.debug() + break + + x = lst[4:258] + assert tuple(x) == tuple(t[4:258]) + x.append(-1) + assert tuple(x) == tuple(t[4:258] + (-1,)) + assert tuple(lst) == t + + lst[200] = 6 + assert tuple(x) == tuple(t[4:258] + (-1,)) + assert tuple(lst) == tuple(t[0:200] + (6,) + t[201:]) + + del lst[200] + #print lst.debug() + assert tuple(lst) == tuple(t[0:200] + t[201:]) + + lst2 = BList(range(limit+1)) + assert tuple(lst2) == tuple(range(limit+1)) + del lst2[1] + del lst2[-1] + assert tuple(lst2) == (0,) + tuple(range(2,limit)) + assert lst2.leaf + assert len(lst2.children) == limit-1 + + lst = BList(range(n)) + lst.insert(200, 0) + assert tuple(lst) == (t[0:200] + (0,) + t[200:]) + del lst[200:] + assert tuple(lst) == tuple(range(200)) + + lst = BList(range(3)) + lst*3 + assert lst*3 == range(3)*3 + + a = BList('spam') + a.extend('eggs') + assert a == list('spameggs') + + x = BList([0]) + for i in range(290) + [1000, 10000, 100000, 1000000, 10000000]: + if len(x*i) != i: + print 'mul failure', i + print (x*i).debug() + break + + little_list = BList([0]) + #big_list = little_list * 2**512 + big_list = little_list * 2**30 Modified: pypy/dist/pypy/objspace/std/test/test_listmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_listmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_listmultiobject.py Wed Oct 17 23:50:43 2007 @@ -7,12 +7,8 @@ from pypy.objspace.std.test.test_rangeobject import AppTestRangeListObject class BaseAppTest_ListMultiObject(AppTestW_ListObject): - - def setup_class(cls): - self._setup_class(cls) - @staticmethod - def _setup_class(cls, conf_switch='withmultilist', impl_tag=''): + def setup_class(cls, conf_switch='withmultilist', impl_tag=''): cls.space = gettestobjspace(**{"objspace.std."+conf_switch: True}) cls.w_impl_used = cls.space.appexec([cls.space.wrap(impl_tag)], """(impl_tag): @@ -22,10 +18,7 @@ if impl_tag: tag=impl_tag else: - #import py.test - #py.test.skip('test not enabled (impl_tag not set)') - return True #XXX This hack because the above - # doesn't work... + skip('test not enabled (impl_tag not set)') return tag in __pypy__.internal_repr(obj) return impl_used """) @@ -41,12 +34,14 @@ l = ["6", "8", "3", "1", "5"] l.sort() assert self.impl_used(l) - # These few here ^ would fail before, but for good coverage, + assert self.impl_used([0]) + assert self.impl_used(list([0])) + # These few here ^ would have failed before, but for good coverage, # all the list methods etc. should be tested also... class AppTest_ListMultiObject(BaseAppTest_ListMultiObject): def setup_class(cls): - BaseAppTest_ListMultiObject._setup_class(cls) + BaseAppTest_ListMultiObject.setup_class(cls) def test_slice_with_step(self): l = range(20) @@ -90,7 +85,7 @@ class AppTest_FastSlice(BaseAppTest_ListMultiObject): def setup_class(cls): - BaseAppTest_ListMultiObject._setup_class(cls, 'withfastslice') + BaseAppTest_ListMultiObject.setup_class(cls, 'withfastslice') def test_lazy_slice(self): l = [i for i in range(100)] # force it to not be a range impl @@ -154,7 +149,7 @@ class AppTest_SmartListObject(BaseAppTest_ListMultiObject): def setup_class(cls): - BaseAppTest_ListMultiObject._setup_class(cls, 'withsmartresizablelist', + BaseAppTest_ListMultiObject.setup_class(cls, 'withsmartresizablelist', 'SmartResizableList') @@ -171,7 +166,7 @@ class AppTest_ChunkListObject(BaseAppTest_ListMultiObject): def setup_class(cls): - BaseAppTest_ListMultiObject._setup_class(cls, 'withchunklist', + BaseAppTest_ListMultiObject.setup_class(cls, 'withchunklist', 'ChunkedList') cls.chunk_size_bits = _set_chunk_size_bits(2) From elmom at codespeak.net Thu Oct 18 01:14:57 2007 From: elmom at codespeak.net (elmom at codespeak.net) Date: Thu, 18 Oct 2007 01:14:57 +0200 (CEST) Subject: [pypy-svn] r47524 - pypy/dist/pypy/objspace/std/test Message-ID: <20071017231457.80D608112@code0.codespeak.net> Author: elmom Date: Thu Oct 18 01:14:55 2007 New Revision: 47524 Modified: pypy/dist/pypy/objspace/std/test/test_blistimplementation.py Log: Skip properly and add a test Modified: pypy/dist/pypy/objspace/std/test/test_blistimplementation.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_blistimplementation.py (original) +++ pypy/dist/pypy/objspace/std/test/test_blistimplementation.py Thu Oct 18 01:14:55 2007 @@ -1,11 +1,12 @@ from pypy.objspace.std.test.test_dictmultiobject import FakeSpace from pypy.objspace.std.test.test_listmultiobject import \ BaseAppTest_ListMultiObject -from pypy.objspace.std.blistimplementation import BListImplementation import py.test py.test.skip('Not implemented yet') +from pypy.objspace.std.blistimplementation import BListImplementation + ## Most of these tests are from the reference implementation def _set_get_limit(limit=None): @@ -36,6 +37,12 @@ little_list = [0] # big_list = little_list * 2**512 #XXX App-level OverflowError big_list = little_list * 2**30 + + def test_very_very_long_list(self): + big_list = [0] * 2**30 + for i in range(20): + big_list = big_list * 2**30 + assert len(big_list) > 2**512 def test_very_long_list_insert(self): little_list = [0] From arigo at codespeak.net Thu Oct 18 09:30:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 18 Oct 2007 09:30:05 +0200 (CEST) Subject: [pypy-svn] r47525 - in pypy/dist/pypy/rpython/rctypes: . test Message-ID: <20071018073005.B8AB38091@code0.codespeak.net> Author: arigo Date: Thu Oct 18 09:30:04 2007 New Revision: 47525 Modified: pypy/dist/pypy/rpython/rctypes/rpointer.py pypy/dist/pypy/rpython/rctypes/test/test_rstringbuf.py Log: Fix this test, broken by the fact that we no longer ensure that all variables stay alive until the end of the current block. Add some (unrelated) missing keepalive logic in rpointer. Modified: pypy/dist/pypy/rpython/rctypes/rpointer.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rpointer.py (original) +++ pypy/dist/pypy/rpython/rctypes/rpointer.py Thu Oct 18 09:30:04 2007 @@ -57,8 +57,10 @@ assert s_attr.const == 'contents' v_ptr = hop.inputarg(self, 0) v_c_ptr = self.getvalue(hop.llops, v_ptr) + v_owner = self.getkeepalive(hop.llops, v_ptr) hop.exception_cannot_occur() - return self.r_contents.allocate_instance_ref(hop.llops, v_c_ptr) + return self.r_contents.allocate_instance_ref(hop.llops, v_c_ptr, + v_owner) def rtype_setattr(self, hop): s_attr = hop.args_s[1] Modified: pypy/dist/pypy/rpython/rctypes/test/test_rstringbuf.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_rstringbuf.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_rstringbuf.py Thu Oct 18 09:30:04 2007 @@ -16,6 +16,7 @@ from ctypes import c_char_p, c_long, pointer, sizeof, c_int from pypy.rpython.rctypes.astringbuf import StringBufferType from pypy.rlib.rarithmetic import r_uint +from pypy.rlib.objectmodel import keepalive_until_here class Test_annotation: @@ -221,7 +222,9 @@ for i in range(sizeof(c_long)): buf[i] = c_n_ptr[i] c_long_ptr = cast(buf, POINTER(c_long)) - return c_long_ptr.contents.value + res = c_long_ptr.contents.value + keepalive_until_here(buf) + return res fn = compile(func, [int]) res = fn(0x12345678) assert res == 0x12345678 From arigo at codespeak.net Thu Oct 18 09:37:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 18 Oct 2007 09:37:26 +0200 (CEST) Subject: [pypy-svn] r47526 - pypy/dist/pypy/translator/c/test Message-ID: <20071018073726.F2624810E@code0.codespeak.net> Author: arigo Date: Thu Oct 18 09:37:26 2007 New Revision: 47526 Modified: pypy/dist/pypy/translator/c/test/test_newgc.py Log: Fix this test: id(x) is unrelated to the address of x in moving GCs. Use rffi.cast() instead to cast the object pointer to an int directly. Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Thu Oct 18 09:37:26 2007 @@ -780,6 +780,7 @@ def test_object_alignment(self): # all objects returned by the GC should be aligned on a 8-bytes # boundary, or whatever sizeof(double) is on this platform + from pypy.rpython.lltypesystem import rffi mylist = ['a', 'bc', '84139871', 'ajkdh', '876'] def f(): result = 0 @@ -787,7 +788,8 @@ for j in range(100): for s in mylist: buffer += s - result |= id(buffer) + addr = rffi.cast(lltype.Signed, buffer) + result |= addr return result fn = self.getcompiled(f) From arigo at codespeak.net Thu Oct 18 09:42:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 18 Oct 2007 09:42:31 +0200 (CEST) Subject: [pypy-svn] r47527 - in pypy/dist/pypy/translator: cli/test jvm/test Message-ID: <20071018074231.7DADE810F@code0.codespeak.net> Author: arigo Date: Thu Oct 18 09:42:31 2007 New Revision: 47527 Modified: pypy/dist/pypy/translator/cli/test/test_builtin.py pypy/dist/pypy/translator/jvm/test/test_builtin.py Log: Skip failures that are not relevant so far. Modified: pypy/dist/pypy/translator/cli/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/cli/test/test_builtin.py Thu Oct 18 09:42:31 2007 @@ -20,6 +20,9 @@ self._skip_powerpc("Mono math floating point problem") BaseTestBuiltin.test_builtin_math_frexp(self) + def test_debug_llinterpcall(self): + py.test.skip("so far, debug_llinterpcall is only used on lltypesystem") + class TestCliTime(CliTest, BaseTestTime): pass Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Thu Oct 18 09:42:31 2007 @@ -29,6 +29,8 @@ def test_os_read_binary_crlf(self): py.test.skip('fixme!') + def test_debug_llinterpcall(self): + py.test.skip("so far, debug_llinterpcall is only used on lltypesystem") class TestJvmTime(JvmTest, BaseTestTime): From fijal at codespeak.net Thu Oct 18 09:50:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 09:50:41 +0200 (CEST) Subject: [pypy-svn] r47528 - pypy/dist/pypy/interpreter Message-ID: <20071018075041.0FE42810B@code0.codespeak.net> Author: fijal Date: Thu Oct 18 09:50:40 2007 New Revision: 47528 Modified: pypy/dist/pypy/interpreter/error.py Log: Add custom exception possibility to wrap_oserror Modified: pypy/dist/pypy/interpreter/error.py ============================================================================== --- pypy/dist/pypy/interpreter/error.py (original) +++ pypy/dist/pypy/interpreter/error.py Thu Oct 18 09:50:40 2007 @@ -222,18 +222,19 @@ # 31: ANSI color code "red" ansi_print(text, esc="31", file=file, newline=newline) -def wrap_oserror(space, e): +def wrap_oserror(space, e, exception_name='w_OSError'): assert isinstance(e, OSError) errno = e.errno try: msg = os.strerror(errno) except ValueError: msg = 'error %d' % errno - w_error = space.call_function(space.w_OSError, + exc = getattr(space, exception_name) + w_error = space.call_function(exc, space.wrap(errno), space.wrap(msg)) - return OperationError(space.w_OSError, w_error) - + return OperationError(exc, w_error) +wrap_oserror._annspecialcase_ = 'specialize:arg(2)' ### installing the excepthook for OperationErrors ##def operr_excepthook(exctype, value, traceback): From fijal at codespeak.net Thu Oct 18 10:02:56 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 10:02:56 +0200 (CEST) Subject: [pypy-svn] r47529 - pypy/dist/pypy/rlib Message-ID: <20071018080256.C6DED8117@code0.codespeak.net> Author: fijal Date: Thu Oct 18 10:02:56 2007 New Revision: 47529 Modified: pypy/dist/pypy/rlib/libffi.py Log: Be paranoid when freeing in __del__ Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Thu Oct 18 10:02:56 2007 @@ -160,6 +160,12 @@ class FuncPtr(object): def __init__(self, name, argtypes, restype, funcsym): + # initialize each one of pointers with null + TP = rffi.CArray(rffi.VOIDP) + self.ll_args = lltype.nullptr(TP) + self.ll_cif = lltype.nullptr(FFI_CIFP.TO) + self.ll_argtypes = lltype.nullptr(FFI_TYPE_PP.TO) + self.ll_result = lltype.nullptr(rffi.VOIDP.TO) self.name = name self.argtypes = argtypes self.restype = restype @@ -167,7 +173,6 @@ argnum = len(argtypes) self.argnum = argnum self.pushed_args = 0 - TP = rffi.CArray(rffi.VOIDP) self.ll_args = lltype.malloc(TP, argnum, flavor='raw') self.ll_cif = lltype.malloc(FFI_CIFP.TO, flavor='raw') self.ll_argtypes = lltype.malloc(FFI_TYPE_PP.TO, argnum, flavor='raw') @@ -232,23 +237,31 @@ call._annspecialcase_ = 'specialize:arg(1)' def __del__(self): - argnum = len(self.argtypes) - for i in range(argnum): - lltype.free(self.ll_args[i], flavor='raw') - lltype.free(self.ll_args, flavor='raw') - if self.restype != ffi_type_void: + if self.ll_args: + argnum = len(self.argtypes) + for i in range(argnum): + if self.ll_args[i]: + lltype.free(self.ll_args[i], flavor='raw') + lltype.free(self.ll_args, flavor='raw') + if self.ll_result: lltype.free(self.ll_result, flavor='raw') - lltype.free(self.ll_cif, flavor='raw') - lltype.free(self.ll_argtypes, flavor='raw') + if self.ll_cif: + lltype.free(self.ll_cif, flavor='raw') + if self.ll_argtypes: + lltype.free(self.ll_argtypes, flavor='raw') class CDLL: def __init__(self, libname): + self.ll_libname = lltype.nullptr(rffi.CCHARP.TO) + self.lib = lltype.nullptr(rffi.CCHARP.TO) self.ll_libname = rffi.str2charp(libname) self.lib = dlopen(self.ll_libname) def __del__(self): - c_dlclose(self.lib) - lltype.free(self.ll_libname, flavor='raw') + if self.lib: + c_dlclose(self.lib) + if self.ll_libname: + lltype.free(self.ll_libname, flavor='raw') def getpointer(self, name, argtypes, restype): # these arguments are already casted to proper ffi From fijal at codespeak.net Thu Oct 18 11:05:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 11:05:27 +0200 (CEST) Subject: [pypy-svn] r47530 - pypy/dist/pypy/translator/c/src Message-ID: <20071018090527.3A770811A@code0.codespeak.net> Author: fijal Date: Thu Oct 18 11:05:26 2007 New Revision: 47530 Added: pypy/dist/pypy/translator/c/src/ffi.h pypy/dist/pypy/translator/c/src/ffitarget.h Log: Add ffi.h and ffitarget.h Added: pypy/dist/pypy/translator/c/src/ffi.h ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/c/src/ffi.h Thu Oct 18 11:05:26 2007 @@ -0,0 +1,321 @@ +/* -----------------------------------------------------------------*-C-*- + libffi 2.1 - Copyright (c) 1996-2003 Red Hat, Inc. + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + ``Software''), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR + OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + ----------------------------------------------------------------------- */ + +/* ------------------------------------------------------------------- + The basic API is described in the README file. + + The raw API is designed to bypass some of the argument packing + and unpacking on architectures for which it can be avoided. + + The closure API allows interpreted functions to be packaged up + inside a C function pointer, so that they can be called as C functions, + with no understanding on the client side that they are interpreted. + It can also be used in other cases in which it is necessary to package + up a user specified parameter and a function pointer as a single + function pointer. + + The closure API must be implemented in order to get its functionality, + e.g. for use by gij. Routines are provided to emulate the raw API + if the underlying platform doesn't allow faster implementation. + + More details on the raw and cloure API can be found in: + + http://gcc.gnu.org/ml/java/1999-q3/msg00138.html + + and + + http://gcc.gnu.org/ml/java/1999-q3/msg00174.html + -------------------------------------------------------------------- */ + +#ifndef LIBFFI_H +#define LIBFFI_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Specify which architecture libffi is configured for. */ +#define X86_64 + +/* ---- System configuration information --------------------------------- */ + +#include + +#ifndef LIBFFI_ASM + +#include +#include + +/* LONG_LONG_MAX is not always defined (not if STRICT_ANSI, for example). + But we can find it either under the correct ANSI name, or under GNU + C's internal name. */ +#ifdef LONG_LONG_MAX +# define FFI_LONG_LONG_MAX LONG_LONG_MAX +#else +# ifdef LLONG_MAX +# define FFI_LONG_LONG_MAX LLONG_MAX +# else +# ifdef __GNUC__ +# define FFI_LONG_LONG_MAX __LONG_LONG_MAX__ +# endif +# endif +#endif + +#if SCHAR_MAX == 127 +# define ffi_type_uchar ffi_type_uint8 +# define ffi_type_schar ffi_type_sint8 +#else + #error "char size not supported" +#endif + +#if SHRT_MAX == 32767 +# define ffi_type_ushort ffi_type_uint16 +# define ffi_type_sshort ffi_type_sint16 +#elif SHRT_MAX == 2147483647 +# define ffi_type_ushort ffi_type_uint32 +# define ffi_type_sshort ffi_type_sint32 +#else + #error "short size not supported" +#endif + +#if INT_MAX == 32767 +# define ffi_type_uint ffi_type_uint16 +# define ffi_type_sint ffi_type_sint16 +#elif INT_MAX == 2147483647 +# define ffi_type_uint ffi_type_uint32 +# define ffi_type_sint ffi_type_sint32 +#elif INT_MAX == 9223372036854775807 +# define ffi_type_uint ffi_type_uint64 +# define ffi_type_sint ffi_type_sint64 +#else + #error "int size not supported" +#endif + +#if LONG_MAX == 2147483647 +# if FFI_LONG_LONG_MAX != 9223372036854775807 + #error "no 64-bit data type supported" +# endif +#elif LONG_MAX != 9223372036854775807 + #error "long size not supported" +#endif + +#if LONG_MAX == 2147483647 +# define ffi_type_ulong ffi_type_uint32 +# define ffi_type_slong ffi_type_sint32 +#elif LONG_MAX == 9223372036854775807 +# define ffi_type_ulong ffi_type_uint64 +# define ffi_type_slong ffi_type_sint64 +#else + #error "long size not supported" +#endif + +/* The closure code assumes that this works on pointers, i.e. a size_t */ +/* can hold a pointer. */ + +typedef struct _ffi_type +{ + size_t size; + unsigned short alignment; + unsigned short type; + /*@null@*/ struct _ffi_type **elements; +} ffi_type; + +/* These are defined in types.c */ +extern ffi_type ffi_type_void; +extern ffi_type ffi_type_uint8; +extern ffi_type ffi_type_sint8; +extern ffi_type ffi_type_uint16; +extern ffi_type ffi_type_sint16; +extern ffi_type ffi_type_uint32; +extern ffi_type ffi_type_sint32; +extern ffi_type ffi_type_uint64; +extern ffi_type ffi_type_sint64; +extern ffi_type ffi_type_float; +extern ffi_type ffi_type_double; +extern ffi_type ffi_type_longdouble; +extern ffi_type ffi_type_pointer; + + +typedef enum { + FFI_OK = 0, + FFI_BAD_TYPEDEF, + FFI_BAD_ABI +} ffi_status; + +typedef unsigned FFI_TYPE; + +typedef struct { + ffi_abi abi; + unsigned nargs; + /*@dependent@*/ ffi_type **arg_types; + /*@dependent@*/ ffi_type *rtype; + unsigned bytes; + unsigned flags; +#ifdef FFI_EXTRA_CIF_FIELDS + FFI_EXTRA_CIF_FIELDS; +#endif +} ffi_cif; + +/* ---- Definitions for the raw API -------------------------------------- */ + +#ifndef FFI_SIZEOF_ARG +# if LONG_MAX == 2147483647 +# define FFI_SIZEOF_ARG 4 +# elif LONG_MAX == 9223372036854775807 +# define FFI_SIZEOF_ARG 8 +# endif +#endif + +typedef union { + ffi_sarg sint; + ffi_arg uint; + float flt; + char data[FFI_SIZEOF_ARG]; + void* ptr; +} ffi_raw; + +void ffi_raw_call (/*@dependent@*/ ffi_cif *cif, + void (*fn)(), + /*@out@*/ void *rvalue, + /*@dependent@*/ ffi_raw *avalue); + +void ffi_ptrarray_to_raw (ffi_cif *cif, void **args, ffi_raw *raw); +void ffi_raw_to_ptrarray (ffi_cif *cif, ffi_raw *raw, void **args); +size_t ffi_raw_size (ffi_cif *cif); + +/* This is analogous to the raw API, except it uses Java parameter */ +/* packing, even on 64-bit machines. I.e. on 64-bit machines */ +/* longs and doubles are followed by an empty 64-bit word. */ + +void ffi_java_raw_call (/*@dependent@*/ ffi_cif *cif, + void (*fn)(), + /*@out@*/ void *rvalue, + /*@dependent@*/ ffi_raw *avalue); + +void ffi_java_ptrarray_to_raw (ffi_cif *cif, void **args, ffi_raw *raw); +void ffi_java_raw_to_ptrarray (ffi_cif *cif, ffi_raw *raw, void **args); +size_t ffi_java_raw_size (ffi_cif *cif); + +/* ---- Definitions for closures ----------------------------------------- */ + +#if FFI_CLOSURES + +typedef struct { + char tramp[FFI_TRAMPOLINE_SIZE]; + ffi_cif *cif; + void (*fun)(ffi_cif*,void*,void**,void*); + void *user_data; +} ffi_closure __attribute__((aligned (8))); + +ffi_status +ffi_prep_closure (ffi_closure*, + ffi_cif *, + void (*fun)(ffi_cif*,void*,void**,void*), + void *user_data); + +typedef struct { + char tramp[FFI_TRAMPOLINE_SIZE]; + + ffi_cif *cif; + +#if !FFI_NATIVE_RAW_API + + /* if this is enabled, then a raw closure has the same layout + as a regular closure. We use this to install an intermediate + handler to do the transaltion, void** -> ffi_raw*. */ + + void (*translate_args)(ffi_cif*,void*,void**,void*); + void *this_closure; + +#endif + + void (*fun)(ffi_cif*,void*,ffi_raw*,void*); + void *user_data; + +} ffi_raw_closure; + +ffi_status +ffi_prep_raw_closure (ffi_raw_closure*, + ffi_cif *cif, + void (*fun)(ffi_cif*,void*,ffi_raw*,void*), + void *user_data); + +ffi_status +ffi_prep_java_raw_closure (ffi_raw_closure*, + ffi_cif *cif, + void (*fun)(ffi_cif*,void*,ffi_raw*,void*), + void *user_data); + +#endif /* FFI_CLOSURES */ + +/* ---- Public interface definition -------------------------------------- */ + +ffi_status ffi_prep_cif(/*@out@*/ /*@partial@*/ ffi_cif *cif, + ffi_abi abi, + unsigned int nargs, + /*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type *rtype, + /*@dependent@*/ ffi_type **atypes); + +void ffi_call(/*@dependent@*/ ffi_cif *cif, + void (*fn)(), + /*@out@*/ void *rvalue, + /*@dependent@*/ void **avalue); + +/* Useful for eliminating compiler warnings */ +#define FFI_FN(f) ((void (*)())f) + +/* ---- Definitions shared with assembly code ---------------------------- */ + +#endif + +/* If these change, update src/mips/ffitarget.h. */ +#define FFI_TYPE_VOID 0 +#define FFI_TYPE_INT 1 +#define FFI_TYPE_FLOAT 2 +#define FFI_TYPE_DOUBLE 3 +#if 1 +#define FFI_TYPE_LONGDOUBLE 4 +#else +#define FFI_TYPE_LONGDOUBLE FFI_TYPE_DOUBLE +#endif +#define FFI_TYPE_UINT8 5 +#define FFI_TYPE_SINT8 6 +#define FFI_TYPE_UINT16 7 +#define FFI_TYPE_SINT16 8 +#define FFI_TYPE_UINT32 9 +#define FFI_TYPE_SINT32 10 +#define FFI_TYPE_UINT64 11 +#define FFI_TYPE_SINT64 12 +#define FFI_TYPE_STRUCT 13 +#define FFI_TYPE_POINTER 14 + +/* This should always refer to the last type code (for sanity checks) */ +#define FFI_TYPE_LAST FFI_TYPE_POINTER + +#ifdef __cplusplus +} +#endif + +#endif + Added: pypy/dist/pypy/translator/c/src/ffitarget.h ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/c/src/ffitarget.h Thu Oct 18 11:05:26 2007 @@ -0,0 +1,86 @@ +/* -----------------------------------------------------------------*-C-*- + ffitarget.h - Copyright (c) 1996-2003 Red Hat, Inc. + Target configuration macros for x86 and x86-64. + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + ``Software''), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR + OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + ----------------------------------------------------------------------- */ + +#ifndef LIBFFI_TARGET_H +#define LIBFFI_TARGET_H + +/* ---- System specific configurations ----------------------------------- */ + +#if defined (X86_64) && defined (__i386__) +#undef X86_64 +#define X86 +#endif + +#if defined (X86) && defined (__x86_64__) +#undef X86 +#define X86_64 +#endif + +/* ---- Generic type definitions ----------------------------------------- */ + +#ifndef LIBFFI_ASM +typedef unsigned long ffi_arg; +typedef signed long ffi_sarg; + +typedef enum ffi_abi { + FFI_FIRST_ABI = 0, + + /* ---- Intel x86 Win32 ---------- */ +#ifdef X86_WIN32 + FFI_SYSV, + FFI_STDCALL, + /* TODO: Add fastcall support for the sake of completeness */ + FFI_DEFAULT_ABI = FFI_SYSV, +#endif + + /* ---- Intel x86 and AMD x86-64 - */ +#if !defined(X86_WIN32) && (defined(__i386__) || defined(__x86_64__)) + FFI_SYSV, + FFI_UNIX64, /* Unix variants all use the same ABI for x86-64 */ +#ifdef __i386__ + FFI_DEFAULT_ABI = FFI_SYSV, +#else + FFI_DEFAULT_ABI = FFI_UNIX64, +#endif +#endif + + FFI_LAST_ABI = FFI_DEFAULT_ABI + 1 +} ffi_abi; +#endif + +/* ---- Definitions for closures ----------------------------------------- */ + +#define FFI_CLOSURES 1 + +#ifdef X86_64 +#define FFI_TRAMPOLINE_SIZE 24 +#define FFI_NATIVE_RAW_API 0 +#else +#define FFI_TRAMPOLINE_SIZE 10 +#define FFI_NATIVE_RAW_API 1 /* x86 has native raw api support */ +#endif + +#endif + From fijal at codespeak.net Thu Oct 18 11:09:28 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 11:09:28 +0200 (CEST) Subject: [pypy-svn] r47531 - pypy/dist/pypy/rlib Message-ID: <20071018090928.9F524811B@code0.codespeak.net> Author: fijal Date: Thu Oct 18 11:09:27 2007 New Revision: 47531 Modified: pypy/dist/pypy/rlib/libffi.py Log: Another hack... Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Thu Oct 18 11:09:27 2007 @@ -7,9 +7,11 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import we_are_translated +from pypy.tool.autopath import pypydir +import py -includes = ['dlfcn.h', 'ffi.h'] -include_dirs = ['/usr/include/libffi'] +includes = ['dlfcn.h', 'src/ffi.h'] +include_dirs = [str(py.path.local(pypydir).join('translator', 'c'))] FFI_TYPE_P = lltype.Ptr(lltype.ForwardReference()) FFI_TYPE_PP = rffi.CArrayPtr(FFI_TYPE_P) From fijal at codespeak.net Thu Oct 18 11:09:35 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 11:09:35 +0200 (CEST) Subject: [pypy-svn] r47532 - pypy/dist/pypy/translator/c/src Message-ID: <20071018090935.B526A8120@code0.codespeak.net> Author: fijal Date: Thu Oct 18 11:09:35 2007 New Revision: 47532 Modified: pypy/dist/pypy/translator/c/src/ffi.h Log: Fix include Modified: pypy/dist/pypy/translator/c/src/ffi.h ============================================================================== --- pypy/dist/pypy/translator/c/src/ffi.h (original) +++ pypy/dist/pypy/translator/c/src/ffi.h Thu Oct 18 11:09:35 2007 @@ -60,7 +60,7 @@ /* ---- System configuration information --------------------------------- */ -#include +#include #ifndef LIBFFI_ASM From fijal at codespeak.net Thu Oct 18 11:34:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 11:34:46 +0200 (CEST) Subject: [pypy-svn] r47533 - in pypy/dist/pypy/module/_ffi: . test Message-ID: <20071018093446.D7595811A@code0.codespeak.net> Author: fijal Date: Thu Oct 18 11:34:46 2007 New Revision: 47533 Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: Support for None as a result type Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Thu Oct 18 11:34:46 2007 @@ -62,10 +62,16 @@ raise OperationError(space.w_ValueError, space.wrap( "Uknown type letter %s" % key)) - def ptr(self, space, name, w_argtypes, restype): + def ptr(self, space, name, w_argtypes, w_restype): """ Get a pointer for function name with provided argtypes and restype """ + if space.is_w(w_restype, space.w_None): + restype = 'v' + ffi_restype = ffi_type_void + else: + restype = space.str_w(w_restype) + ffi_restype = self.get_type(restype) w = space.wrap w_argtypes = space.newtuple(space.unpackiterable(w_argtypes)) w_key = space.newtuple([w(name), w_argtypes, w(restype)]) @@ -79,7 +85,6 @@ argtypes_w = space.unpackiterable(w_argtypes) argtypes = [space.str_w(w_arg) for w_arg in argtypes_w] ffi_argtypes = [self.get_type(arg) for arg in argtypes] - ffi_restype = self.get_type(restype) try: ptr = self.cdll.getpointer(name, ffi_argtypes, ffi_restype) w_funcptr = W_FuncPtr(ptr, argtypes, restype) @@ -88,7 +93,7 @@ except KeyError: raise OperationError(space.w_AttributeError, space.wrap( "No symbol %s found in library %s" % (name, self.name))) - ptr.unwrap_spec = ['self', ObjSpace, str, W_Root, str] + ptr.unwrap_spec = ['self', ObjSpace, str, W_Root, W_Root] def descr_new_cdll(space, w_type, name): try: @@ -219,7 +224,9 @@ unwrap_value(space, push, self.ptr, i, argtype, w_arg, to_free) i += 1 try: - return wrap_value(space, ptr_call, self.ptr, None, self.restype) + if self.restype != 'v': + return wrap_value(space, ptr_call, self.ptr, None, self.restype) + return space.w_None finally: for elem in to_free: lltype.free(elem, flavor='raw') Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Thu Oct 18 11:34:46 2007 @@ -25,6 +25,10 @@ struct x* next; }; + void nothing() + { + } + char inner_struct_elem(struct x *x1) { return x1->next->x3; @@ -226,6 +230,13 @@ assert ptr1 is None assert X(get_array_elem_s(a, 1)).x2 == 3 + def test_bad_parameters(self): + import _ffi + lib = _ffi.CDLL(self.lib_name) + nothing = lib.ptr('nothing', [], None) + assert nothing() is None + raises(AttributeError, "lib.ptr('get_charx', [], None)") + def test_implicit_structure(self): skip("Does not work yet") import _ffi From fijal at codespeak.net Thu Oct 18 11:56:39 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 11:56:39 +0200 (CEST) Subject: [pypy-svn] r47534 - in pypy/dist/pypy: rlib translator/c/src Message-ID: <20071018095639.130428125@code0.codespeak.net> Author: fijal Date: Thu Oct 18 11:56:38 2007 New Revision: 47534 Removed: pypy/dist/pypy/translator/c/src/ffi.h pypy/dist/pypy/translator/c/src/ffitarget.h Modified: pypy/dist/pypy/rlib/libffi.py Log: Resort to another hack. Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Thu Oct 18 11:56:38 2007 @@ -10,8 +10,15 @@ from pypy.tool.autopath import pypydir import py -includes = ['dlfcn.h', 'src/ffi.h'] -include_dirs = [str(py.path.local(pypydir).join('translator', 'c'))] +includes = ['dlfcn.h', 'ffi.h'] +include_dirs = [] +pot_incl = py.path.local('/usr/include/libffi') +if pot_incl.check(): + include_dirs.append(str(pot_incl)) +lib_dirs = [] +pot_lib = py.path.local('/usr/lib/libffi') +if pot_lib.check(): + include_dirs.append(str(pot_lib)) FFI_TYPE_P = lltype.Ptr(lltype.ForwardReference()) FFI_TYPE_PP = rffi.CArrayPtr(FFI_TYPE_P) @@ -20,6 +27,7 @@ _includes_ = includes _libraries_ = ['ffi'] _include_dirs_ = include_dirs + _lib_dirs_ = lib_dirs RTLD_LOCAL = rffi_platform.DefinedConstantInteger('RTLD_LOCAL') RTLD_GLOBAL = rffi_platform.DefinedConstantInteger('RTLD_GLOBAL') From fijal at codespeak.net Thu Oct 18 11:56:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 11:56:57 +0200 (CEST) Subject: [pypy-svn] r47535 - pypy/dist/pypy/rpython/tool Message-ID: <20071018095657.D0DF0811B@code0.codespeak.net> Author: fijal Date: Thu Oct 18 11:56:57 2007 New Revision: 47535 Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py Log: Add library_dirs as possible attribute Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Thu Oct 18 11:56:57 2007 @@ -114,8 +114,10 @@ self.close() include_dirs = getattr(self.config, '_include_dirs_', []) libraries = getattr(self.config, '_libraries_', []) + library_dirs = getattr(self.config, '_library_dirs_', []) return try_compile_cache([self.path], include_dirs=include_dirs, - libraries=libraries) + libraries=libraries, + library_dirs=library_dirs) def configure(CConfig): """Examine the local system by running the C compiler. @@ -143,8 +145,9 @@ include_dirs = getattr(CConfig, '_include_dirs_', []) libraries = getattr(CConfig, '_libraries_', []) + library_dirs = getattr(CConfig, '_library_dirs_', []) infolist = list(run_example_code(writer.path, include_dirs, - libraries)) + libraries, library_dirs)) assert len(infolist) == len(entries) resultinfo = {} @@ -477,9 +480,10 @@ } """ -def run_example_code(filepath, include_dirs=[], libraries=[]): +def run_example_code(filepath, include_dirs=[], libraries=[], library_dirs=[]): output = build_executable_cache([filepath], include_dirs=include_dirs, - libraries=libraries) + libraries=libraries, + library_dirs=library_dirs) section = None for line in output.splitlines(): line = line.strip() From arigo at codespeak.net Thu Oct 18 12:23:41 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 18 Oct 2007 12:23:41 +0200 (CEST) Subject: [pypy-svn] r47536 - in pypy/dist/pypy: rlib rpython/lltypesystem translator/c translator/tool Message-ID: <20071018102341.D8D89811A@code0.codespeak.net> Author: arigo Date: Thu Oct 18 12:23:40 2007 New Revision: 47536 Modified: pypy/dist/pypy/rlib/libffi.py pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/c/node.py pypy/dist/pypy/translator/tool/cbuild.py Log: * proper library_dirs support in genc * typos in libffi Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Thu Oct 18 12:23:40 2007 @@ -18,7 +18,7 @@ lib_dirs = [] pot_lib = py.path.local('/usr/lib/libffi') if pot_lib.check(): - include_dirs.append(str(pot_lib)) + lib_dirs.append(str(pot_lib)) FFI_TYPE_P = lltype.Ptr(lltype.ForwardReference()) FFI_TYPE_PP = rffi.CArrayPtr(FFI_TYPE_P) @@ -27,7 +27,7 @@ _includes_ = includes _libraries_ = ['ffi'] _include_dirs_ = include_dirs - _lib_dirs_ = lib_dirs + _library_dirs_ = lib_dirs RTLD_LOCAL = rffi_platform.DefinedConstantInteger('RTLD_LOCAL') RTLD_GLOBAL = rffi_platform.DefinedConstantInteger('RTLD_GLOBAL') @@ -96,7 +96,7 @@ def external(name, args, result, **kwds): return rffi.llexternal(name, args, result, includes=includes, - include_dirs=include_dirs, + include_dirs=include_dirs, library_dirs=lib_dirs, libraries=['dl', 'ffi'], **kwds) c_dlopen = external('dlopen', [rffi.CCHARP, rffi.INT], rffi.VOIDP, Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Thu Oct 18 12:23:40 2007 @@ -27,9 +27,9 @@ return self.TP def llexternal(name, args, result, _callable=None, sources=[], includes=[], - libraries=[], include_dirs=[], sandboxsafe=False, - canraise=False, _nowrapper=False, calling_conv='c', - threadsafe='auto'): + libraries=[], include_dirs=[], library_dirs=[], + sandboxsafe=False, threadsafe='auto', + canraise=False, _nowrapper=False, calling_conv='c'): """Build an external function that will invoke the C function 'name' with the given 'args' types and 'result' type. @@ -53,6 +53,7 @@ includes=tuple(includes), libraries=tuple(libraries), include_dirs=tuple(include_dirs), + library_dirs=tuple(library_dirs), _callable=_callable, _safe_not_sandboxed=sandboxsafe, _debugexc=True, # on top of llinterp Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Thu Oct 18 12:23:40 2007 @@ -144,24 +144,26 @@ from pypy.translator.c.symboltable import SymbolTable # XXX fix symboltable #self.symboltable = SymbolTable() - cfile, extra, extraincludes = gen_source(db, modulename, targetdir, - defines = defines, - exports = self.exports, - symboltable = self.symboltable, - libraries = self.libraries) + cfile, extra, include_dirs, library_dirs = \ + gen_source(db, modulename, targetdir, + defines = defines, + exports = self.exports, + symboltable = self.symboltable, + libraries = self.libraries) else: if self.config.translation.instrument: defines['INSTRUMENT'] = 1 if CBuilder.have___thread: if not self.config.translation.no__thread: defines['USE___THREAD'] = 1 - cfile, extra, extraincludes = \ + cfile, extra, include_dirs, library_dirs = \ gen_source_standalone(db, modulename, targetdir, entrypointname = pfname, defines = defines) self.c_source_filename = py.path.local(cfile) self.extrafiles = extra - self.extraincludes = extraincludes.keys() + self.include_dirs = include_dirs.keys() + self.library_dirs = library_dirs.keys() if self.standalone: self.gen_makefile(targetdir) return cfile @@ -190,10 +192,11 @@ def compile(self): assert self.c_source_filename assert not self._compiled - extra_includes = self.extraincludes + extra_includes = self.include_dirs compile_c_module([self.c_source_filename] + self.extrafiles, self.c_source_filename.purebasename, include_dirs = [autopath.this_dir] + extra_includes, + library_dirs = self.library_dirs, libraries=self.libraries) self._compiled = True @@ -251,13 +254,14 @@ [self.c_source_filename] + self.extrafiles, include_dirs = [autopath.this_dir, python_inc] + extra_includes, libraries = self.libraries, + library_dirs = self.library_dirs, compiler_exe = cc, profbased = profbased) def compile(self): assert self.c_source_filename assert not self._compiled compiler = self.getccompiler(extra_includes=[str(self.targetdir)] + - self.extraincludes) + self.include_dirs) if sys.platform == 'darwin': compiler.compile_extra.append('-mdynamic-no-pic') if self.config.translation.compilerflags: @@ -623,6 +627,7 @@ includes = {} sources = {} include_dirs = {} + library_dirs = {} for node in database.globalcontainers(): if hasattr(node, 'includes'): for include in node.includes: @@ -633,9 +638,12 @@ if hasattr(node, 'include_dirs'): for include_dir in node.include_dirs: include_dirs[include_dir] = True + if hasattr(node, 'library_dirs'): + for library_dir in node.library_dirs: + library_dirs[library_dir] = True includes = includes.keys() includes.sort() - return includes, sources, include_dirs + return includes, sources, include_dirs, library_dirs def gen_source_standalone(database, modulename, targetdir, entrypointname, defines={}): @@ -666,7 +674,7 @@ for line in database.gcpolicy.pre_gc_code(): print >> fi, line - includes, sources, include_dirs = extra_information(database) + includes, sources, include_dirs, library_dirs = extra_information(database) for include in includes: print >> fi, '#include <%s>' % (include,) fi.close() @@ -695,7 +703,7 @@ print >>fi, "#define INSTRUMENT_NCOUNTER %d" % n fi.close() - return filename, sg.getextrafiles(), include_dirs + return filename, sg.getextrafiles(), include_dirs, library_dirs def gen_source(database, modulename, targetdir, defines={}, exports={}, @@ -725,7 +733,7 @@ for line in database.gcpolicy.pre_gc_code(): print >> fi, line - includes, sources, include_dirs = extra_information(database) + includes, sources, include_dirs, library_dirs = extra_information(database) for include in includes: print >> fi, '#include <%s>' % (include,) fi.close() @@ -857,7 +865,7 @@ f.write(SETUP_PY % locals()) f.close() - return filename, sg.getextrafiles(), include_dirs + return filename, sg.getextrafiles(), include_dirs, library_dirs SETUP_PY = ''' @@ -876,7 +884,8 @@ ext_modules = [Extension(name = "%(modulename)s", sources = ["%(modulename)s.c"], extra_compile_args = extra_compile_args, - include_dirs = [PYPY_INCLUDE_DIR], + include_dirs = [PYPY_INCLUDE_DIR] + %(include_dirs)r, + library_dirs = %(library_dirs)r, libraries = %(libraries)r)]) ''' Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Thu Oct 18 12:23:40 2007 @@ -705,6 +705,8 @@ self.libraries = obj.libraries if hasattr(obj, 'include_dirs'): self.include_dirs = obj.include_dirs + if hasattr(obj, 'library_dirs'): + self.library_dirs = obj.library_dirs self.make_funcgens() #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) Modified: pypy/dist/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/cbuild.py (original) +++ pypy/dist/pypy/translator/tool/cbuild.py Thu Oct 18 12:23:40 2007 @@ -39,17 +39,16 @@ opt += '/Op' gcv['OPT'] = opt -def compile_c_module(cfiles, modname, include_dirs=None, libraries=[]): +def compile_c_module(cfiles, modname, include_dirs=[], libraries=[], + library_dirs=[]): #try: # from distutils.log import set_threshold # set_threshold(10000) #except ImportError: # print "ERROR IMPORTING" # pass - if include_dirs is None: - include_dirs = [] - - library_dirs = [] + include_dirs = list(include_dirs) + library_dirs = list(library_dirs) if sys.platform == 'darwin': # support Fink & Darwinports for s in ('/sw/', '/opt/local/'): if s + 'include' not in include_dirs and \ @@ -75,6 +74,8 @@ str(dirpath.join(modname))) for dir in [gcv['INCLUDEPY']] + list(include_dirs): cmd += ' -I%s' % dir + for dir in library_dirs: + cmd += ' -L%s' % dir os.system(cmd) else: from distutils.dist import Distribution From fijal at codespeak.net Thu Oct 18 12:31:29 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 12:31:29 +0200 (CEST) Subject: [pypy-svn] r47537 - pypy/dist/pypy/module/_ffi/test Message-ID: <20071018103129.44CE6811A@code0.codespeak.net> Author: fijal Date: Thu Oct 18 12:31:28 2007 New Revision: 47537 Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: A (passing) test for bad arguments. Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Thu Oct 18 12:31:28 2007 @@ -236,6 +236,9 @@ nothing = lib.ptr('nothing', [], None) assert nothing() is None raises(AttributeError, "lib.ptr('get_charx', [], None)") + raises(ValueError, "lib.ptr('get_char', ['xx'], None)") + raises(ValueError, "lib.ptr('get_char', ['x'], None)") + raises(ValueError, "lib.ptr('get_char', [], 'x')") def test_implicit_structure(self): skip("Does not work yet") From fijal at codespeak.net Thu Oct 18 12:46:33 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 12:46:33 +0200 (CEST) Subject: [pypy-svn] r47539 - in pypy/dist/pypy/module/_ffi: . test Message-ID: <20071018104633.A9776811A@code0.codespeak.net> Author: fijal Date: Thu Oct 18 12:46:33 2007 New Revision: 47539 Modified: pypy/dist/pypy/module/_ffi/__init__.py pypy/dist/pypy/module/_ffi/app_ffi.py pypy/dist/pypy/module/_ffi/array.py pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/structure.py pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: Paranoia++, check for wrong arguments in even more places Modified: pypy/dist/pypy/module/_ffi/__init__.py ============================================================================== --- pypy/dist/pypy/module/_ffi/__init__.py (original) +++ pypy/dist/pypy/module/_ffi/__init__.py Thu Oct 18 12:46:33 2007 @@ -14,6 +14,7 @@ 'FuncPtr' : 'interp_ffi.W_FuncPtr', 'StructureInstance' : 'structure.W_StructureInstance', 'ArrayInstance' : 'array.W_ArrayInstance', + '_get_type' : 'interp_ffi._get_type', } appleveldefs = { Modified: pypy/dist/pypy/module/_ffi/app_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/app_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/app_ffi.py Thu Oct 18 12:46:33 2007 @@ -1,7 +1,13 @@ # NOT_RPYTHON class Structure(object): + def check_fields(self, fields): + import _ffi + for name, letter in fields: + _ffi._get_type(letter) + def __init__(self, fields): + self.check_fields(fields) self.fields = fields def __call__(self, *args, **kwds): @@ -16,7 +22,9 @@ class Array(object): def __init__(self, of): + import _ffi self.of = of + _ffi._get_type(of) def __call__(self, size): from _ffi import ArrayInstance Modified: pypy/dist/pypy/module/_ffi/array.py ============================================================================== --- pypy/dist/pypy/module/_ffi/array.py (original) +++ pypy/dist/pypy/module/_ffi/array.py Thu Oct 18 12:46:33 2007 @@ -10,7 +10,7 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError, wrap_oserror from pypy.module._ffi.structure import native_fmttable -from pypy.module._ffi.interp_ffi import unwrap_value, wrap_value +from pypy.module._ffi.interp_ffi import unwrap_value, wrap_value, _get_type def push_elem(ll_array, pos, value): TP = lltype.typeOf(value) @@ -27,6 +27,7 @@ def __init__(self, space, of, length): self.alloced = False self.of = of + _get_type(space, of) self.length = length size = native_fmttable[of]['size'] * length self.ll_array = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Thu Oct 18 12:46:33 2007 @@ -47,6 +47,14 @@ 'P' : rffi.VOIDP, } +def _get_type(space, key): + try: + return TYPEMAP[key] + except KeyError: + raise OperationError(space.w_ValueError, space.wrap( + "Uknown type letter %s" % key)) +_get_type.unwrap_spec = [ObjSpace, str] + class W_CDLL(Wrappable): def __init__(self, space, name): self.cdll = CDLL(name) @@ -56,11 +64,7 @@ def get_type(self, key): space = self.space - try: - return TYPEMAP[key] - except KeyError: - raise OperationError(space.w_ValueError, space.wrap( - "Uknown type letter %s" % key)) + return _get_type(space, key) def ptr(self, space, name, w_argtypes, w_restype): """ Get a pointer for function name with provided argtypes Modified: pypy/dist/pypy/module/_ffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_ffi/structure.py (original) +++ pypy/dist/pypy/module/_ffi/structure.py Thu Oct 18 12:46:33 2007 @@ -13,7 +13,7 @@ # the other one is in rlib/libffi, we should refactor it to reuse the same # logic, I'll not touch it by now, and refactor it later from pypy.module.struct.nativefmttable import native_fmttable as struct_native_fmttable -from pypy.module._ffi.interp_ffi import wrap_value, unwrap_value +from pypy.module._ffi.interp_ffi import wrap_value, unwrap_value, _get_type native_fmttable = {} for key, value in struct_native_fmttable.items(): @@ -28,7 +28,9 @@ if not len(l_w) == 2: raise OperationError(space.w_ValueError, space.wrap( "Expected list of 2-size tuples")) - fields.append((space.str_w(l_w[0]), space.str_w(l_w[1]))) + name, code = space.str_w(l_w[0]), space.str_w(l_w[1]) + _get_type(space, code) # be paranoid about types + fields.append((name, code)) return fields def size_and_pos(fields): Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Thu Oct 18 12:46:33 2007 @@ -239,6 +239,14 @@ raises(ValueError, "lib.ptr('get_char', ['xx'], None)") raises(ValueError, "lib.ptr('get_char', ['x'], None)") raises(ValueError, "lib.ptr('get_char', [], 'x')") + raises(ValueError, "_ffi.Structure(['x1', 'xx'])") + S = _ffi.Structure([('x1', 'i')]) + S.fields[0] = ('x1', 'xx') + raises(ValueError, "S()") + raises(ValueError, "_ffi.Array('xx')") + A = _ffi.Array('i') + A.of = 'xx' + raises(ValueError, 'A(1)') def test_implicit_structure(self): skip("Does not work yet") From mwh at codespeak.net Thu Oct 18 12:49:36 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Thu, 18 Oct 2007 12:49:36 +0200 (CEST) Subject: [pypy-svn] r47540 - pypy/dist/pypy/translator/goal Message-ID: <20071018104936.DB1D0811B@code0.codespeak.net> Author: mwh Date: Thu Oct 18 12:49:36 2007 New Revision: 47540 Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py Log: now tuatara is back: 1. don't bother trying to build llvm for now 2. tproxies are enabled by default, no need to build that 3. build a semispace build too Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py ============================================================================== --- pypy/dist/pypy/translator/goal/bench-cronjob.py (original) +++ pypy/dist/pypy/translator/goal/bench-cronjob.py Thu Oct 18 12:49:36 2007 @@ -176,15 +176,15 @@ def main(backends=[]): if backends == []: #_ prefix means target specific option, # prefix to outcomment backends = [backend.strip() for backend in """ - llvm--_faassen + #llvm--_faassen c c--stackless--_faassen c--_faassen c--thread c--_objspace=taint c--_allworkingmodules - c--_objspace-std-withtproxy--_faassen - c--gc=framework--_faassen + c--gcframework=marksweep--_faassen + c--gcframework=semispace--_faassen c--_objspace-std-withrope cli """.split('\n') if backend.strip() and not backend.strip().startswith('#')] From arigo at codespeak.net Thu Oct 18 13:13:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 18 Oct 2007 13:13:23 +0200 (CEST) Subject: [pypy-svn] r47541 - in pypy/dist/pypy/rpython: . lltypesystem Message-ID: <20071018111323.034C68115@code0.codespeak.net> Author: arigo Date: Thu Oct 18 13:13:22 2007 New Revision: 47541 Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py pypy/dist/pypy/rpython/rlist.py Log: Sprinkle overflow checks in rlist. Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rlist.py Thu Oct 18 13:13:22 2007 @@ -17,6 +17,7 @@ from pypy.rpython.lltypesystem import rstr from pypy.rpython import robject from pypy.rlib.objectmodel import debug_assert +from pypy.rlib.rarithmetic import ovfcheck # ____________________________________________________________ # @@ -268,18 +269,9 @@ def _ll_list_resize_really(l, newsize): """ - Ensure ob_item has room for at least newsize elements, and set - ob_size to newsize. If newsize > ob_size on entry, the content - of the new slots at exit is undefined heap trash; it's the caller's - responsiblity to overwrite them with sane values. - The number of allocated elements may grow, shrink, or stay the same. - Failure is impossible if newsize <= self.allocated on entry, although - that partly relies on an assumption that the system realloc() never - fails when passed a number of bytes <= the number of bytes last - allocated (the C standard doesn't guarantee this, but it's hard to - imagine a realloc implementation where it wouldn't be true). - Note that self->ob_item may change, and even if newsize is less - than ob_size on entry. + Ensure l.items has room for at least newsize elements, and set + l.length to newsize. Note that l.items may change, and even if + newsize is less than l.length on entry. """ allocated = len(l.items) @@ -287,16 +279,21 @@ # for additional growth. The over-allocation is mild, but is # enough to give linear-time amortized behavior over a long # sequence of appends() in the presence of a poorly-performing - # system realloc(). + # system malloc(). # The growth pattern is: 0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ... - ## (newsize < 9 ? 3 : 6) - if newsize < 9: - some = 3 - else: - some = 6 - new_allocated = (newsize >> 3) + some + newsize - if newsize == 0: + if newsize <= 0: + debug_assert(newsize == 0, "negative list length") new_allocated = 0 + else: + if newsize < 9: + some = 3 + else: + some = 6 + some += newsize >> 3 + try: + new_allocated = ovfcheck(newsize + some) + except OverflowError: + raise MemoryError # XXX consider to have a real realloc items = l.items newitems = malloc(typeOf(l).TO.items.TO, new_allocated) @@ -457,5 +454,5 @@ index = iter.index if index >= l.ll_length(): raise StopIteration - iter.index = index + 1 + iter.index = index + 1 # cannot overflow because index < l.length return l.ll_getitem_fast(index) Modified: pypy/dist/pypy/rpython/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/rlist.py (original) +++ pypy/dist/pypy/rpython/rlist.py Thu Oct 18 13:13:22 2007 @@ -9,6 +9,7 @@ from pypy.rpython.lltypesystem.lltype import nullptr, Char, UniChar from pypy.rpython import robject from pypy.rlib.objectmodel import malloc_zero_filled, debug_assert +from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.annlowlevel import ADTInterface ADTIFixedList = ADTInterface(None, { @@ -469,6 +470,19 @@ # Low-level methods. These can be run for testing, but are meant to # be direct_call'ed from rtyped flow graphs, which means that they will # get flowed and annotated, mostly with SomePtr. +# +# === a note about overflows === +# +# The maximal length of RPython lists is bounded by the assumption that +# we can never allocate arrays more than sys.maxint bytes in size. +# Our arrays have a length and some GC headers, so a list of characters +# could come near sys.maxint in length (but not reach it). A list of +# pointers could only come near sys.maxint/sizeof(void*) elements. There +# is the list of Voids that could reach exactly sys.maxint elements, +# but for now let's ignore this case -- the reasoning is that even if +# the length of a Void list overflows, nothing bad memory-wise can be +# done with it. So in the sequel we don't bother checking for overflow +# when we compute "ll_length() + 1". def ll_alloc_and_set(LIST, count, item): if count < 0: @@ -530,14 +544,14 @@ def ll_append(l, newitem): length = l.ll_length() - l._ll_resize_ge(length+1) + l._ll_resize_ge(length+1) # see "a note about overflows" above l.ll_setitem_fast(length, newitem) ll_append.oopspec = 'list.append(l, newitem)' # this one is for the special case of insert(0, x) def ll_prepend(l, newitem): length = l.ll_length() - l._ll_resize_ge(length+1) + l._ll_resize_ge(length+1) # see "a note about overflows" above dst = length while dst > 0: src = dst - 1 @@ -549,7 +563,10 @@ def ll_concat(RESLIST, l1, l2): len1 = l1.ll_length() len2 = l2.ll_length() - newlength = len1 + len2 + try: + newlength = ovfcheck(len1 + len2) + except OverflowError: + raise MemoryError l = RESLIST.ll_newlist(newlength) j = 0 while j < len1: @@ -567,7 +584,7 @@ length = l.ll_length() debug_assert(0 <= index, "negative list insertion index") debug_assert(index <= length, "list insertion index out of bound") - l._ll_resize_ge(length+1) + l._ll_resize_ge(length+1) # see "a note about overflows" above dst = length while dst > index: src = dst - 1 @@ -739,7 +756,10 @@ def ll_extend(l1, l2): len1 = l1.ll_length() len2 = l2.ll_length() - newlength = len1 + len2 + try: + newlength = ovfcheck(len1 + len2) + except OverflowError: + raise MemoryError l1._ll_resize_ge(newlength) i = 0 j = len1 @@ -754,9 +774,13 @@ def ll_extend_with_str_slice_startonly(lst, s, getstrlen, getstritem, start): len1 = lst.ll_length() len2 = getstrlen(s) + count2 = len2 - start debug_assert(start >= 0, "unexpectedly negative str slice start") - debug_assert(start <= len2, "str slice start larger than str length") - newlength = len1 + len2 - start + assert count2 >= 0, "str slice start larger than str length" + try: + newlength = ovfcheck(len1 + count2) + except OverflowError: + raise MemoryError lst._ll_resize_ge(newlength) i = start j = len1 @@ -775,10 +799,14 @@ len2 = getstrlen(s) debug_assert(start >= 0, "unexpectedly negative str slice start") debug_assert(start <= len2, "str slice start larger than str length") - debug_assert(stop >= start, "str slice stop smaller than start") if stop > len2: stop = len2 - newlength = len1 + stop - start + count2 = stop - start + assert count2 >= 0, "str slice stop smaller than start" + try: + newlength = ovfcheck(len1 + count2) + except OverflowError: + raise MemoryError lst._ll_resize_ge(newlength) i = start j = len1 @@ -793,8 +821,11 @@ def ll_extend_with_str_slice_minusone(lst, s, getstrlen, getstritem): len1 = lst.ll_length() len2m1 = getstrlen(s) - 1 - debug_assert(len2m1 >= 0, "empty string is sliced with [:-1]") - newlength = len1 + len2m1 + assert len2m1 >= 0, "empty string is sliced with [:-1]" + try: + newlength = ovfcheck(len1 + len2m1) + except OverflowError: + raise MemoryError lst._ll_resize_ge(newlength) i = 0 j = len1 @@ -810,7 +841,10 @@ if count <= 0: return len1 = lst.ll_length() - newlength = len1 + count + try: + newlength = ovfcheck(len1 + count) + except OverflowError: + raise MemoryError lst._ll_resize_ge(newlength) j = len1 if listItemType(lst) is UniChar: @@ -972,7 +1006,10 @@ length = l.ll_length() if factor < 0: factor = 0 - resultlen = length * factor + try: + resultlen = ovfcheck(length * factor) + except OverflowError: + raise MemoryError res = l res._ll_resize(resultlen) #res._ll_resize_ge(resultlen) @@ -991,7 +1028,10 @@ length = l.ll_length() if factor < 0: factor = 0 - resultlen = length * factor + try: + resultlen = ovfcheck(length * factor) + except OverflowError: + raise MemoryError res = RESLIST.ll_newlist(resultlen) j = 0 while j < resultlen: From arigo at codespeak.net Thu Oct 18 13:33:38 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 18 Oct 2007 13:33:38 +0200 (CEST) Subject: [pypy-svn] r47542 - pypy/dist/pypy/module/bz2/test Message-ID: <20071018113338.3265980B2@code0.codespeak.net> Author: arigo Date: Thu Oct 18 13:33:37 2007 New Revision: 47542 Modified: pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py Log: Skip "huge" tests by default. Modified: pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py Thu Oct 18 13:33:37 2007 @@ -2,6 +2,8 @@ from pypy.module.bz2.test.support import CheckAllocation import os +HUGE_OK = False + if os.name == "nt": from py.test import skip skip("bz2 module is not available on Windows") @@ -35,6 +37,7 @@ cls.space = space cls.w_TEXT = space.wrap(TEXT) cls.w_decompress = space.wrap(decompress) + cls.w_HUGE_OK = space.wrap(HUGE_OK) def test_creation(self): from bz2 import BZ2Compressor @@ -55,6 +58,8 @@ assert self.decompress(data) == self.TEXT def test_compress_huge_data(self): + if not self.HUGE_OK: + skip("skipping test requiring lots of memory") from bz2 import BZ2Compressor HUGE_DATA = self.TEXT * 10000 @@ -146,6 +151,7 @@ cls.w_TEXT = space.wrap(TEXT) cls.w_DATA = space.wrap(DATA) cls.w_decompress = space.wrap(decompress) + cls.w_HUGE_OK = space.wrap(HUGE_OK) def test_compress_function(self): from bz2 import compress @@ -158,6 +164,8 @@ assert self.decompress(data) == self.TEXT def test_compress_function_huge_data(self): + if not self.HUGE_OK: + skip("skipping test requiring lots of memory") from bz2 import compress HUGE_DATA = self.TEXT * 10000 From arigo at codespeak.net Thu Oct 18 13:34:03 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 18 Oct 2007 13:34:03 +0200 (CEST) Subject: [pypy-svn] r47543 - pypy/dist/pypy/module/bz2 Message-ID: <20071018113403.A4395811C@code0.codespeak.net> Author: arigo Date: Thu Oct 18 13:34:03 2007 New Revision: 47543 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: Remove this '<< 32', bogus for RPython. Replace it with overflow checking. Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Thu Oct 18 13:34:03 2007 @@ -80,16 +80,19 @@ else: BIGCHUNK = 512 * 1024 -MAXINT = sys.maxint - if BZ_CONFIG_ERROR: - if rffi.sizeof(rffi.LONG) >= 8 or rffi.sizeof(rffi.LONGLONG) >= 8: + if rffi.sizeof(rffi.LONG) >= 8: def _bzs_total_out(bzs): return (bzs.c_total_out_hi32 << 32) + bzs.c_total_out_lo32 else: + # we can't return a long long value from here, because most + # callers wouldn't be able to handle it anyway def _bzs_total_out(bzs): + if bzs.c_total_out_hi32 != 0 or bzs.c_total_out_lo32 > sys.maxint: + raise MemoryError return bzs.c_total_out_lo32 else: + XXX # this case needs fixing (old bz2 library?) def _bzs_total_out(bzs): return bzs.total_out From fijal at codespeak.net Thu Oct 18 13:37:01 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 13:37:01 +0200 (CEST) Subject: [pypy-svn] r47544 - in pypy/dist/pypy/module/_ffi: . test Message-ID: <20071018113701.11279811C@code0.codespeak.net> Author: fijal Date: Thu Oct 18 13:37:01 2007 New Revision: 47544 Modified: pypy/dist/pypy/module/_ffi/TODO pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: * Support for longlongs and friends * More tests. Modified: pypy/dist/pypy/module/_ffi/TODO ============================================================================== --- pypy/dist/pypy/module/_ffi/TODO (original) +++ pypy/dist/pypy/module/_ffi/TODO Thu Oct 18 13:37:01 2007 @@ -2,6 +2,3 @@ * long support with all messy corners (when to create long integer and such), also short overflowing -* add faulty parameters checking (they end up with fatal rpython error KeyErro - or sth like that by now) - Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Thu Oct 18 13:37:01 2007 @@ -9,6 +9,13 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.unroll import unrolling_iterable +from pypy.module.struct.standardfmttable import min_max_acc_method +from pypy.module.struct.nativefmttable import native_fmttable + +class FfiValueError(Exception): + def __init__(self, msg): + self.msg = msg + TYPEMAP = { # XXX A mess with unsigned/signed/normal chars :-/ 'c' : ffi_type_uchar, @@ -129,6 +136,38 @@ push_func(add_arg, argdesc, ll_str) return ll_str +def make_size_checker(format, size, signed): + min, max, _ = min_max_acc_method(size, signed) + + def checker(value): + if value < min: + raise FfiValueError("%d too small for format %s" % (value, format)) + elif value > max: + raise FfiValueError("%d too large for format %s" % (value, format)) + return checker + +_SIZE_CHECKERS = { + 'h' : True, + 'H' : False, + 'I' : False, + 'c' : False, + 'b' : True, + 'B' : False, + 'i' : True, + 'I' : False, + 'l' : True, + 'L' : False, + 'q' : True, + 'Q' : False, +} + +# XXX check for single float as well +SIZE_CHECKERS = {} +for c, signed in _SIZE_CHECKERS.items(): + SIZE_CHECKERS[c] = make_size_checker(c, native_fmttable[c]['size'], signed) +del _SIZE_CHECKERS +unroll_size_checkers = unrolling_iterable(SIZE_CHECKERS.items()) + def unwrap_value(space, push_func, add_arg, argdesc, tp, w_arg, to_free): w = space.wrap # XXX how to handle LONGLONG here? @@ -158,16 +197,22 @@ else: raise OperationError(space.w_TypeError, w( "Expected structure, array or simple type")) - elif tp == "c" or tp == "b" or tp == "B": - s = space.str_w(w_arg) - if len(s) != 1: - raise OperationError(space.w_ValueError, w( - "Expected string of length one as character")) - s = s[0] - push_func(add_arg, argdesc, s) else: - #assert tp "iIhHlLqQ" - push_func(add_arg, argdesc, space.int_w(w_arg)) + if tp == "c" or tp == "b" or tp == "B": + s = space.str_w(w_arg) + if len(s) != 1: + raise OperationError(space.w_ValueError, w( + "Expected string of length one as character")) + val = ord(s[0]) + else: + val = space.int_w(w_arg) + for c, checker in unroll_size_checkers: + if tp == c: + try: + checker(val) + except FfiValueError, e: + raise OperationError(space.w_ValueError, w(e.msg)) + push_func(add_arg, argdesc, val) unwrap_value._annspecialcase_ = 'specialize:arg(1)' ll_typemap_iter = unrolling_iterable(LL_TYPEMAP.items()) @@ -185,8 +230,8 @@ if not res: return space.w_None return space.wrap(rffi.cast(rffi.INT, res)) - #elif c == 'q' or c == 'Q' or c == 'L': - # return space.newlong(func(arg, ll_type)) + elif c == 'q' or c == 'Q' or c == 'L': + return space.wrap(func(add_arg, argdesc, ll_type)) elif c == 'f' or c == 'd': return space.wrap(float(func(add_arg, argdesc, ll_type))) elif c == 'c' or c == 'b' or c == 'B': Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Thu Oct 18 13:37:01 2007 @@ -73,6 +73,16 @@ { return array[num]; } + + long long some_huge_value() + { + return 1LL<<42; + } + + unsigned long long some_huge_uvalue() + { + return 1LL<<42; + } ''')) compile_c_module([c_file], 'x') return str(udir.join('x.so')) @@ -102,8 +112,8 @@ get_char = lib.ptr('get_char', ['s', 'H'], 'c') assert get_char('dupa', 2) == 'p' assert get_char('dupa', 1) == 'u' - skip("this module does not do overflow checking by now") - raises(OverflowError, "get_char('xxx', 2 ** 17)") + raises(ValueError, "get_char('xxx', 2 ** 17)") + raises(ValueError, "get_char('xxx', -1)") def test_returning_str(self): import _ffi @@ -263,5 +273,11 @@ def test_longs_ulongs(self): - skip("Not implemented yet") - + import _ffi + lib = _ffi.CDLL(self.lib_name) + some_huge_value = lib.ptr('some_huge_value', [], 'q') + assert some_huge_value() == 1<<42 + some_huge_uvalue = lib.ptr('some_huge_uvalue', [], 'Q') + assert some_huge_uvalue() == 1<<42 + x = lib.ptr('some_huge_value', ['Q'], None) + raises(ValueError, "x(-1)") From arigo at codespeak.net Thu Oct 18 14:36:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 18 Oct 2007 14:36:53 +0200 (CEST) Subject: [pypy-svn] r47545 - pypy/extradoc/sprintinfo/bern2007 Message-ID: <20071018123653.DA3EA811D@code0.codespeak.net> Author: arigo Date: Thu Oct 18 14:36:52 2007 New Revision: 47545 Modified: pypy/extradoc/sprintinfo/bern2007/people.txt Log: My dates and arrival time. Modified: pypy/extradoc/sprintinfo/bern2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/bern2007/people.txt (original) +++ pypy/extradoc/sprintinfo/bern2007/people.txt Thu Oct 18 14:36:52 2007 @@ -5,14 +5,14 @@ column are known to be coming but there are no details available yet from them. -==================== ============== ===================== +==================== ================= ===================== Name Arrive/Depart Accomodation -==================== ============== ===================== -Carl Friedrich Bolz ? ? -Adrian Kuhn - private -Niko Matsakis 22.10 / 26.10 private (Zurich) -Armin Rigo ? ? -==================== ============== ===================== +==================== ================= ===================== +Carl Friedrich Bolz ? ? +Adrian Kuhn - private +Niko Matsakis 22.10 / 26.10 private (Zurich) +Armin Rigo 22.10(9:26)/26.10 at the Uni +==================== ================= ===================== People on the following list were present at previous sprints: From cfbolz at codespeak.net Thu Oct 18 14:41:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 18 Oct 2007 14:41:44 +0200 (CEST) Subject: [pypy-svn] r47546 - in pypy/dist/pypy: config config/test doc/config module/_stackless rlib rpython/memory/gc rpython/memory/test translator/c translator/c/test Message-ID: <20071018124144.8EE37811D@code0.codespeak.net> Author: cfbolz Date: Thu Oct 18 14:41:43 2007 New Revision: 47546 Added: pypy/dist/pypy/doc/config/translation.gctransformer.txt Modified: pypy/dist/pypy/config/pypyoption.py pypy/dist/pypy/config/test/test_pypyoption.py pypy/dist/pypy/config/translationoption.py pypy/dist/pypy/module/_stackless/__init__.py pypy/dist/pypy/rlib/rgc.py pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/c/test/test_boehm.py pypy/dist/pypy/translator/c/test/test_newgc.py Log: make the --gc option more userfriendly: let the user specify framework GCs with the --gc option too. Add a new translation.gctransformer option for the function that the translation.gc option had until now. Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Thu Oct 18 14:41:43 2007 @@ -48,7 +48,7 @@ ("objspace.usemodules._stackless", True), ("objspace.usemodules._cslib", True), ("objspace.usemodules.cclp", True), - ("translation.gc", 'framework'), + ("translation.gc", 'marksweep'), ], }, cmdline='--objspace -o'), Modified: pypy/dist/pypy/config/test/test_pypyoption.py ============================================================================== --- pypy/dist/pypy/config/test/test_pypyoption.py (original) +++ pypy/dist/pypy/config/test/test_pypyoption.py Thu Oct 18 14:41:43 2007 @@ -21,14 +21,19 @@ conf.translation.stacklessgc = True assert conf.translation.stackless assert conf.translation.type_system == "lltype" - assert conf.translation.gc == "framework" + assert conf.translation.gctransformer == "framework" + assert conf.translation.gc == "marksweep" + conf = get_pypy_config() + conf.translation.gc = "boehm" + py.test.raises(ValueError, "conf.translation.stacklessgc = True") + def test_frameworkgc(): for name in ["marksweep", "semispace"]: conf = get_pypy_config() - assert conf.translation.gc != "framework" - conf.translation.frameworkgc = name - assert conf.translation.gc == "framework" + assert conf.translation.gctransformer != "framework" + conf.translation.gc = name + assert conf.translation.gctransformer == "framework" def test_check_documentation(): from pypy.doc.config.confrest import all_optiondescrs Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Thu Oct 18 14:41:43 2007 @@ -41,22 +41,27 @@ default=False, cmdline="--llvm-via-c", requires=[("translation.backend", "llvm")]), ChoiceOption("gc", "Garbage Collection Strategy", - ["boehm", "ref", "framework", "none"], + ["boehm", "ref", "marksweep", "semispace", "statistics", "none"], "ref", requires={ - "ref": [("translation.rweakref", False)], # XXX - "none": [("translation.rweakref", False)], # XXX + "ref": [("translation.rweakref", False), # XXX + ("translation.gctransformer", "ref")], + "none": [("translation.rweakref", False), # XXX + ("translation.gctransformer", "none")], + "semispace": [("translation.gctransformer", "framework")], + "marksweep": [("translation.gctransformer", "framework")], + "statistics": [("translation.gctransformer", "framework")], + "boehm": [("translation.gctransformer", "boehm")], }, cmdline="--gc"), + ChoiceOption("gctransformer", "GC transformer that is used - internal", + ["boehm", "ref", "framework", "none"], + cmdline=None), + BoolOption("stacklessgc", "Use stackless to find roots in a framework GC", default=False, cmdline="--stacklessgc", - requires=[("translation.gc", "framework"), - ("translation.stackless", True)]), - ChoiceOption("frameworkgc", "Select one of our custom GCs", - ["marksweep", "semispace", "statistics"], - "marksweep", cmdline="--frameworkgc", requires={ - "marksweep": [("translation.gc", "framework")], - "semispace": [("translation.gc", "framework")], - }), + requires=[("translation.gctransformer", "framework"), + ("translation.stackless", True)], + suggests=[("translation.gc", "marksweep")]), BoolOption("thread", "enable use of threading primitives", default=False, cmdline="--thread", requires=[("translation.gc", "boehm")]), Added: pypy/dist/pypy/doc/config/translation.gctransformer.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/config/translation.gctransformer.txt Thu Oct 18 14:41:43 2007 @@ -0,0 +1 @@ +internal option Modified: pypy/dist/pypy/module/_stackless/__init__.py ============================================================================== --- pypy/dist/pypy/module/_stackless/__init__.py (original) +++ pypy/dist/pypy/module/_stackless/__init__.py Thu Oct 18 14:41:43 2007 @@ -27,7 +27,7 @@ from pypy.module._stackless.interp_greenlet import post_install as post_install_greenlet post_install_greenlet(self) - if self.space.config.translation.gc == 'framework': + if self.space.config.translation.gc == 'marksweep': from pypy.module._stackless.interp_clonable import post_install as post_install_clonable self.extra_interpdef('clonable', 'interp_clonable.AppClonableCoroutine') self.extra_interpdef('fork', 'interp_clonable.fork') Modified: pypy/dist/pypy/rlib/rgc.py ============================================================================== --- pypy/dist/pypy/rlib/rgc.py (original) +++ pypy/dist/pypy/rlib/rgc.py Thu Oct 18 14:41:43 2007 @@ -42,7 +42,7 @@ config = rtyper.getconfig() # if the gc policy doesn't support allocation pools, lltype # pools as Void. - if config.translation.gc != 'framework': + if config.translation.gc != 'marksweep': from pypy.annotation.model import s_None return rtyper.getrepr(s_None) else: @@ -66,7 +66,7 @@ opname = 'gc_x_swap_pool' config = hop.rtyper.getconfig() - if config.translation.gc != 'framework': + if config.translation.gc != 'marksweep': # when the gc policy doesn't support pools, just return # the argument (which is lltyped as Void anyway) opname = 'same_as' @@ -95,7 +95,7 @@ from pypy.rpython.memory.gc.marksweep import X_CLONE, X_CLONE_PTR config = hop.rtyper.getconfig() - if config.translation.gc != 'framework': + if config.translation.gc != 'marksweep': # if the gc policy does not support allocation pools, # gc_clone always raises RuntimeError hop.exception_is_here() Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Thu Oct 18 14:41:43 2007 @@ -175,16 +175,16 @@ def choose_gc_from_config(config): """Return a (GCClass, GC_PARAMS) from the given config object. """ - config.translation.gc = "framework" - if config.translation.frameworkgc == "marksweep": + assert config.translation.gctransformer == "framework" + if config.translation.gc == "marksweep": GC_PARAMS = {'start_heap_size': 8*1024*1024} # XXX adjust from pypy.rpython.memory.gc.marksweep import MarkSweepGC return MarkSweepGC, GC_PARAMS - if config.translation.frameworkgc == "statistics": + if config.translation.gc == "statistics": GC_PARAMS = {'start_heap_size': 8*1024*1024} # XXX adjust from pypy.rpython.memory.gc.marksweep import PrintingMarkSweepGC return PrintingMarkSweepGC, GC_PARAMS - elif config.translation.frameworkgc == "semispace": + elif config.translation.gc == "semispace": GC_PARAMS = {'space_size': 8*1024*1024} # XXX adjust from pypy.rpython.memory.gc.semispace import SemiSpaceGC return SemiSpaceGC, GC_PARAMS Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Thu Oct 18 14:41:43 2007 @@ -80,7 +80,6 @@ return run class GenericGCTests(GCTest): - gcname = "framework" def heap_usage(self, statistics): try: @@ -397,6 +396,7 @@ assert res == 0 class TestMarkSweepGC(GenericGCTests): + gcname = "marksweep" class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): GC_PARAMS = {'start_heap_size': 4096 } @@ -620,6 +620,7 @@ class TestStacklessMarkSweepGC(TestMarkSweepGC): + gcname = "marksweep" stacklessgc = True class gcpolicy(gc.StacklessFrameworkGcPolicy): @@ -649,6 +650,7 @@ class TestPrintingGC(GenericGCTests): + gcname = "statistics" class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): @@ -657,6 +659,7 @@ root_stack_depth = 200 class TestSemiSpaceGC(GenericGCTests): + gcname = "semispace" class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Thu Oct 18 14:41:43 2007 @@ -107,7 +107,7 @@ def get_gcpolicyclass(self): if self.gcpolicy is None: - name = self.config.translation.gc + name = self.config.translation.gctransformer if self.config.translation.stacklessgc: name = "%s+stacklessgc" % (name,) return gc.name_to_gcpolicy[name] Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Thu Oct 18 14:41:43 2007 @@ -27,8 +27,6 @@ config = get_pypy_config(translating=True) config.translation.gc = self.gcpolicy config.translation.stacklessgc = self.stacklessgc - if self.gcpolicy == "framework": - config.translation.frameworkgc = self.frameworkgc config.translation.simplifying = True t = TranslationContext(config=config) self.t = t Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Thu Oct 18 14:41:43 2007 @@ -278,8 +278,7 @@ from pypy.translator.c.test.test_boehm import AbstractGCTestClass class TestUsingFramework(AbstractGCTestClass): - gcpolicy = "framework" - frameworkgc = "marksweep" + gcpolicy = "marksweep" should_be_moving = False def test_empty_collect(self): @@ -827,7 +826,7 @@ py.test.skip("fails for some reason I couldn't figure out yet :-(") class TestSemiSpaceGC(TestUsingFramework): - frameworkgc = "semispace" + gcpolicy = "semispace" should_be_moving = True def test_many_ids(self): From fijal at codespeak.net Thu Oct 18 15:04:48 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 15:04:48 +0200 (CEST) Subject: [pypy-svn] r47547 - in pypy/dist/pypy/module/_ffi: . test Message-ID: <20071018130448.CE002811F@code0.codespeak.net> Author: fijal Date: Thu Oct 18 15:04:47 2007 New Revision: 47547 Modified: pypy/dist/pypy/module/_ffi/__init__.py pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: More rpythonism Modified: pypy/dist/pypy/module/_ffi/__init__.py ============================================================================== --- pypy/dist/pypy/module/_ffi/__init__.py (original) +++ pypy/dist/pypy/module/_ffi/__init__.py Thu Oct 18 15:04:47 2007 @@ -14,7 +14,7 @@ 'FuncPtr' : 'interp_ffi.W_FuncPtr', 'StructureInstance' : 'structure.W_StructureInstance', 'ArrayInstance' : 'array.W_ArrayInstance', - '_get_type' : 'interp_ffi._get_type', + '_get_type' : 'interp_ffi._w_get_type', } appleveldefs = { Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Thu Oct 18 15:04:47 2007 @@ -59,8 +59,13 @@ return TYPEMAP[key] except KeyError: raise OperationError(space.w_ValueError, space.wrap( - "Uknown type letter %s" % key)) -_get_type.unwrap_spec = [ObjSpace, str] + "Uknown type letter %s" % (key,))) + return lltype.nullptr(FFI_TYPE_P.TO) + +def _w_get_type(space, key): + _get_type(space, key) + return space.w_None +_w_get_type.unwrap_spec = [ObjSpace, str] class W_CDLL(Wrappable): def __init__(self, space, name): @@ -198,12 +203,20 @@ raise OperationError(space.w_TypeError, w( "Expected structure, array or simple type")) else: + # XXX not sure how this will look like at the end regarding + # val annotation if tp == "c" or tp == "b" or tp == "B": s = space.str_w(w_arg) if len(s) != 1: raise OperationError(space.w_ValueError, w( "Expected string of length one as character")) val = ord(s[0]) + elif tp == 'I': + val = space.uint_w(w_arg) + elif tp == 'q': + val = space.r_longlong_w(w_arg) + elif tp == 'Q': + val = space.r_ulonglong_w(w_arg) else: val = space.int_w(w_arg) for c, checker in unroll_size_checkers: Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Thu Oct 18 15:04:47 2007 @@ -83,6 +83,11 @@ { return 1LL<<42; } + + long long pass_ll(long long x) + { + return x; + } ''')) compile_c_module([c_file], 'x') return str(udir.join('x.so')) @@ -281,3 +286,6 @@ assert some_huge_uvalue() == 1<<42 x = lib.ptr('some_huge_value', ['Q'], None) raises(ValueError, "x(-1)") + pass_ll = lib.ptr('pass_ll', ['q'], 'q') + assert pass_ll(1<<42) == 1<<42 + From fijal at codespeak.net Thu Oct 18 15:05:39 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 15:05:39 +0200 (CEST) Subject: [pypy-svn] r47548 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071018130539.EB2CB811F@code0.codespeak.net> Author: fijal Date: Thu Oct 18 15:05:38 2007 New Revision: 47548 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: * XXX * memo CArrayPtr Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Thu Oct 18 15:05:38 2007 @@ -286,6 +286,7 @@ if addr in ALLOCATED: raise Exception("internal ll2ctypes error - " "double conversion from lltype to ctypes?") + # XXX don't store here immortal structures ALLOCATED[addr] = self def _free(self): Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Thu Oct 18 15:05:38 2007 @@ -221,6 +221,7 @@ def CArrayPtr(tp): return lltype.Ptr(CArray(tp)) +CArray._annspecialcase_ = 'specialize:memo' def COpaque(name, hints=None, **kwds): if hints is None: From fijal at codespeak.net Thu Oct 18 15:06:23 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 15:06:23 +0200 (CEST) Subject: [pypy-svn] r47549 - pypy/dist/pypy/module/struct Message-ID: <20071018130623.DA6D38120@code0.codespeak.net> Author: fijal Date: Thu Oct 18 15:06:20 2007 New Revision: 47549 Modified: pypy/dist/pypy/module/struct/standardfmttable.py Log: Refactor struct for easier reuse Modified: pypy/dist/pypy/module/struct/standardfmttable.py ============================================================================== --- pypy/dist/pypy/module/struct/standardfmttable.py (original) +++ pypy/dist/pypy/module/struct/standardfmttable.py Thu Oct 18 15:06:20 2007 @@ -65,17 +65,7 @@ native_int_size = struct.calcsize("l") -def make_int_packer(size, signed, cpython_checks_range, _memo={}): - if cpython_checks_range: - check_range = True - else: - check_range = not PACK_ACCEPTS_BROKEN_INPUT - key = (size, signed, check_range) - try: - return _memo[key] - except KeyError: - pass - +def min_max_acc_method(size, signed): if signed: min = -(2 ** (8*size-1)) max = (2 ** (8*size-1)) - 1 @@ -100,6 +90,19 @@ accept_method = 'accept_ulonglong_arg' min = r_ulonglong(min) max = r_ulonglong(max) + return min, max, accept_method + +def make_int_packer(size, signed, cpython_checks_range, _memo={}): + if cpython_checks_range: + check_range = True + else: + check_range = not PACK_ACCEPTS_BROKEN_INPUT + key = (size, signed, check_range) + try: + return _memo[key] + except KeyError: + pass + min, max, accept_method = min_max_acc_method(size, signed) if size > 1: plural = "s" else: From fijal at codespeak.net Thu Oct 18 15:10:23 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 15:10:23 +0200 (CEST) Subject: [pypy-svn] r47550 - pypy/dist/pypy/module/_ffi Message-ID: <20071018131023.698F98120@code0.codespeak.net> Author: fijal Date: Thu Oct 18 15:10:22 2007 New Revision: 47550 Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py Log: More rpython fun Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Thu Oct 18 15:10:22 2007 @@ -202,30 +202,32 @@ else: raise OperationError(space.w_TypeError, w( "Expected structure, array or simple type")) + if tp == "c" or tp == "b" or tp == "B": + s = space.str_w(w_arg) + if len(s) != 1: + raise OperationError(space.w_ValueError, w( + "Expected string of length one as character")) + val = ord(s[0]) + push_func(add_arg, argdesc, val) else: - # XXX not sure how this will look like at the end regarding - # val annotation - if tp == "c" or tp == "b" or tp == "B": - s = space.str_w(w_arg) - if len(s) != 1: - raise OperationError(space.w_ValueError, w( - "Expected string of length one as character")) - val = ord(s[0]) - elif tp == 'I': - val = space.uint_w(w_arg) - elif tp == 'q': - val = space.r_longlong_w(w_arg) - elif tp == 'Q': - val = space.r_ulonglong_w(w_arg) - else: - val = space.int_w(w_arg) for c, checker in unroll_size_checkers: if tp == c: + if tp == "q": + val = space.r_longlong_w(w_arg) + elif tp == "Q": + val = space.r_ulonglong_w(w_arg) + elif tp == "I": + val = space.uint_w(w_arg) + elif tp == "L": + val = space.ulong_w(w_arg) + else: + val = space.int_w(w_arg) try: checker(val) except FfiValueError, e: raise OperationError(space.w_ValueError, w(e.msg)) - push_func(add_arg, argdesc, val) + push_func(add_arg, argdesc, val) + unwrap_value._annspecialcase_ = 'specialize:arg(1)' ll_typemap_iter = unrolling_iterable(LL_TYPEMAP.items()) From fijal at codespeak.net Thu Oct 18 15:24:19 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 15:24:19 +0200 (CEST) Subject: [pypy-svn] r47551 - pypy/dist/pypy/module/_ffi Message-ID: <20071018132419.3C90980C5@code0.codespeak.net> Author: fijal Date: Thu Oct 18 15:24:17 2007 New Revision: 47551 Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py Log: There is no space.ulong_w Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Thu Oct 18 15:24:17 2007 @@ -216,10 +216,8 @@ val = space.r_longlong_w(w_arg) elif tp == "Q": val = space.r_ulonglong_w(w_arg) - elif tp == "I": + elif tp == "I" or tp == "L": val = space.uint_w(w_arg) - elif tp == "L": - val = space.ulong_w(w_arg) else: val = space.int_w(w_arg) try: From fijal at codespeak.net Thu Oct 18 16:21:18 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 16:21:18 +0200 (CEST) Subject: [pypy-svn] r47552 - pypy/dist/pypy/module/_ffi Message-ID: <20071018142118.78A52811F@code0.codespeak.net> Author: fijal Date: Thu Oct 18 16:21:16 2007 New Revision: 47552 Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py Log: Seems to be rpython this time. Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Thu Oct 18 16:21:16 2007 @@ -152,12 +152,10 @@ return checker _SIZE_CHECKERS = { - 'h' : True, - 'H' : False, - 'I' : False, - 'c' : False, 'b' : True, 'B' : False, + 'h' : True, + 'H' : False, 'i' : True, 'I' : False, 'l' : True, @@ -170,7 +168,6 @@ SIZE_CHECKERS = {} for c, signed in _SIZE_CHECKERS.items(): SIZE_CHECKERS[c] = make_size_checker(c, native_fmttable[c]['size'], signed) -del _SIZE_CHECKERS unroll_size_checkers = unrolling_iterable(SIZE_CHECKERS.items()) def unwrap_value(space, push_func, add_arg, argdesc, tp, w_arg, to_free): @@ -202,7 +199,7 @@ else: raise OperationError(space.w_TypeError, w( "Expected structure, array or simple type")) - if tp == "c" or tp == "b" or tp == "B": + if tp == "c": s = space.str_w(w_arg) if len(s) != 1: raise OperationError(space.w_ValueError, w( @@ -212,11 +209,11 @@ else: for c, checker in unroll_size_checkers: if tp == c: - if tp == "q": + if c == "q": val = space.r_longlong_w(w_arg) - elif tp == "Q": + elif c == "Q": val = space.r_ulonglong_w(w_arg) - elif tp == "I" or tp == "L": + elif c == "I" or c == "L" or c =="B": val = space.uint_w(w_arg) else: val = space.int_w(w_arg) From fijal at codespeak.net Thu Oct 18 16:37:38 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 16:37:38 +0200 (CEST) Subject: [pypy-svn] r47553 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071018143738.769318123@code0.codespeak.net> Author: fijal Date: Thu Oct 18 16:37:38 2007 New Revision: 47553 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: Add my dates Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Thu Oct 18 16:37:38 2007 @@ -14,7 +14,7 @@ Jacob Hallen lives there Laura Creighton lives there Christian Tismer Laura and Jacob's -Maciej Fijalkowski ? Laura and Jacob's +Maciej Fijalkowski 19-26 Laura and Jacob's ==================== ============== ===================== From fijal at codespeak.net Thu Oct 18 17:17:02 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 17:17:02 +0200 (CEST) Subject: [pypy-svn] r47554 - pypy/dist/pypy/config Message-ID: <20071018151702.F39AC80C2@code0.codespeak.net> Author: fijal Date: Thu Oct 18 17:17:01 2007 New Revision: 47554 Modified: pypy/dist/pypy/config/pypyoption.py Log: Add _ffi to allworkingmodules Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Thu Oct 18 17:17:01 2007 @@ -24,7 +24,7 @@ working_modules.update(dict.fromkeys( ["_socket", "unicodedata", "mmap", "fcntl", "rctime", "select", "crypt", "signal", "dyngram", "readline", "termios", "zlib", - "struct", "md5", "sha", "bz2" + "struct", "md5", "sha", "bz2", "_ffi" ] )) From fijal at codespeak.net Thu Oct 18 17:30:35 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Oct 2007 17:30:35 +0200 (CEST) Subject: [pypy-svn] r47555 - pypy/dist/pypy/config Message-ID: <20071018153035.DACE38128@code0.codespeak.net> Author: fijal Date: Thu Oct 18 17:30:35 2007 New Revision: 47555 Modified: pypy/dist/pypy/config/pypyoption.py Log: Remove _ffi Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Thu Oct 18 17:30:35 2007 @@ -24,7 +24,7 @@ working_modules.update(dict.fromkeys( ["_socket", "unicodedata", "mmap", "fcntl", "rctime", "select", "crypt", "signal", "dyngram", "readline", "termios", "zlib", - "struct", "md5", "sha", "bz2", "_ffi" + "struct", "md5", "sha", "bz2" ] )) From arigo at codespeak.net Thu Oct 18 18:24:18 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 18 Oct 2007 18:24:18 +0200 (CEST) Subject: [pypy-svn] r47556 - in pypy/dist/pypy: config rpython/memory/gc rpython/memory/test Message-ID: <20071018162418.48A0980E8@code0.codespeak.net> Author: arigo Date: Thu Oct 18 18:24:16 2007 New Revision: 47556 Added: pypy/dist/pypy/rpython/memory/gc/generation.py Modified: pypy/dist/pypy/config/translationoption.py pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/test/test_gc.py Log: (arigato, cfbolz): a simple generational GC, written as a subclass of the SemiSpaceGC adding a nursery. Does not translate yet, the transformer does not add the write barriers. Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Thu Oct 18 18:24:16 2007 @@ -41,7 +41,8 @@ default=False, cmdline="--llvm-via-c", requires=[("translation.backend", "llvm")]), ChoiceOption("gc", "Garbage Collection Strategy", - ["boehm", "ref", "marksweep", "semispace", "statistics", "none"], + ["boehm", "ref", "marksweep", "semispace", "statistics", + "generation", "none"], "ref", requires={ "ref": [("translation.rweakref", False), # XXX ("translation.gctransformer", "ref")], @@ -50,6 +51,7 @@ "semispace": [("translation.gctransformer", "framework")], "marksweep": [("translation.gctransformer", "framework")], "statistics": [("translation.gctransformer", "framework")], + "generation": [("translation.gctransformer", "framework")], "boehm": [("translation.gctransformer", "boehm")], }, cmdline="--gc"), Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Thu Oct 18 18:24:16 2007 @@ -188,6 +188,11 @@ GC_PARAMS = {'space_size': 8*1024*1024} # XXX adjust from pypy.rpython.memory.gc.semispace import SemiSpaceGC return SemiSpaceGC, GC_PARAMS + elif config.translation.gc == "generation": + GC_PARAMS = {'space_size': 8*1024*1024, # XXX adjust + 'nursery_size': 12*1024} + from pypy.rpython.memory.gc.generation import GenerationGC + return GenerationGC, GC_PARAMS else: raise ValueError("unknown value for frameworkgc: %r" % ( config.translation.frameworkgc,)) Added: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Thu Oct 18 18:24:16 2007 @@ -0,0 +1,163 @@ +import sys +from pypy.rpython.memory.gc.semispace import SemiSpaceGC +from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage +from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rlib.objectmodel import free_non_gc_object, debug_assert + +nonnull_endmarker = llmemory.raw_malloc(llmemory.sizeof(lltype.Char)) +llmemory.raw_memclear(nonnull_endmarker, llmemory.sizeof(lltype.Char)) + + +class GenerationGC(SemiSpaceGC): + """A basic generational GC: it's a SemiSpaceGC with an additional + nursery for young objects. A write barrier is used to ensure that + old objects that contain pointers to young objects are in a linked + list, chained to each other via their 'forw' header field. + """ + inline_simple_malloc = True + + def __init__(self, AddressLinkedList, + nursery_size=128, + space_size=4096, + max_space_size=sys.maxint//2+1, + get_roots=None): + SemiSpaceGC.__init__(self, AddressLinkedList, + space_size = space_size, + max_space_size = max_space_size, + get_roots = get_roots) + self.nursery_size = nursery_size + + def setup(self): + SemiSpaceGC.setup(self) + self.nursery = llarena.arena_malloc(self.nursery_size, True) + debug_assert(bool(self.nursery), "couldn't allocate nursery") + self.nursery_free = self.nursery + self.nursery_top = self.nursery + self.nursery_size + self.old_objects_pointing_to_young = nonnull_endmarker + # ^^^ the head of a linked list inside the old objects space + + def is_in_nursery(self, addr): + return self.nursery <= addr < self.nursery_top + + def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, + contains_weakptr=False): + if (has_finalizer or contains_weakptr or not can_collect or + raw_malloc_usage(size) >= self.nursery_size // 2): + # "non-simple" case or object too big: don't use the nursery + return SemiSpaceGC.malloc_fixedsize(self, typeid, size, + can_collect, has_finalizer, + contains_weakptr) + size_gc_header = self.gcheaderbuilder.size_gc_header + totalsize = size_gc_header + size + result = self.nursery_free + if raw_malloc_usage(totalsize) > self.nursery_top - result: + result = self.collect_nursery() + llarena.arena_reserve(result, totalsize) + self.init_gc_object(result, typeid) + self.nursery_free = result + totalsize + return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) + + def semispace_collect(self, size_changing=False): + self.reset_forwarding() # we are doing a full collection anyway + SemiSpaceGC.semispace_collect(self, size_changing) + + def reset_forwarding(self): + obj = self.old_objects_pointing_to_young + while obj != nonnull_endmarker: + hdr = self.header(obj) + obj = hdr.forw + hdr.forw = llmemory.NULL + self.old_objects_pointing_to_young = nonnull_endmarker + + def collect_nursery(self): + if self.nursery_size > self.top_of_space - self.free: + # the semispace is running out, do a full collect + self.obtain_free_space(self.nursery_size) + debug_assert(self.nursery_free == self.nursery, + "nursery not collected") + else: + scan = self.free + self.collect_oldrefs_to_nursery() + self.collect_roots_in_nursery() + self.scan_objects_just_copied_out_of_nursery(scan) + self.notify_objects_just_moved() + return self.nursery_free + + def reset_nursery(self): + llarena.arena_reset(self.nursery, self.nursery_size, True) + self.nursery_free = self.nursery + + def notify_objects_just_moved(self): + self.reset_nursery() + SemiSpaceGC.notify_objects_just_moved(self) + + # NB. we can use self.copy() to move objects out of the nursery, + # but only if the object was really in the nursery. + + def collect_oldrefs_to_nursery(self): + # Follow the old_objects_pointing_to_young list and move the + # young objects they point to out of the nursery. The 'forw' + # fields are reset to NULL along the way. + obj = self.old_objects_pointing_to_young + while obj != nonnull_endmarker: + self.trace_and_drag_out_of_nursery(obj) + hdr = self.header(obj) + obj = hdr.forw + hdr.forw = llmemory.NULL + self.old_objects_pointing_to_young = nonnull_endmarker + + def collect_roots_in_nursery(self): + roots = self.get_roots() + while 1: + root = roots.pop() + if root == NULL: + break + obj = root.address[0] + if self.is_in_nursery(obj): + root.address[0] = self.copy(obj) + free_non_gc_object(roots) + + def scan_objects_just_copied_out_of_nursery(self, scan): + while scan < self.free: + curr = scan + self.size_gc_header() + self.trace_and_drag_out_of_nursery(curr) + scan += self.size_gc_header() + self.get_size(curr) + return scan + + def trace_and_drag_out_of_nursery(self, obj): + """obj must not be in the nursery. This copies all the + young objects it references out of the nursery. + """ + gc_info = self.header(obj) + typeid = gc_info.typeid + offsets = self.offsets_to_gc_pointers(typeid) + i = 0 + while i < len(offsets): + pointer = obj + offsets[i] + if self.is_in_nursery(pointer.address[0]): + pointer.address[0] = self.copy(pointer.address[0]) + i += 1 + if self.is_varsize(typeid): + offset = self.varsize_offset_to_variable_part( + typeid) + length = (obj + self.varsize_offset_to_length(typeid)).signed[0] + offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) + itemlength = self.varsize_item_sizes(typeid) + i = 0 + while i < length: + item = obj + offset + itemlength * i + j = 0 + while j < len(offsets): + pointer = item + offsets[j] + if self.is_in_nursery(pointer.address[0]): + pointer.address[0] = self.copy(pointer.address[0]) + j += 1 + i += 1 + + def write_barrier(self, addr, addr_to, addr_struct): + if not self.is_in_nursery(addr_struct) and self.is_in_nursery(addr): + oldhdr = self.header(addr_struct) + if oldhdr.forw == NULL: + oldhdr.forw = self.old_objects_pointing_to_young + self.old_objects_pointing_to_young = addr_struct + addr_to.address[0] = addr Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Thu Oct 18 18:24:16 2007 @@ -283,3 +283,6 @@ class TestGrowingSemiSpaceGC(TestSemiSpaceGC): GC_PARAMS = {'space_size': 64} + +class TestGenerationalGC(GCTest): + from pypy.rpython.memory.gc.generation import GenerationGC as GCClass From antocuni at codespeak.net Thu Oct 18 18:28:16 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 18 Oct 2007 18:28:16 +0200 (CEST) Subject: [pypy-svn] r47557 - in pypy/extradoc/talk/dls2007: . image ui Message-ID: <20071018162816.BFADD811D@code0.codespeak.net> Author: antocuni Date: Thu Oct 18 18:28:16 2007 New Revision: 47557 Added: pypy/extradoc/talk/dls2007/image/ pypy/extradoc/talk/dls2007/image/translation.png (contents, props changed) pypy/extradoc/talk/dls2007/rpython-talk.txt (contents, props changed) pypy/extradoc/talk/dls2007/rpython.pdf (contents, props changed) pypy/extradoc/talk/dls2007/ui/ - copied from r47544, pypy/extradoc/talk/dls2006/ui/ Log: - the final pdf version of my paper at DLS - some slides of the talk Added: pypy/extradoc/talk/dls2007/image/translation.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/dls2007/rpython-talk.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/dls2007/rpython-talk.txt Thu Oct 18 18:28:16 2007 @@ -0,0 +1,202 @@ +.. include:: + +======= +RPython +======= + +A Step Towards Reconciling Dynamically and Statically Typed OO Languages +------------------------------------------------------------------------ + +:Author: Antonio Cuni - DISI, Universit? degli Studi di Genova +:Date: 22 October 2007 +:Location: DLS'07 OOPSLA Montreal CA + + +Dynamic languages for .NET and JVM +================================== + +- .NET and JVM: widespread platforms + +- Designed for static languages + +- Great Python implementations: **IronPython**, **Jython** + +- Much slower than e.g. C# and Java + + +Dynamic vs. static +================== + +- Dynamic languages: + + * Flexibility + + * Rapid development cycle + + * **Metaprogramming** + +- Static languages: + + - Speed + + - Nothing more :-) + + +RPython Quick Facts +=================== + +- Restricted subset of Python + +- Statically typed (type inference) + +- Still allows metaprogramming + +- RPython programs still run under {C,J,Iron}Python + +- Three backends: C, .NET, JVM + +- Almost as fast as C, C#, Java + + +Type inference +============== + +- Top-down, starting from an entry point + +- Assign the most precise type to each variable + +- Fail if you try to mix incompatible types + +:: + + def main(): def not_RPython(flag): + print add(40, 2) if flag: + return 42 + def add(a, b): else: + return a+b return 'hello world' + + +Other restrictions +================== + +- Globals are assumed to be constant + +- ``yield`` and generators not supported + +- No special __methods__ + +- No run-time definition of new functions and classes + +- Cannot modify classes at run-time + +- Cannot change the __class__ of an object + +- Single inheritance, with limited support for mixins + + +Still pythonic, though +====================== + +- No syntactic restriction + +- Functions and classes are first-order values + +- List and dictionaries works as in Python + + * Although they must be homogeneous + + * list of int, dict from string to floats, etc. are OK + +- Most of methods of ``list``, ``dict`` and ``str`` are supported + +- Exceptions work + + +Init-time, translation-time, run-time +===================================== + +.. image:: image/translation.png + + +Metaprogramming +=============== + +- RPython restrictions only apply to live objects + +- No restrictions about how they are created + + * Full Python is allowed at init-time + +- Python as a metaprogramming language for RPython + +- Code generation considered harmful + + +Compute complex constants +========================= + +:: + + def fibo(N): + sequence = [] + a, b = 1, 1 + for i in xrange(N): + sequence.append(a) + a, b = b, a+b + return sequence + + fibo_numbers = fibo(100) # computed at init-time + + +Metaclasses run at init-time +============================ + +:: + + class MyClass(object): + def foo(self): ... + + class __extend__(MyClass): + def bar(self): ... + + def main(): + obj = MyClass() + obj.bar() + + +Dynamic classes/functions at init-time +====================================== + +:: + + def make_adder(N): + def add(x): + return x+N + return add + + add10 = make_adder(10) + add20 = make_adder(20) + def main(): + print add10(32) + print add20(22) + + +The Translation Toolchain +========================= + +- **CPython**: \*.py --> Python bytecode + +- **FlowObjSpace**: bytecode --> flow graphs + +- **Annotator**: type inference on flow graphs + + * High level Python types (``List(Integer)``) + +- **RTyper**: high level types -> low level types + + * lltype for C, ootype for CLI and JVM + +- **Backends**: code generation + + * C, CLI (.NET), JVM + Added: pypy/extradoc/talk/dls2007/rpython.pdf ============================================================================== Binary file. No diff available. From arigo at codespeak.net Thu Oct 18 18:32:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 18 Oct 2007 18:32:13 +0200 (CEST) Subject: [pypy-svn] r47558 - in pypy/dist/pypy: annotation/test doc doc/config interpreter/pyparser interpreter/pyparser/test lang/test lib/test2 module/_curses module/_curses/test module/rbench rlib rlib/test rpython/lltypesystem rpython/lltypesystem/test rpython/memory/gc rpython/memory/gc/test rpython/memory/gctransform rpython/module rpython/module/test rpython/tool rpython/tool/test translator/jvm/test translator/llvm/test translator/oosupport/test_template Message-ID: <20071018163213.CC833811A@code0.codespeak.net> Author: arigo Date: Thu Oct 18 18:32:13 2007 New Revision: 47558 Modified: pypy/dist/pypy/annotation/test/test_signature.py (props changed) pypy/dist/pypy/doc/config/objspace.usemodules._curses.txt (props changed) pypy/dist/pypy/doc/config/objspace.usemodules.termios.txt (props changed) pypy/dist/pypy/doc/config/objspace.usemodules.zlib.txt (props changed) pypy/dist/pypy/doc/config/translation.gctransformer.txt (props changed) pypy/dist/pypy/doc/sandbox.txt (props changed) pypy/dist/pypy/interpreter/pyparser/future.py (props changed) pypy/dist/pypy/interpreter/pyparser/test/test_futureautomaton.py (props changed) pypy/dist/pypy/lang/test/ (props changed) pypy/dist/pypy/lib/test2/pickledtasklet.py (props changed) pypy/dist/pypy/lib/test2/test_stackless.py (props changed) pypy/dist/pypy/module/_curses/__init__.py (props changed) pypy/dist/pypy/module/_curses/app_curses.py (props changed) pypy/dist/pypy/module/_curses/fficurses.py (props changed) pypy/dist/pypy/module/_curses/interp_curses.py (props changed) pypy/dist/pypy/module/_curses/test/__init__.py (props changed) pypy/dist/pypy/module/_curses/test/test_curses.py (props changed) pypy/dist/pypy/module/rbench/ (props changed) pypy/dist/pypy/rlib/rzlib.py (props changed) pypy/dist/pypy/rlib/test/test_rzlib.py (props changed) pypy/dist/pypy/rpython/lltypesystem/rffi.py (contents, props changed) pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (props changed) pypy/dist/pypy/rpython/memory/gc/ (props changed) pypy/dist/pypy/rpython/memory/gc/generation.py (props changed) pypy/dist/pypy/rpython/memory/gc/test/ (props changed) pypy/dist/pypy/rpython/memory/gctransform/log.py (props changed) pypy/dist/pypy/rpython/module/ll_termios.py (props changed) pypy/dist/pypy/rpython/module/test/test_ll_termios.py (props changed) pypy/dist/pypy/rpython/tool/rfficache.py (props changed) pypy/dist/pypy/rpython/tool/test/test_rfficache.py (props changed) pypy/dist/pypy/translator/jvm/test/test_rarithmetic.py (props changed) pypy/dist/pypy/translator/jvm/test/test_weakref.py (props changed) pypy/dist/pypy/translator/llvm/test/test_rffi.py (props changed) pypy/dist/pypy/translator/llvm/test/test_threading.py (props changed) pypy/dist/pypy/translator/oosupport/test_template/objectmodel.py (props changed) Log: fixeol Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Thu Oct 18 18:32:13 2007 @@ -293,10 +293,10 @@ # char * CCHARP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) - -# int *, unsigned int *, etc. -#INTP = ... see setup() above - + +# int *, unsigned int *, etc. +#INTP = ... see setup() above + # double * DOUBLEP = lltype.Ptr(lltype.Array(DOUBLE, hints={'nolength': True})) From arigo at codespeak.net Thu Oct 18 19:40:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 18 Oct 2007 19:40:50 +0200 (CEST) Subject: [pypy-svn] r47559 - in pypy/dist/pypy/rpython/memory: gc gctransform gctransform/test Message-ID: <20071018174050.C2F9380EB@code0.codespeak.net> Author: arigo Date: Thu Oct 18 19:40:49 2007 New Revision: 47559 Modified: pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctransform/refcounting.py pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py pypy/dist/pypy/rpython/memory/gctransform/transform.py Log: (arigato, cfbolz before he ran for his train): * gctransformer support for write_barrier * fix and expand the write_barrier_support test * clean up the interface of push_alive() and pop_alive() * fix tests that depends on choose_gc_from_config() picking a default GC if none is specified. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Thu Oct 18 19:40:49 2007 @@ -4,6 +4,7 @@ class GCBase(object): _alloc_flavor_ = "raw" moving_gc = False + needs_write_barrier = False def set_query_functions(self, is_varsize, getfinalizer, offsets_to_gc_pointers, @@ -175,7 +176,8 @@ def choose_gc_from_config(config): """Return a (GCClass, GC_PARAMS) from the given config object. """ - assert config.translation.gctransformer == "framework" + if config.translation.gctransformer != "framework": # for tests + config.translation.gc = "marksweep" # crash if inconsistent if config.translation.gc == "marksweep": GC_PARAMS = {'start_heap_size': 8*1024*1024} # XXX adjust from pypy.rpython.memory.gc.marksweep import MarkSweepGC @@ -194,5 +196,5 @@ from pypy.rpython.memory.gc.generation import GenerationGC return GenerationGC, GC_PARAMS else: - raise ValueError("unknown value for frameworkgc: %r" % ( - config.translation.frameworkgc,)) + raise ValueError("unknown value for translation.gc: %r" % ( + config.translation.gc,)) Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Thu Oct 18 19:40:49 2007 @@ -15,6 +15,7 @@ list, chained to each other via their 'forw' header field. """ inline_simple_malloc = True + needs_write_barrier = True def __init__(self, AddressLinkedList, nursery_size=128, Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Thu Oct 18 19:40:49 2007 @@ -9,6 +9,7 @@ from pypy.rlib.rarithmetic import ovfcheck from pypy.rlib.objectmodel import debug_assert from pypy.translator.backendopt import graphanalyze +from pypy.translator.backendopt.support import var_needsgc from pypy.annotation import model as annmodel from pypy.rpython import annlowlevel from pypy.rpython.rbuiltin import gen_cast @@ -248,6 +249,15 @@ else: self.id_ptr = None + if GCClass.needs_write_barrier: + self.write_barrier_ptr = getfn(GCClass.write_barrier.im_func, + [s_gc, annmodel.SomeAddress(), + annmodel.SomeAddress(), + annmodel.SomeAddress()], + annmodel.s_None, + inline=True) + else: + self.write_barrier_ptr = None self.statistics_ptr = getfn(GCClass.statistics.im_func, [s_gc, annmodel.SomeInteger()], annmodel.SomeInteger()) @@ -561,6 +571,60 @@ else: hop.rename('cast_ptr_to_int') # works nicely for non-moving GCs + def transform_generic_set(self, hop): + if self.write_barrier_ptr is None: + super(FrameworkGCTransformer, self).transform_generic_set(hop) + else: + v_struct = hop.spaceop.args[0] + v_newvalue = hop.spaceop.args[-1] + assert isinstance(v_newvalue.concretetype, lltype.Ptr) + opname = hop.spaceop.opname + assert opname in ('setfield', 'setarrayitem', 'setinteriorfield') + offsets = hop.spaceop.args[1:-1] + CURTYPE = v_struct.concretetype.TO + v_currentofs = None + for ofs in offsets: + if ofs.concretetype is lltype.Void: + # a field in a structure + fieldname = ofs.value + fieldofs = llmemory.offsetof(CURTYPE, fieldname) + v_offset = rmodel.inputconst(lltype.Signed, fieldofs) + CURTYPE = getattr(CURTYPE, fieldname) + else: + # an index in an array + assert ofs.concretetype is lltype.Signed + firstitem = llmemory.ArrayItemsOffset(CURTYPE) + itemsize = llmemory.sizeof(CURTYPE.OF) + c_firstitem = rmodel.inputconst(lltype.Signed, firstitem) + c_itemsize = rmodel.inputconst(lltype.Signed, itemsize) + v_index = hop.spaceop.args[1] + v_offset = hop.genop("int_mul", [c_itemsize, v_index], + resulttype = lltype.Signed) + v_offset = hop.genop("int_add", [c_firstitem, v_offset], + resulttype = lltype.Signed) + CURTYPE = CURTYPE.OF + if v_currentofs is None: + v_currentofs = v_offset + else: + v_currentofs = hop.genop("int_add", + [v_currentofs, v_offset], + resulttype = lltype.Signed) + # XXX for some GCs we could skip the write_barrier if v_newvalue + # is a constant + v_newvalue = hop.genop("cast_ptr_to_adr", [v_newvalue], + resulttype = llmemory.Address) + v_structaddr = hop.genop("cast_ptr_to_adr", [v_struct], + resulttype = llmemory.Address) + v_fieldaddr = hop.genop("adr_add", [v_structaddr, v_currentofs], + resulttype = llmemory.Address) + hop.genop("direct_call", [self.write_barrier_ptr, + v_newvalue, + v_fieldaddr, + v_structaddr]) + + def var_needs_set_transform(self, var): + return var_needsgc(var) + def push_alive_nopyobj(self, var, llops): pass @@ -573,10 +637,10 @@ if self.gcdata.gc.moving_gc: # moving GCs don't borrow, so the caller does not need to keep # the arguments alive - livevars = [var for var in self.livevars_after_op + livevars = [var for var in hop.livevars_after_op() if not var_ispyobj(var)] else: - livevars = self.livevars_after_op + self.current_op_keeps_alive + livevars = hop.livevars_after_op() + hop.current_op_keeps_alive() livevars = [var for var in livevars if not var_ispyobj(var)] self.num_pushs += len(livevars) if not livevars: Modified: pypy/dist/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/refcounting.py Thu Oct 18 19:40:49 2007 @@ -154,11 +154,11 @@ def gct_gc_protect(self, hop): """ protect this object from gc (make it immortal) """ - self.push_alive(hop.spaceop.args[0]) + self.push_alive(hop.spaceop.args[0], hop.llops) def gct_gc_unprotect(self, hop): """ get this object back into gc control """ - self.pop_alive(hop.spaceop.args[0]) + self.pop_alive(hop.spaceop.args[0], hop.llops) def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size): v_raw = hop.genop("direct_call", [self.malloc_fixedsize_ptr, c_size], Modified: pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py Thu Oct 18 19:40:49 2007 @@ -1,6 +1,6 @@ from pypy.objspace.flow.model import Constant, SpaceOperation from pypy.annotation.model import SomeInteger -from pypy.rpython.memory.gc.base import GCBase +from pypy.rpython.memory.gc.marksweep import MarkSweepGC from pypy.rpython.memory.gctransform.test.test_transform import rtype from pypy.rpython.memory.gctransform.transform import GcHighLevelOp from pypy.rpython.memory.gctransform.framework import FrameworkGCTransformer, CollectAnalyzer @@ -63,27 +63,49 @@ class WriteBarrierTransformer(FrameworkGCTransformer): GC_PARAMS = {} - class GC_CLASS(GCBase): - def write_barrier(self, addr, addr_to, addr_struct): - addr_to.address[0] = addr + class GCClass(MarkSweepGC): + needs_write_barrier = True -def test_write_barrier_support(): - py.test.skip("no write barrier support yet!") +def write_barrier_check(spaceop): t = TranslationContext() t.buildannotator().build_types(lambda x:x, [SomeInteger()]) t.buildrtyper().specialize() - llops = LowLevelOpList() - PTR_TYPE = lltype.Ptr(lltype.GcStruct('S', ('x', lltype.Signed))) - spaceop = SpaceOperation( - "setfield", - [varoftype(PTR_TYPE), Constant('x', lltype.Void)], - varoftype(lltype.Void)) transformer = WriteBarrierTransformer(t) + llops = LowLevelOpList() hop = GcHighLevelOp(transformer, spaceop, 0, llops) hop.dispatch() found = False + print spaceop, '======>' for op in llops: + print '\t', op if op.opname == 'direct_call': found = True - break assert found + +def test_write_barrier_support_setfield(): + PTR_TYPE2 = lltype.Ptr(lltype.GcStruct('T', ('y', lltype.Signed))) + PTR_TYPE = lltype.Ptr(lltype.GcStruct('S', ('x', PTR_TYPE2))) + write_barrier_check(SpaceOperation( + "setfield", + [varoftype(PTR_TYPE), Constant('x', lltype.Void), + varoftype(PTR_TYPE2)], + varoftype(lltype.Void))) + +def test_write_barrier_support_setarrayitem(): + PTR_TYPE2 = lltype.Ptr(lltype.GcStruct('T', ('y', lltype.Signed))) + ARRAYPTR = lltype.Ptr(lltype.GcArray(PTR_TYPE2)) + write_barrier_check(SpaceOperation( + "setarrayitem", + [varoftype(ARRAYPTR), varoftype(lltype.Signed), + varoftype(PTR_TYPE2)], + varoftype(lltype.Void))) + +def test_write_barrier_support_setinteriorfield(): + PTR_TYPE2 = lltype.Ptr(lltype.GcStruct('T', ('y', lltype.Signed))) + ARRAYPTR2 = lltype.Ptr(lltype.GcArray(('a', lltype.Signed), + ('b', PTR_TYPE2))) + write_barrier_check(SpaceOperation( + "setinteriorfield", + [varoftype(ARRAYPTR2), varoftype(lltype.Signed), + Constant('b', lltype.Void), varoftype(PTR_TYPE2)], + varoftype(lltype.Void))) Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Thu Oct 18 19:40:49 2007 @@ -38,10 +38,16 @@ self.spaceop = op self.index = index self.llops = llops - gct.livevars_after_op = [ + + def livevars_after_op(self): + gct = self.gctransformer + return [ var for var in gct.livevars if gct.var_last_needed_in[var] > self.index] - gct.current_op_keeps_alive = [ + + def current_op_keeps_alive(self): + gct = self.gctransformer + return [ var for var in self.spaceop.args if gct.var_last_needed_in.get(var) == self.index] @@ -60,9 +66,9 @@ 'cast_pointer', 'getsubstruct', 'getinteriorfield'): # XXX more operations? - gct.push_alive(v_result) + gct.push_alive(v_result, self.llops) elif opname not in ('direct_call', 'indirect_call'): - gct.push_alive(v_result) + gct.push_alive(v_result, self.llops) @@ -157,7 +163,7 @@ return is_borrowed def transform_block(self, block, is_borrowed): - self.llops = LowLevelOpList() + llops = LowLevelOpList() #self.curr_block = block self.livevars = [var for var in block.inputargs if var_needsgc(var) and not is_borrowed(var)] @@ -175,7 +181,7 @@ self.var_last_needed_in[var] = len(block.operations) + 1 for i, op in enumerate(block.operations): - hop = GcHighLevelOp(self, op, i, self.llops) + hop = GcHighLevelOp(self, op, i, llops) hop.dispatch() if len(block.exits) != 0: # i.e not the return block @@ -186,7 +192,7 @@ deadinallexits.difference_update(sets.Set(link.args)) for var in deadinallexits: - self.pop_alive(var) + self.pop_alive(var, llops) for link in block.exits: livecounts = dict.fromkeys(sets.Set(self.livevars) - deadinallexits, 1) @@ -201,8 +207,7 @@ livecounts[v] = -1 self.links_to_split[link] = livecounts - block.operations[:] = self.llops - self.llops = None + block.operations[:] = llops self.livevars = None self.var_last_needed_in = None @@ -294,22 +299,18 @@ v_old = hop.genop('g' + opname[1:], hop.inputargs()[:-1], resulttype=v_new.concretetype) - self.push_alive(v_new) + self.push_alive(v_new, hop.llops) hop.rename('bare_' + opname) - self.pop_alive(v_old) + self.pop_alive(v_old, hop.llops) - def push_alive(self, var, llops=None): - if llops is None: - llops = self.llops + def push_alive(self, var, llops): if var_ispyobj(var): self.push_alive_pyobj(var, llops) else: self.push_alive_nopyobj(var, llops) - def pop_alive(self, var, llops=None): - if llops is None: - llops = self.llops + def pop_alive(self, var, llops): if var_ispyobj(var): self.pop_alive_pyobj(var, llops) else: @@ -358,10 +359,10 @@ BaseGCTransformer.__init__(self, parenttransformer.translator) self.parenttransformer = parenttransformer - def push_alive(self, var, llops=None): + def push_alive(self, var, llops): pass - def pop_alive(self, var, llops=None): + def pop_alive(self, var, llops): pass def gct_malloc(self, hop): From arigo at codespeak.net Thu Oct 18 19:50:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 18 Oct 2007 19:50:05 +0200 (CEST) Subject: [pypy-svn] r47560 - in pypy/dist/pypy/rpython/memory: gctransform test Message-ID: <20071018175005.4F9E380F2@code0.codespeak.net> Author: arigo Date: Thu Oct 18 19:50:05 2007 New Revision: 47560 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: Transformed-GC test for the GenerationGC. Fix bogus call to write_barrier(). Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Thu Oct 18 19:50:05 2007 @@ -618,6 +618,7 @@ v_fieldaddr = hop.genop("adr_add", [v_structaddr, v_currentofs], resulttype = llmemory.Address) hop.genop("direct_call", [self.write_barrier_ptr, + self.c_const_gc, v_newvalue, v_fieldaddr, v_structaddr]) Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Thu Oct 18 19:50:05 2007 @@ -395,6 +395,37 @@ res = run([]) assert res == 0 + +class GenericMovingGCTests(GenericGCTests): + def test_many_ids(self): + py.test.skip("fails for bad reasons in lltype.py :-(") + class A(object): + pass + def f(): + from pypy.rpython.lltypesystem import lltype, rffi + alist = [A() for i in range(50)] + idarray = lltype.malloc(rffi.INTP.TO, len(alist), flavor='raw') + # Compute the id of all the elements of the list. The goal is + # to not allocate memory, so that if the GC needs memory to + # remember the ids, it will trigger some collections itself + i = 0 + while i < len(alist): + idarray[i] = id(alist[i]) + i += 1 + j = 0 + while j < 2: + if j == 1: # allocate some stuff between the two iterations + [A() for i in range(20)] + i = 0 + while i < len(alist): + assert idarray[i] == id(alist[i]) + i += 1 + j += 1 + lltype.free(idarray, flavor='raw') + run = self.runner(f) + run([]) + + class TestMarkSweepGC(GenericGCTests): gcname = "marksweep" class gcpolicy(gc.FrameworkGcPolicy): @@ -658,7 +689,7 @@ GC_PARAMS = {'start_heap_size': 4096 } root_stack_depth = 200 -class TestSemiSpaceGC(GenericGCTests): +class TestSemiSpaceGC(GenericMovingGCTests): gcname = "semispace" class gcpolicy(gc.FrameworkGcPolicy): @@ -667,30 +698,13 @@ GC_PARAMS = {'space_size': 2048} root_stack_depth = 200 - def test_many_ids(self): - py.test.skip("fails for bad reasons in lltype.py :-(") - class A(object): - pass - def f(): - from pypy.rpython.lltypesystem import lltype, rffi - alist = [A() for i in range(50)] - idarray = lltype.malloc(rffi.INTP.TO, len(alist), flavor='raw') - # Compute the id of all the elements of the list. The goal is - # to not allocate memory, so that if the GC needs memory to - # remember the ids, it will trigger some collections itself - i = 0 - while i < len(alist): - idarray[i] = id(alist[i]) - i += 1 - j = 0 - while j < 2: - if j == 1: # allocate some stuff between the two iterations - [A() for i in range(20)] - i = 0 - while i < len(alist): - assert idarray[i] == id(alist[i]) - i += 1 - j += 1 - lltype.free(idarray, flavor='raw') - run = self.runner(f) - run([]) +class TestGenerationGC(GenericMovingGCTests): + gcname = "generation" + + class gcpolicy(gc.FrameworkGcPolicy): + class transformerclass(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc.generation import GenerationGC as \ + GCClass + GC_PARAMS = {'space_size': 2048, + 'nursery_size': 128} + root_stack_depth = 200 From mwh at codespeak.net Thu Oct 18 22:04:38 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Thu, 18 Oct 2007 22:04:38 +0200 (CEST) Subject: [pypy-svn] r47565 - pypy/dist/pypy/translator/goal Message-ID: <20071018200438.AA6A480F2@code0.codespeak.net> Author: mwh Date: Thu Oct 18 22:04:38 2007 New Revision: 47565 Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py Log: moving config targets r' us Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py ============================================================================== --- pypy/dist/pypy/translator/goal/bench-cronjob.py (original) +++ pypy/dist/pypy/translator/goal/bench-cronjob.py Thu Oct 18 22:04:38 2007 @@ -176,15 +176,15 @@ def main(backends=[]): if backends == []: #_ prefix means target specific option, # prefix to outcomment backends = [backend.strip() for backend in """ - #llvm--_faassen + llvm--_faassen c c--stackless--_faassen c--_faassen c--thread c--_objspace=taint c--_allworkingmodules - c--gcframework=marksweep--_faassen - c--gcframework=semispace--_faassen + c--gc=marksweep--_faassen + c--gc=semispace--_faassen c--_objspace-std-withrope cli """.split('\n') if backend.strip() and not backend.strip().startswith('#')] From cfbolz at codespeak.net Thu Oct 18 22:43:41 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 18 Oct 2007 22:43:41 +0200 (CEST) Subject: [pypy-svn] r47566 - pypy/dist/pypy/rpython/memory/test Message-ID: <20071018204341.A304E8128@code0.codespeak.net> Author: cfbolz Date: Thu Oct 18 22:43:40 2007 New Revision: 47566 Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: a GC tests involving instances. It fails for a strange reason on the StacklessGC MarkSweep version. Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Thu Oct 18 22:43:40 2007 @@ -91,6 +91,31 @@ else: return -1 # xxx + def test_instances(self): + class A(object): + pass + class B(A): + def __init__(self, something): + self.something = something + def malloc_a_lot(): + i = 0 + first = None + while i < 10: + i += 1 + a = somea = A() + a.last = first + first = a + j = 0 + while j < 30: + b = B(somea) + b.last = first + j += 1 + return 0 + run, statistics = self.runner(malloc_a_lot, statistics=True) + run([]) + heap_size = self.heap_usage(statistics) + + def test_llinterp_lists(self): def malloc_a_lot(): i = 0 @@ -659,6 +684,9 @@ GC_PARAMS = {'start_heap_size': 4096 } root_stack_depth = 200 + def test_instances(self): + py.test.skip("fails for a stupid reasons") + def test_x_become(self): from pypy.rlib import objectmodel S = lltype.GcStruct("S", ('x', lltype.Signed)) From antocuni at codespeak.net Fri Oct 19 00:06:13 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 19 Oct 2007 00:06:13 +0200 (CEST) Subject: [pypy-svn] r47568 - pypy/extradoc/talk/dls2007 Message-ID: <20071018220613.31B998108@code0.codespeak.net> Author: antocuni Date: Fri Oct 19 00:06:12 2007 New Revision: 47568 Modified: pypy/extradoc/talk/dls2007/rpython-talk.txt Log: the talk should be complete now Modified: pypy/extradoc/talk/dls2007/rpython-talk.txt ============================================================================== --- pypy/extradoc/talk/dls2007/rpython-talk.txt (original) +++ pypy/extradoc/talk/dls2007/rpython-talk.txt Fri Oct 19 00:06:12 2007 @@ -61,7 +61,7 @@ Type inference ============== -- Top-down, starting from an entry point +- Top-down, starting from an entry point; whole program analysis - Assign the most precise type to each variable @@ -200,3 +200,92 @@ * C, CLI (.NET), JVM + +Benchmarks +========== + +- Classic Martin Richard's test + +- Available in Java, C#, RPython + + =========== ============ ========= + Language Result Factor + =========== ============ ========= + C# 6.94 ms 1.00x + RPython 7.25 ms 1.04x + IronPython 1675.00 ms 241.35x + ----------- ------------ --------- + Java 1.77 ms 1.00x + RPython 2.10 ms 1.18x + Jython 2918.90 ms 1641.80x + =========== ============ ========= + + +What's good about RPython +========================= + +- Pythonic enough to be usable + +- Very fast + +- Testable under CPython + + +Things to improve +================= + +- Originally an implementation detail + +- Not designed to be user-friendly; crazy error messages + +- Lack of documentation/reference manual + +- Lack of separate compilation + +- Integration with the hosting platform + + * Good for C/Posix + + * Proof of concept for .NET + + * Doesn't exist for JVM + + +About PyPy +========== + +- Python in (R)Python + +- High level interpreter written in RPython + + * Easy to understand + + * Easy to extend + +- Translation toolchain (in full Python) + + * Works as a general compiler (e.g. Javascript, Prolog interpreter) + +- Low-level aspects inserted by the TT + + * Garbage collector + + * Threading model/Stackless + + * Additional language features + + * JIT compiler (only for the C backend so far) + + +Acknowledgments +=============== + +- The whole PyPy Team + + * RPython is **not** mine :-) + +- Davide Ancona + +- Massimo Ancona + +- Nicholas D. Matsakis From cfbolz at codespeak.net Fri Oct 19 00:13:18 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Oct 2007 00:13:18 +0200 (CEST) Subject: [pypy-svn] r47569 - pypy/extradoc/talk/dls2007 Message-ID: <20071018221318.E17EB8101@code0.codespeak.net> Author: cfbolz Date: Fri Oct 19 00:13:15 2007 New Revision: 47569 Modified: pypy/extradoc/talk/dls2007/rpython-talk.txt Log: like it. two small changes, feel free to revert Modified: pypy/extradoc/talk/dls2007/rpython-talk.txt ============================================================================== --- pypy/extradoc/talk/dls2007/rpython-talk.txt (original) +++ pypy/extradoc/talk/dls2007/rpython-talk.txt Fri Oct 19 00:13:15 2007 @@ -83,7 +83,7 @@ - ``yield`` and generators not supported -- No special __methods__ +- No special __methods__ (except __init__ and __del__) - No run-time definition of new functions and classes @@ -236,7 +236,7 @@ - Originally an implementation detail -- Not designed to be user-friendly; crazy error messages +- Not designed to be user-friendly; terse error messages - Lack of documentation/reference manual From cfbolz at codespeak.net Fri Oct 19 01:02:18 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Oct 2007 01:02:18 +0200 (CEST) Subject: [pypy-svn] r47570 - pypy/dist/pypy/rpython/memory/test Message-ID: <20071018230218.73A9F80FB@code0.codespeak.net> Author: cfbolz Date: Fri Oct 19 01:02:17 2007 New Revision: 47570 Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py Log: a failing test for the generational GC: there are several things wrong with the generational GC, this test was supposed to test one of them, but fails because of another one: - the debug_assert in collect_nursery can fail, when a collect_nursery is called, causes a full collect which executes finalizers, which allocate things in the nursery. This is what the test is supposed to show (and will, once the second problem is fixed, I guess). - the GC can crash if the fromspace _and_ the nursery are both full of surviving objects. Then a full collect cannot fit all those objects into the tospace! The checks in malloc_varsize and malloc_fixedsize (without fast) should trigger earlier. This is what the test is showing. Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Fri Oct 19 01:02:17 2007 @@ -286,3 +286,36 @@ class TestGenerationalGC(GCTest): from pypy.rpython.memory.gc.generation import GenerationGC as GCClass + + def test_finalizer_calls_malloc_during_minor_collect(self): + py.test.skip("fails") + class B(object): + pass + b = B() + b.nextid = 0 + b.num_deleted = 0 + b.all = [] + class A(object): + def __init__(self): + self.id = b.nextid + b.nextid += 1 + def __del__(self): + b.num_deleted += 1 + b.all.append(D(b.num_deleted)) + class D(object): + # make a big object that does not use malloc_varsize + def __init__(self, x): + self.x00 = self.x01 = self.x02 = self.x03 = self.x04 = x + self.x10 = self.x11 = self.x12 = self.x13 = self.x14 = x + self.x20 = self.x21 = self.x22 = self.x23 = self.x24 = x + def f(x): + i = 0 + all = [None] * x + a = A() + while i < x: + d = D(i) + all[i] = d + i += 1 + return b.num_deleted + len(all) + res = self.interpret(f, [500]) + assert res == 1 + 5 From arigo at codespeak.net Fri Oct 19 10:52:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 10:52:22 +0200 (CEST) Subject: [pypy-svn] r47571 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071019085222.D60998120@code0.codespeak.net> Author: arigo Date: Fri Oct 19 10:52:21 2007 New Revision: 47571 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: These obscure casts are now unnecessary (fixed in llmemory in rev 47434). Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Fri Oct 19 10:52:21 2007 @@ -652,12 +652,6 @@ c_type = rmodel.inputconst(lltype.Void, llmemory.Address) for k,var in enumerate(livevars): c_k = rmodel.inputconst(lltype.Signed, k) - # this cast doesn't actually do anything after translation to C - # however, on top of the llinterp it is needed - # see comment in llmemory before cast_ptr_to_weakrefptr - if var.concretetype == llmemory.WeakRefPtr: - var = hop.llops.genop("cast_weakrefptr_to_ptr", [var], - resulttype=WEAKREFPTR) v_adr = gen_cast(hop.llops, llmemory.Address, var) hop.genop("raw_store", [base_addr, c_type, c_k, v_adr]) return livevars @@ -677,18 +671,6 @@ c_k = rmodel.inputconst(lltype.Signed, k) v_newaddr = hop.genop("raw_load", [base_addr, c_type, c_k], resulttype=llmemory.Address) - if var.concretetype == llmemory.WeakRefPtr: - # these casts doesn't actually do anything after - # translation to C however, on top of the llinterp it is - # needed see comment in llmemory before - # cast_ptr_to_weakrefptr - # XXX not nice - nvar = hop.llops.genop("cast_adr_to_ptr", [v_newaddr], - resulttype=WEAKREFPTR) - nvar = hop.llops.genop("cast_ptr_to_weakrefptr", [nvar], - resulttype=llmemory.WeakRefPtr) - v_newaddr = hop.llops.genop("cast_ptr_to_adr", [nvar], - resulttype=llmemory.Address) hop.genop("gc_reload_possibly_moved", [v_newaddr, var]) def compute_borrowed_vars(self, graph): From cfbolz at codespeak.net Fri Oct 19 10:54:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Oct 2007 10:54:22 +0200 (CEST) Subject: [pypy-svn] r47572 - in pypy/dist/pypy/rpython/memory/gctransform: . test Message-ID: <20071019085422.278A38120@code0.codespeak.net> Author: cfbolz Date: Fri Oct 19 10:54:21 2007 New Revision: 47572 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py pypy/dist/pypy/rpython/memory/gctransform/transform.py Log: don't add a write barrier for setfield(X, 'name', Constant(...)). illegal for some strange write barriers, but I don't care right now. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Fri Oct 19 10:54:21 2007 @@ -572,11 +572,14 @@ hop.rename('cast_ptr_to_int') # works nicely for non-moving GCs def transform_generic_set(self, hop): - if self.write_barrier_ptr is None: + from pypy.objspace.flow.model import Constant + v_struct = hop.spaceop.args[0] + v_newvalue = hop.spaceop.args[-1] + # XXX for some GCs the skipping if the newvalue is a constant won't be + # ok + if self.write_barrier_ptr is None or isinstance(v_newvalue, Constant): super(FrameworkGCTransformer, self).transform_generic_set(hop) else: - v_struct = hop.spaceop.args[0] - v_newvalue = hop.spaceop.args[-1] assert isinstance(v_newvalue.concretetype, lltype.Ptr) opname = hop.spaceop.opname assert opname in ('setfield', 'setarrayitem', 'setinteriorfield') @@ -609,8 +612,6 @@ v_currentofs = hop.genop("int_add", [v_currentofs, v_offset], resulttype = lltype.Signed) - # XXX for some GCs we could skip the write_barrier if v_newvalue - # is a constant v_newvalue = hop.genop("cast_ptr_to_adr", [v_newvalue], resulttype = llmemory.Address) v_structaddr = hop.genop("cast_ptr_to_adr", [v_struct], Modified: pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py Fri Oct 19 10:54:21 2007 @@ -66,7 +66,7 @@ class GCClass(MarkSweepGC): needs_write_barrier = True -def write_barrier_check(spaceop): +def write_barrier_check(spaceop, needs_write_barrier=True): t = TranslationContext() t.buildannotator().build_types(lambda x:x, [SomeInteger()]) t.buildrtyper().specialize() @@ -80,7 +80,7 @@ print '\t', op if op.opname == 'direct_call': found = True - assert found + assert found == needs_write_barrier def test_write_barrier_support_setfield(): PTR_TYPE2 = lltype.Ptr(lltype.GcStruct('T', ('y', lltype.Signed))) @@ -91,6 +91,16 @@ varoftype(PTR_TYPE2)], varoftype(lltype.Void))) + +def test_dont_add_write_barrier_for_constant_new_value(): + PTR_TYPE2 = lltype.Ptr(lltype.GcStruct('T', ('y', lltype.Signed))) + PTR_TYPE = lltype.Ptr(lltype.GcStruct('S', ('x', PTR_TYPE2))) + write_barrier_check(SpaceOperation( + "setfield", + [varoftype(PTR_TYPE), Constant('x', lltype.Void), + Constant('foo', varoftype(PTR_TYPE2))], + varoftype(lltype.Void)), needs_write_barrier=False) + def test_write_barrier_support_setarrayitem(): PTR_TYPE2 = lltype.Ptr(lltype.GcStruct('T', ('y', lltype.Signed))) ARRAYPTR = lltype.Ptr(lltype.GcArray(PTR_TYPE2)) Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Fri Oct 19 10:54:21 2007 @@ -179,7 +179,7 @@ if not var_needsgc(var): continue self.var_last_needed_in[var] = len(block.operations) + 1 - + for i, op in enumerate(block.operations): hop = GcHighLevelOp(self, op, i, llops) hop.dispatch() From arigo at codespeak.net Fri Oct 19 13:16:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 13:16:11 +0200 (CEST) Subject: [pypy-svn] r47575 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20071019111611.6F4B78129@code0.codespeak.net> Author: arigo Date: Fri Oct 19 13:16:09 2007 New Revision: 47575 Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py Log: Support for partial arena_reset() that only clear a subrange of the whole arena. The non-translated version can check that no existing object crosses the start or the end of the cleared range. Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llarena.py Fri Oct 19 13:16:09 2007 @@ -24,19 +24,30 @@ self.freed = False self.reset(zero) - def reset(self, zero): + def reset(self, zero, start=0, size=None): self.check() - for ptr in self.objectptrs.itervalues(): - obj = ptr._obj - obj._free() - del Arena.object_arena_location[obj] - self.objectptrs.clear() - self.objectsizes.clear() + if size is None: + stop = self.nbytes + else: + stop = start + size + assert 0 <= start <= stop <= self.nbytes + for offset, ptr in self.objectptrs.items(): + size = self.objectsizes[offset] + if offset < start: # object is before the cleared area + assert offset + size <= start, "object overlaps cleared area" + elif offset + size > stop: # object is after the cleared area + assert offset >= stop, "object overlaps cleared area" + else: + obj = ptr._obj + obj._free() + del Arena.object_arena_location[obj] + del self.objectptrs[offset] + del self.objectsizes[offset] if zero: initialbyte = "0" else: initialbyte = "#" - self.usagemap[:] = array.array('c', initialbyte * self.nbytes) + self.usagemap[start:stop] = array.array('c', initialbyte*(stop-start)) def check(self): if self.freed: @@ -208,13 +219,12 @@ arena_addr.arena.reset(False) arena_addr.arena.freed = True -def arena_reset(arena_addr, myarenasize, zero): +def arena_reset(arena_addr, size, zero): """Free all objects in the arena, which can then be reused. - The arena is filled with zeroes if 'zero' is True.""" + The arena is filled with zeroes if 'zero' is True. This can also + be used on a subrange of the arena.""" assert isinstance(arena_addr, fakearenaaddress) - assert arena_addr.offset == 0 - assert myarenasize == arena_addr.arena.nbytes - arena_addr.arena.reset(zero) + arena_addr.arena.reset(zero, arena_addr.offset, size) def arena_reserve(addr, size, check_alignment=True): """Mark some bytes in an arena as reserved, and returns addr. @@ -295,9 +305,9 @@ llfakeimpl=arena_free, sandboxsafe=True) -def llimpl_arena_reset(arena_addr, myarenasize, zero): +def llimpl_arena_reset(arena_addr, size, zero): if zero: - clear_large_memory_chunk(arena_addr, myarenasize) + clear_large_memory_chunk(arena_addr, size) register_external(arena_reset, [llmemory.Address, int, bool], None, 'll_arena.arena_reset', llimpl=llimpl_arena_reset, Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py Fri Oct 19 13:16:09 2007 @@ -118,6 +118,7 @@ precomputed_size = round_up_for_allocation(llmemory.sizeof(SX)) def test_look_inside_object(): + # this code is also used in translation tests below myarenasize = 50 a = arena_malloc(myarenasize, False) b = a + round_up_for_allocation(llmemory.sizeof(lltype.Char)) @@ -132,6 +133,47 @@ arena_free(a) return 42 +def test_partial_arena_reset(): + a = arena_malloc(50, False) + def reserve(i): + b = a + i * llmemory.raw_malloc_usage(precomputed_size) + arena_reserve(b, precomputed_size) + return b + blist = [] + plist = [] + for i in range(4): + b = reserve(i) + (b + llmemory.offsetof(SX, 'x')).signed[0] = 100 + i + blist.append(b) + plist.append(llmemory.cast_adr_to_ptr(b, SPTR)) + # clear blist[1] and blist[2] but not blist[0] nor blist[3] + arena_reset(blist[1], llmemory.raw_malloc_usage(precomputed_size)*2, False) + # re-reserve object at index 1 and 2 + blist[1] = reserve(1) + blist[2] = reserve(2) + # check via object pointers + assert plist[0].x == 100 + assert plist[3].x == 103 + py.test.raises(RuntimeError, "plist[1].x") # marked as freed + py.test.raises(RuntimeError, "plist[2].x") # marked as freed + # check via addresses + assert (blist[0] + llmemory.offsetof(SX, 'x')).signed[0] == 100 + assert (blist[3] + llmemory.offsetof(SX, 'x')).signed[0] == 103 + py.test.raises(lltype.UninitializedMemoryAccess, + "(blist[1] + llmemory.offsetof(SX, 'x')).signed[0]") + py.test.raises(lltype.UninitializedMemoryAccess, + "(blist[2] + llmemory.offsetof(SX, 'x')).signed[0]") + # clear and zero-fill the area over blist[0] and blist[1] + arena_reset(blist[0], llmemory.raw_malloc_usage(precomputed_size)*2, True) + # re-reserve and check it's zero + blist[0] = reserve(0) + blist[1] = reserve(1) + assert (blist[0] + llmemory.offsetof(SX, 'x')).signed[0] == 0 + assert (blist[1] + llmemory.offsetof(SX, 'x')).signed[0] == 0 + assert (blist[3] + llmemory.offsetof(SX, 'x')).signed[0] == 103 + py.test.raises(lltype.UninitializedMemoryAccess, + "(blist[2] + llmemory.offsetof(SX, 'x')).signed[0]") + def test_address_eq_as_int(): a = arena_malloc(50, False) arena_reserve(a, precomputed_size) From arigo at codespeak.net Fri Oct 19 13:22:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 13:22:04 +0200 (CEST) Subject: [pypy-svn] r47576 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071019112204.610368129@code0.codespeak.net> Author: arigo Date: Fri Oct 19 13:22:04 2007 New Revision: 47576 Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py Log: for the convenience of debugging the GCs, NULL compares as the smallest address. Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Fri Oct 19 13:22:04 2007 @@ -353,7 +353,16 @@ return NotImplemented def __lt__(self, other): - raise TypeError("cannot compare fakeaddresses with '<'") + # for the convenience of debugging the GCs, NULL compares as the + # smallest address even when compared with a non-fakearenaaddress + if not isinstance(other, fakeaddress): + raise TypeError("cannot compare fakeaddress and %r" % ( + other.__class__.__name__,)) + if not other: + return False # self < NULL => False + if not self: + return True # NULL < non-null-other => True + raise TypeError("cannot compare non-NULL fakeaddresses with '<'") def __le__(self, other): return self == other or self < other def __gt__(self, other): From arigo at codespeak.net Fri Oct 19 13:54:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 13:54:50 +0200 (CEST) Subject: [pypy-svn] r47577 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20071019115450.19551812D@code0.codespeak.net> Author: arigo Date: Fri Oct 19 13:54:49 2007 New Revision: 47577 Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py Log: More hack hack hacking to attempt to order fakeaddresses that ultimately come from an arena. Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llarena.py Fri Oct 19 13:54:49 2007 @@ -14,7 +14,7 @@ pass class Arena(object): - object_arena_location = {} # {container: (arena, offset)} + object_arena_location = {} # {topcontainer: (arena, offset)} def __init__(self, nbytes, zero): self.nbytes = nbytes @@ -39,10 +39,10 @@ assert offset >= stop, "object overlaps cleared area" else: obj = ptr._obj - obj._free() - del Arena.object_arena_location[obj] + del Arena.object_arena_location[lltype.top_container(obj)] del self.objectptrs[offset] del self.objectsizes[offset] + obj._free() if zero: initialbyte = "0" else: @@ -81,7 +81,8 @@ self.usagemap[offset:offset+bytes] = array.array('c', pattern) self.objectptrs[offset] = addr2.ptr self.objectsizes[offset] = bytes - Arena.object_arena_location[addr2.ptr._obj] = self, offset + top = lltype.top_container(addr2.ptr._obj) + Arena.object_arena_location[top] = self, offset # common case: 'size' starts with a GCHeaderOffset. In this case # we can also remember that the real object starts after the header. while isinstance(size, RoundedUpForAllocation): @@ -94,7 +95,8 @@ assert objoffset not in self.objectptrs self.objectptrs[objoffset] = objaddr.ptr self.objectsizes[objoffset] = bytes - hdrbytes - Arena.object_arena_location[objaddr.ptr._obj] = self, objoffset + top = lltype.top_container(objaddr.ptr._obj) + Arena.object_arena_location[top] = self, objoffset return addr2 class fakearenaaddress(llmemory.fakeaddress): @@ -147,28 +149,53 @@ def __nonzero__(self): return True + def compare_with_fakeaddr(self, other): + if not other: + return None, None + obj = other.ptr._obj + top = lltype.top_container(obj) + if top not in Arena.object_arena_location: + return None, None + arena, offset = Arena.object_arena_location[top] + # common case: top is a FixedSizeArray of size 1 with just obj in it + T = lltype.typeOf(top) + if (top is obj or (isinstance(T, lltype.FixedSizeArray) and + T.OF == lltype.typeOf(obj))): + # in this case, addr(obj) == addr(top) + pass + else: + # here, it's likely that addr(obj) is a bit larger than addr(top). + # We could estimate the correct offset but it's a bit messy; + # instead, let's check the answer doesn't depend on it + if self.arena is arena: + objectsize = arena.objectsizes[offset] + if offset < self.offset < offset+objectsize: + raise AssertionError( + "comparing an inner address with a " + "fakearenaaddress that points in the " + "middle of the same object") + offset += objectsize // 2 # arbitrary + return arena, offset + def __eq__(self, other): if isinstance(other, fakearenaaddress): - return self.arena is other.arena and self.offset == other.offset + arena = other.arena + offset = other.offset elif isinstance(other, llmemory.fakeaddress): - if other.ptr and other.ptr._obj in Arena.object_arena_location: - arena, offset = Arena.object_arena_location[other.ptr._obj] - return self.arena is arena and self.offset == offset - else: - return False + arena, offset = self.compare_with_fakeaddr(other) else: return llmemory.fakeaddress.__eq__(self, other) + return self.arena is arena and self.offset == offset def __lt__(self, other): if isinstance(other, fakearenaaddress): arena = other.arena offset = other.offset elif isinstance(other, llmemory.fakeaddress): - if other.ptr and other.ptr._obj in Arena.object_arena_location: - arena, offset = Arena.object_arena_location[other.ptr._obj] - else: - # arbitrarily, 'self' > any address not in any arena - return False + arena, offset = self.compare_with_fakeaddr(other) + if arena is None: + return False # self < other-not-in-any-arena => False + # (arbitrarily) else: raise TypeError("comparing a %s and a %s" % ( self.__class__.__name__, other.__class__.__name__)) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py Fri Oct 19 13:54:49 2007 @@ -112,8 +112,25 @@ assert lt(a+19, b) assert lt(a, b+19) + c = b + round_up_for_allocation(llmemory.sizeof(lltype.Char)) + arena_reserve(c, precomputed_size) + assert lt(b, c) + assert lt(a, c) + assert lt(llmemory.NULL, c) + d = c + llmemory.offsetof(SX, 'x') + assert lt(c, d) + assert lt(b, d) + assert lt(a, d) + assert lt(llmemory.NULL, d) + e = c + precomputed_size + assert lt(d, e) + assert lt(c, e) + assert lt(b, e) + assert lt(a, e) + assert lt(llmemory.NULL, e) -SX = lltype.Struct('S', ('x',lltype.Signed)) + +SX = lltype.Struct('S', ('foo',lltype.Signed), ('x',lltype.Signed)) SPTR = lltype.Ptr(SX) precomputed_size = round_up_for_allocation(llmemory.sizeof(SX)) From arigo at codespeak.net Fri Oct 19 13:57:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 13:57:12 +0200 (CEST) Subject: [pypy-svn] r47578 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071019115712.D86778134@code0.codespeak.net> Author: arigo Date: Fri Oct 19 13:57:12 2007 New Revision: 47578 Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py Log: Fix. Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llarena.py Fri Oct 19 13:57:12 2007 @@ -160,7 +160,7 @@ # common case: top is a FixedSizeArray of size 1 with just obj in it T = lltype.typeOf(top) if (top is obj or (isinstance(T, lltype.FixedSizeArray) and - T.OF == lltype.typeOf(obj))): + top.getitem(0) is obj)): # in this case, addr(obj) == addr(top) pass else: From arigo at codespeak.net Fri Oct 19 14:19:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 14:19:00 +0200 (CEST) Subject: [pypy-svn] r47579 - in pypy/dist/pypy/rpython: . memory Message-ID: <20071019121900.B11D28129@code0.codespeak.net> Author: arigo Date: Fri Oct 19 14:18:58 2007 New Revision: 47579 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/memory/gcwrapper.py Log: This fixes the tracer for destructor invocations from gcwrapper. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Fri Oct 19 14:18:58 2007 @@ -57,11 +57,12 @@ if tracing: self.tracer = Tracer() - def eval_graph(self, graph, args=()): + def eval_graph(self, graph, args=(), recursive=False): llframe = self.frame_class(graph, args, self) - if self.tracer: + if self.tracer and not recursive: self.tracer.start() retval = None + old_active_frame = self.active_frame try: try: retval = llframe.eval() @@ -82,10 +83,12 @@ self.tracer.dump(line + '\n') raise finally: + self.active_frame = old_active_frame if self.tracer: if retval is not None: self.tracer.dump(' ---> %r\n' % (retval,)) - self.tracer.stop() + if not recursive: + self.tracer.stop() return retval def print_traceback(self): Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Fri Oct 19 14:18:58 2007 @@ -122,16 +122,12 @@ assert not type_contains_pyobjs(TYPE), "not implemented" def ll_finalizer(addr): - old_active_frame = self.llinterp.active_frame try: - try: - v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) - self.llinterp.eval_graph(destrgraph, [v]) - except llinterp.LLException: - raise RuntimeError( - "a finalizer raised an exception, shouldn't happen") - finally: - self.llinterp.active_frame = old_active_frame + v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) + self.llinterp.eval_graph(destrgraph, [v], recursive=True) + except llinterp.LLException: + raise RuntimeError( + "a finalizer raised an exception, shouldn't happen") return ll_finalizer From arigo at codespeak.net Fri Oct 19 14:19:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 14:19:26 +0200 (CEST) Subject: [pypy-svn] r47580 - pypy/dist/pypy/rpython/memory/test Message-ID: <20071019121926.EB6B38129@code0.codespeak.net> Author: arigo Date: Fri Oct 19 14:19:26 2007 New Revision: 47580 Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py Log: Move the test in the base class. It fails also with SemiSpaceGC... Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Fri Oct 19 14:19:26 2007 @@ -274,21 +274,9 @@ res = self.interpret(f, []) assert res == 0 - -class TestMarkSweepGC(GCTest): - from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass - -class TestSemiSpaceGC(GCTest): - from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass - -class TestGrowingSemiSpaceGC(TestSemiSpaceGC): - GC_PARAMS = {'space_size': 64} - -class TestGenerationalGC(GCTest): - from pypy.rpython.memory.gc.generation import GenerationGC as GCClass - def test_finalizer_calls_malloc_during_minor_collect(self): - py.test.skip("fails") + # originally a GenerationGC test, this has also found bugs in other GCs + py.test.skip("in-progress") class B(object): pass b = B() @@ -318,4 +306,17 @@ i += 1 return b.num_deleted + len(all) res = self.interpret(f, [500]) - assert res == 1 + 5 + assert res == 1 + 500 + + +class TestMarkSweepGC(GCTest): + from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass + +class TestSemiSpaceGC(GCTest): + from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass + +class TestGrowingSemiSpaceGC(TestSemiSpaceGC): + GC_PARAMS = {'space_size': 64} + +class TestGenerationalGC(GCTest): + from pypy.rpython.memory.gc.generation import GenerationGC as GCClass From arigo at codespeak.net Fri Oct 19 14:52:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 14:52:05 +0200 (CEST) Subject: [pypy-svn] r47581 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071019125205.90B498134@code0.codespeak.net> Author: arigo Date: Fri Oct 19 14:52:05 2007 New Revision: 47581 Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py Log: Found an alternative to lltype.top_container() allowing a couple of hackish lines to be removed. Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llarena.py Fri Oct 19 14:52:05 2007 @@ -14,7 +14,7 @@ pass class Arena(object): - object_arena_location = {} # {topcontainer: (arena, offset)} + object_arena_location = {} # {container: (arena, offset)} def __init__(self, nbytes, zero): self.nbytes = nbytes @@ -39,7 +39,7 @@ assert offset >= stop, "object overlaps cleared area" else: obj = ptr._obj - del Arena.object_arena_location[lltype.top_container(obj)] + del Arena.object_arena_location[obj] del self.objectptrs[offset] del self.objectsizes[offset] obj._free() @@ -81,8 +81,7 @@ self.usagemap[offset:offset+bytes] = array.array('c', pattern) self.objectptrs[offset] = addr2.ptr self.objectsizes[offset] = bytes - top = lltype.top_container(addr2.ptr._obj) - Arena.object_arena_location[top] = self, offset + Arena.object_arena_location[addr2.ptr._obj] = self, offset # common case: 'size' starts with a GCHeaderOffset. In this case # we can also remember that the real object starts after the header. while isinstance(size, RoundedUpForAllocation): @@ -95,8 +94,7 @@ assert objoffset not in self.objectptrs self.objectptrs[objoffset] = objaddr.ptr self.objectsizes[objoffset] = bytes - hdrbytes - top = lltype.top_container(objaddr.ptr._obj) - Arena.object_arena_location[top] = self, objoffset + Arena.object_arena_location[objaddr.ptr._obj] = self, objoffset return addr2 class fakearenaaddress(llmemory.fakeaddress): @@ -153,18 +151,17 @@ if not other: return None, None obj = other.ptr._obj - top = lltype.top_container(obj) - if top not in Arena.object_arena_location: - return None, None - arena, offset = Arena.object_arena_location[top] - # common case: top is a FixedSizeArray of size 1 with just obj in it - T = lltype.typeOf(top) - if (top is obj or (isinstance(T, lltype.FixedSizeArray) and - top.getitem(0) is obj)): - # in this case, addr(obj) == addr(top) - pass - else: - # here, it's likely that addr(obj) is a bit larger than addr(top). + innerobject = False + while obj not in Arena.object_arena_location: + obj = obj._parentstructure() + if obj is None: + return None, None # not found in the arena + innerobject = True + arena, offset = Arena.object_arena_location[obj] + if innerobject: + # 'obj' is really inside the object allocated from the arena, + # so it's likely that its address "should be" a bit larger than + # what 'offset' says. # We could estimate the correct offset but it's a bit messy; # instead, let's check the answer doesn't depend on it if self.arena is arena: From fijal at codespeak.net Fri Oct 19 14:58:02 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Oct 2007 14:58:02 +0200 (CEST) Subject: [pypy-svn] r47582 - in pypy/dist/pypy/module/_ffi: . test Message-ID: <20071019125802.808548136@code0.codespeak.net> Author: fijal Date: Fri Oct 19 14:58:00 2007 New Revision: 47582 Modified: pypy/dist/pypy/module/_ffi/array.py pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: Add array pre-initialization. Segfaults occasionally Modified: pypy/dist/pypy/module/_ffi/array.py ============================================================================== --- pypy/dist/pypy/module/_ffi/array.py (original) +++ pypy/dist/pypy/module/_ffi/array.py Fri Oct 19 14:58:00 2007 @@ -24,7 +24,14 @@ get_elem._annspecialcase_ = 'specialize:arg(2)' class W_ArrayInstance(Wrappable): - def __init__(self, space, of, length): + def __init__(self, space, of, w_length_or_iterable): + self.ll_array = lltype.nullptr(rffi.VOIDP.TO) + if space.is_true(space.isinstance(w_length_or_iterable, space.w_int)): + length = space.int_w(w_length_or_iterable) + items_w = None + else: + items_w = space.unpackiterable(w_length_or_iterable) + length = len(items_w) self.alloced = False self.of = of _get_type(space, of) @@ -32,7 +39,11 @@ size = native_fmttable[of]['size'] * length self.ll_array = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', zero=True) - self.alloced = True + if items_w: + for num in range(len(items_w)): + w_item = items_w[num] + unwrap_value(space, push_elem, self.ll_array, num, self.of, + w_item, None) # XXX don't allow negative indexes, nor slices @@ -55,12 +66,12 @@ return space.wrap(rffi.cast(rffi.INT, self.ll_array)) def __del__(self): - if self.alloced: + if self.ll_array: lltype.free(self.ll_array, flavor='raw') -def descr_new_array_instance(space, w_type, of, size): - return space.wrap(W_ArrayInstance(space, of, size)) -descr_new_array_instance.unwrap_spec = [ObjSpace, W_Root, str, int] +def descr_new_array_instance(space, w_type, of, w_size_or_iterable): + return space.wrap(W_ArrayInstance(space, of, w_size_or_iterable)) +descr_new_array_instance.unwrap_spec = [ObjSpace, W_Root, str, W_Root] W_ArrayInstance.typedef = TypeDef( 'ArrayInstance', Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Fri Oct 19 14:58:00 2007 @@ -231,6 +231,9 @@ assert get_array_elem(a, 7) == 1 assert get_array_elem(a, 6) == 2 assert a[3] == 0 + a = A([1, 2, 3, 4]) + assert get_array_elem(a, 0) == 1 + assert a[3] == 4 def test_array_of_structure(self): import _ffi @@ -289,3 +292,15 @@ pass_ll = lib.ptr('pass_ll', ['q'], 'q') assert pass_ll(1<<42) == 1<<42 + def test_callback(self): + skip("Segfaults") + import _ffi + libc = _ffi.CDLL('libc.so.6') + to_sort = "kljhgfa" + ll_to_sort = _ffi.Array('c')(to_sort) + qsort = libc.ptr('qsort', ['P', 'i', 'i', 'P'], None) + def compare(a, b): + return a < b + qsort(ll_to_sort, len(to_sort), 1, + CallbackPtr(compare, ['i', 'i'], 'i')) + From antocuni at codespeak.net Fri Oct 19 15:04:22 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 19 Oct 2007 15:04:22 +0200 (CEST) Subject: [pypy-svn] r47583 - pypy/extradoc/talk/dls2007 Message-ID: <20071019130422.7ABB8813D@code0.codespeak.net> Author: antocuni Date: Fri Oct 19 15:04:21 2007 New Revision: 47583 Added: pypy/extradoc/talk/dls2007/rpython-talk.latex Removed: pypy/extradoc/talk/dls2007/rpython-talk.txt Log: port the talk from rst to latex beamer Added: pypy/extradoc/talk/dls2007/rpython-talk.latex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/dls2007/rpython-talk.latex Fri Oct 19 15:04:21 2007 @@ -0,0 +1,390 @@ +\documentclass[14pt]{beamer} +\usepackage[latin1]{inputenc} + +% settings for code snippets +\usepackage{listings} +\usepackage{fancyvrb} +\lstset{language=Python, + basicstyle=\footnotesize\ttfamily, + frame=none, + stringstyle=\color{blue}, + fancyvrb=true, + xleftmargin=2pt,xrightmargin=2pt, + showstringspaces=false} + + +\usetheme{Boadilla} +%\usetheme{Warsaw} +\setbeamercovered{transparent} + +\title[RPython, a dynamic static language]{RPython\\A Step Towards Reconciling Dynamically and Statically Typed OO Languages} +\author[Antonio Cuni]{Antonio Cuni -- DISI, Universit? degli Studi di Genova} +\institute[DSL'07 OOPSLA]{DLS'07 OOPSLA Montreal CA} +\date{October 22, 2007} + +\begin{document} + +\begin{frame} + \titlepage +\end{frame} + +\begin{frame} + \frametitle{Dynamic languages for .NET and JVM} + \begin{itemize} + \item .NET and JVM: widespread platforms + \item Designed for static languages + \item Great Python implementations: \textbf{IronPython}, \textbf{Jython} + \item Much slower than e.g. C\# and Java + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Dynamic vs. static} + \begin{alertblock}{Dynamic languages} + \begin{itemize} + \item Flexibility + \item Rapid development cycle + \item \textbf{Metaprogramming} + \end{itemize} + \end{alertblock} + + \pause + \begin{alertblock}{Static languages} + \begin{itemize} + \item Speed + \item Nothing more :-) + \end{itemize} + \end{alertblock} +\end{frame} + + +\begin{frame} + \frametitle{RPython Quick Facts} + + \begin{itemize} + \item Restricted subset of Python + \item Statically typed (type inference) + \item Still allows metaprogramming + \item RPython programs still run under {\{}C,J,Iron{\}}Python + \item Three backends: C, .NET, JVM + \item Almost as fast as C, C{\#}, Java + \end{itemize} +\end{frame} + + +\begin{frame}[fragile] + \frametitle{Type inference} + \begin{itemize} + \item Top-down, starting from an entry point; whole program analysis + \item Assign the most precise type to each variable + \item Fail if you try to mix incompatible types + \end{itemize} + + \pause + + \begin{columns} + \begin{column}{0.45\textwidth} + \begin{exampleblock}{RPython} + \begin{lstlisting} +def main(): + print add(40, 2) + +def add(a, b): + return a+b + \end{lstlisting} + \end{exampleblock} + \end{column} + + \pause + + \begin{column}{0.45\textwidth} + \begin{exampleblock}{Not RPython} + \begin{lstlisting} +def fn(flag): + if flag: + return 42 + else: + return 'hello' + \end{lstlisting} + \end{exampleblock} + \end{column} + \end{columns} +\end{frame} + + +\begin{frame} + \frametitle{Other restrictions} + + \begin{itemize} + \item Globals are assumed to be constant + \item \texttt{yield} and generators not supported + \item No special \_\_methods\_\_ (except \_\_init\_\_ and \_\_del\_\_) + \item No run-time definition of new functions and classes + \item Cannot modify classes at run-time + \item Cannot change the {\_}{\_}class{\_}{\_} of an object + \item Single inheritance, with limited support for mixins + \end{itemize} +\end{frame} + + + \begin{frame} + \frametitle{Still pythonic, though} + + \begin{itemize} + \item No syntactic restriction + \item Functions and classes are first-order values + \item Exceptions work + + \pause + \begin{alertblock}{Lists and dictionaries} + \begin{itemize} + \item Work, but they must be homogeneous + \item list of int, dict from string to floats, etc. are OK + \item list of \emph{int and strings} is not + \item Most of methods of \texttt{list}, \texttt{dict} and + \texttt{str} are supported + \end{itemize} + \end{alertblock} + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Init-time, translation-time, run-time} + \begin{center} + \includegraphics[scale=0.45]{image/translation.pdf} + \end{center} +\end{frame} + + +\begin{frame} + \frametitle{Metaprogramming} + \begin{itemize} + \item RPython restrictions only apply to live objects + \item No restrictions about how they are created + \begin{itemize} + \item Full Python is allowed at init-time + \end{itemize} + \item Python as a metaprogramming language for RPython + \item Code generation considered harmful + \end{itemize} +\end{frame} + + +\begin{frame}[fragile] + \frametitle{Compute complex constants} + + \begin{exampleblock}{Fibonacci's numbers} + \begin{lstlisting} +def fibo(N): + sequence = [] + a, b = 1, 1 + for i in xrange(N): + sequence.append(a) + a, b = b, a+b + return sequence + +# computed at init-time +fibo_numbers = fibo(100) + \end{lstlisting} + \end{exampleblock} +\end{frame} + + +\begin{frame}[fragile] + \frametitle{Metaclasses run at init-time} + + \begin{exampleblock}{\texttt{\_\_extend\_\_} metaclass} + \begin{lstlisting} +class MyClass(object): + def foo(self): ... + +class __extend__(MyClass): + def bar(self): ... + +def main(): + obj = MyClass() + obj.bar() + \end{lstlisting} + \end{exampleblock} +\end{frame} + + + +\begin{frame}[fragile] + \frametitle{Dynamic classes/functions at init-time} + \begin{exampleblock}{``Static'' nested scopes work} + \begin{lstlisting} +def make_adder(N): + def add(x): + return x+N + return add + +add10 = make_adder(10) +add20 = make_adder(20) + +def main(): + print add10(32) + print add20(22) + \end{lstlisting} + \end{exampleblock} +\end{frame} + + +\begin{frame} + \frametitle{The Translation Toolchain} + \begin{itemize} + \item \textbf{CPython}: *.py -{}-{\textgreater} Python bytecode + \item \textbf{FlowObjSpace}: bytecode -{}-{\textgreater} flow graphs + \item \textbf{Annotator}: type inference on flow graphs + \begin{itemize} + \item High level Python types (\texttt{List(Integer)}) + \end{itemize} + \item \textbf{RTyper}: high level types -{\textgreater} low level types + \begin{itemize} + \item lltype for C, ootype for CLI and JVM + \end{itemize} + \item \textbf{Backends}: code generation + \begin{itemize} + \item C, CLI (.NET), JVM + \end{itemize} + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Benchmarks} + + \begin{itemize} + \item Classic Martin Richard's test + \item Available in Java, C{\#}, RPython + + \begin{table}[ht] + \begin{tabular}{|l|r|r|} + \hline + \textbf{Language} & + \textbf{Result} & + \textbf{Factor} \\ + \hline + \multicolumn{3}{|c|}{Results on Microsoft CLR}\\ + \hline + C\# & + 6.94 ms & + 1.00x \\ + + RPython & + 7.25 ms & + 1.04x \\ + + IronPython & + 1675.00 ms & + 241.35x \\ + \hline + + \multicolumn{3}{|c|}{Results on JVM}\\ + \hline + Java & + 1.77 ms & + 1.00x \\ + + RPython & + 2.10 ms & + 1.18x \\ + + Jython & + 2918.90 ms & + 1641.80x \\ + \hline + \end{tabular} + \end{table} + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{What's good about RPython} + \begin{itemize} + \item Pythonic enough to be usable + \item Very fast + \item Testable under CPython + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Things to improve} + \begin{itemize} + \item Originally an implementation detail + \item Not designed to be user-friendly; terse error messages + \item Lack of documentation/reference manual + \item Lack of separate compilation + \item Integration with the hosting platform + \begin{itemize} + \item Good for C/Posix + \item Proof of concept for .NET + \item Doesn't exist for JVM + \end{itemize} + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{About PyPy (1)} + + \begin{alertblock}{Pyton in (R)Python} + \begin{itemize} + \item High level interpreter written in RPython + \item Easy to understand + \item Easy to extend + \end{itemize} + \end{alertblock} + + \pause + \begin{alertblock}{Translation Toolchain} + \begin{itemize} + \item Written in full Python + \item Works as a general compiler + \item Especially for interpreters (e.g. Javascript, Prolog) + \end{itemize} + \end{alertblock} +\end{frame} + +\begin{frame} + \frametitle{About PyPy (2)} + + \begin{alertblock}{Low-level aspects inserted by the TT} + \begin{itemize} + \item Garbage collector + \item Threading model/Stackless + \item Additional language features + \item JIT compiler (only for the C backend so far) + \end{itemize} + \end{alertblock} + + \pause + \begin{alertblock}{PyPy you can get} + \begin{itemize} + \item pypy-c (about 2x slower than CPython) + \item pypy-c-jit (up to 60x \textbf{faster} than CPython) + \item pypy.NET (about 6x slower than IronPython) + \item pypy-jvm (about 30\% \textbf{faster} than Jython) + \end{itemize} + \end{alertblock} +\end{frame} + + +\begin{frame} + \frametitle{Acknowledgments} + \begin{itemize} + \item The whole PyPy Team + \begin{itemize} + \item RPython is \textbf{not} mine :-) + \end{itemize} + + \item Davide Ancona + \item Massimo Ancona + \item Nicholas D. Matsakis + \end{itemize} +\end{frame} + +\end{document} From fijal at codespeak.net Fri Oct 19 15:49:32 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Oct 2007 15:49:32 +0200 (CEST) Subject: [pypy-svn] r47585 - pypy/dist/pypy Message-ID: <20071019134932.3BD5E8133@code0.codespeak.net> Author: fijal Date: Fri Oct 19 15:49:28 2007 New Revision: 47585 Modified: pypy/dist/pypy/conftest.py Log: Adapt conftest.py to new py.test interface Modified: pypy/dist/pypy/conftest.py ============================================================================== --- pypy/dist/pypy/conftest.py (original) +++ pypy/dist/pypy/conftest.py Fri Oct 19 15:49:28 2007 @@ -282,10 +282,8 @@ _pygame_imported = False class IntTestFunction(PyPyTestFunction): - def _haskeyword(self, keyword): - if keyword == 'interplevel': - return True - return super(IntTestFunction, self)._haskeyword(keyword) + def _keywords(self): + return super(IntTestFunction, self)._keywords + ['interplevel'] def execute(self, target, *args): co = target.func_code @@ -315,8 +313,8 @@ "if conftest.option.view is False") class AppTestFunction(PyPyTestFunction): - def _haskeyword(self, keyword): - return keyword == 'applevel' or super(AppTestFunction, self)._haskeyword(keyword) + def _keywords(self): + return ['applevel'] + super(AppTestFunction, self)._keywords() def execute(self, target, *args): assert not args @@ -363,9 +361,8 @@ class IntClassCollector(PyPyClassCollector): Function = IntTestFunction - def _haskeyword(self, keyword): - return keyword == 'interplevel' or \ - super(IntClassCollector, self)._haskeyword(keyword) + def _keywords(self): + return super(IntClassCollector, self)._keywords() + ['interplevel'] class AppClassInstance(py.test.collect.Instance): Function = AppTestMethod @@ -383,9 +380,8 @@ class AppClassCollector(PyPyClassCollector): Instance = AppClassInstance - def _haskeyword(self, keyword): - return keyword == 'applevel' or \ - super(AppClassCollector, self)._haskeyword(keyword) + def _keywords(self): + return super(AppClassCollector, self)._keywords() + ['applevel'] def setup(self): super(AppClassCollector, self).setup() From fijal at codespeak.net Fri Oct 19 16:00:55 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Oct 2007 16:00:55 +0200 (CEST) Subject: [pypy-svn] r47586 - in pypy/dist/pypy/translator/js: . examples/bnb/data test Message-ID: <20071019140055.C6E848133@code0.codespeak.net> Author: fijal Date: Fri Oct 19 16:00:53 2007 New Revision: 47586 Modified: pypy/dist/pypy/translator/js/examples/bnb/data/bnb.html pypy/dist/pypy/translator/js/opcodes.py pypy/dist/pypy/translator/js/test/test_bltn.py Log: add add_ovf_* operations to js backends (rlist uses them) Modified: pypy/dist/pypy/translator/js/examples/bnb/data/bnb.html ============================================================================== --- pypy/dist/pypy/translator/js/examples/bnb/data/bnb.html (original) +++ pypy/dist/pypy/translator/js/examples/bnb/data/bnb.html Fri Oct 19 16:00:53 2007 @@ -4,7 +4,7 @@ Bub'n'bros - + Modified: pypy/dist/pypy/translator/js/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/js/opcodes.py (original) +++ pypy/dist/pypy/translator/js/opcodes.py Fri Oct 19 16:00:53 2007 @@ -34,6 +34,8 @@ opcodes = {'int_mul': '*', 'int_add': '+', + 'int_add_ovf': '+', # XXX overflow + 'int_add_ovf_nonneg': 'x', # XXX overflow 'int_sub': '-', 'int_sub_ovf': '-', # XXX overflow 'int_floordiv': '/', Modified: pypy/dist/pypy/translator/js/test/test_bltn.py ============================================================================== --- pypy/dist/pypy/translator/js/test/test_bltn.py (original) +++ pypy/dist/pypy/translator/js/test/test_bltn.py Fri Oct 19 16:00:53 2007 @@ -57,7 +57,7 @@ SomeNodeInstance.some_callback = callback fn = compile_function(callback_stuff, []) - assert check_source_contains(fn, "\.some_callback = callback") + assert not check_source_contains(fn, "\.some_callback = callback") def test_get_elements(): from pypy.translator.js.modules import dom From fijal at codespeak.net Fri Oct 19 16:02:31 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Oct 2007 16:02:31 +0200 (CEST) Subject: [pypy-svn] r47587 - in pypy/dist/pypy/translator/js: examples/bnb/data test Message-ID: <20071019140231.3A8488133@code0.codespeak.net> Author: fijal Date: Fri Oct 19 16:02:30 2007 New Revision: 47587 Modified: pypy/dist/pypy/translator/js/examples/bnb/data/bnb.html pypy/dist/pypy/translator/js/test/test_bltn.py Log: This wasn't expected to go with this checkin Modified: pypy/dist/pypy/translator/js/examples/bnb/data/bnb.html ============================================================================== --- pypy/dist/pypy/translator/js/examples/bnb/data/bnb.html (original) +++ pypy/dist/pypy/translator/js/examples/bnb/data/bnb.html Fri Oct 19 16:02:30 2007 @@ -4,7 +4,7 @@ Bub'n'bros - + Modified: pypy/dist/pypy/translator/js/test/test_bltn.py ============================================================================== --- pypy/dist/pypy/translator/js/test/test_bltn.py (original) +++ pypy/dist/pypy/translator/js/test/test_bltn.py Fri Oct 19 16:02:30 2007 @@ -57,7 +57,7 @@ SomeNodeInstance.some_callback = callback fn = compile_function(callback_stuff, []) - assert not check_source_contains(fn, "\.some_callback = callback") + assert check_source_contains(fn, "\.some_callback = callback") def test_get_elements(): from pypy.translator.js.modules import dom From fijal at codespeak.net Fri Oct 19 16:05:22 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Oct 2007 16:05:22 +0200 (CEST) Subject: [pypy-svn] r47588 - pypy/dist/pypy/translator/js Message-ID: <20071019140522.7DAFC812E@code0.codespeak.net> Author: fijal Date: Fri Oct 19 16:05:22 2007 New Revision: 47588 Modified: pypy/dist/pypy/translator/js/opcodes.py Log: Proper operations Modified: pypy/dist/pypy/translator/js/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/js/opcodes.py (original) +++ pypy/dist/pypy/translator/js/opcodes.py Fri Oct 19 16:05:22 2007 @@ -33,9 +33,10 @@ } opcodes = {'int_mul': '*', + 'int_mul_ovf' : '*', # XXX overflow 'int_add': '+', 'int_add_ovf': '+', # XXX overflow - 'int_add_ovf_nonneg': 'x', # XXX overflow + 'int_add_nonneg_ovf': '+', # XXX overflow 'int_sub': '-', 'int_sub_ovf': '-', # XXX overflow 'int_floordiv': '/', From arigo at codespeak.net Fri Oct 19 16:41:28 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 16:41:28 +0200 (CEST) Subject: [pypy-svn] r47589 - pypy/dist/pypy/doc/config Message-ID: <20071019144128.8A9AC8125@code0.codespeak.net> Author: arigo Date: Fri Oct 19 16:41:27 2007 New Revision: 47589 Removed: pypy/dist/pypy/doc/config/translation.frameworkgc.txt Modified: pypy/dist/pypy/doc/config/translation.gc.txt Log: Fix config documentation. Modified: pypy/dist/pypy/doc/config/translation.gc.txt ============================================================================== --- pypy/dist/pypy/doc/config/translation.gc.txt (original) +++ pypy/dist/pypy/doc/config/translation.gc.txt Fri Oct 19 16:41:27 2007 @@ -3,10 +3,11 @@ - "ref": reference counting. Takes very long to translate and the result is slow. - - "framework": one of our custom garbage collectors. Takes moderately long - and is the fastest option without external dependencies. See - :config:`translation.frameworkgc`. + - "marksweep": naive mark & sweep. - - "boehm": use the Boehm conservative GC + - "semispace": a copying semi-space GC. + - "generation": a generational GC using the semi-space GC for the + older generation. + - "boehm": use the Boehm conservative GC. From antocuni at codespeak.net Fri Oct 19 16:42:15 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 19 Oct 2007 16:42:15 +0200 (CEST) Subject: [pypy-svn] r47590 - pypy/extradoc/talk/dls2007 Message-ID: <20071019144215.777BE812E@code0.codespeak.net> Author: antocuni Date: Fri Oct 19 16:42:14 2007 New Revision: 47590 Added: pypy/extradoc/talk/dls2007/rpython-talk.pdf (contents, props changed) pypy/extradoc/talk/dls2007/rpython-talk.pdf.info Log: add the generated pdf and the info file for keyjnote Added: pypy/extradoc/talk/dls2007/rpython-talk.pdf ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/dls2007/rpython-talk.pdf.info ============================================================================== --- (empty file) +++ pypy/extradoc/talk/dls2007/rpython-talk.pdf.info Fri Oct 19 16:42:14 2007 @@ -0,0 +1,11 @@ +AvailableTransitions=[Crossfade] +TransitionDuration = 100 +EstimatedDuration = 25*60 +MinutesOnly = True + +PageProps = { + 1: { + 'reset': FirstTimeOnly, + 'progress': False, + }, +} From fijal at codespeak.net Fri Oct 19 16:54:10 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Oct 2007 16:54:10 +0200 (CEST) Subject: [pypy-svn] r47593 - pypy/dist/pypy/module/_ffi Message-ID: <20071019145410.713D080AF@code0.codespeak.net> Author: fijal Date: Fri Oct 19 16:54:10 2007 New Revision: 47593 Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py Log: Add an additional cast. This keeps us on the safe-side of lltype.typeOf without passing type by hand Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Fri Oct 19 16:54:10 2007 @@ -172,10 +172,12 @@ def unwrap_value(space, push_func, add_arg, argdesc, tp, w_arg, to_free): w = space.wrap - # XXX how to handle LONGLONG here? - # they're probably long, so we'll not get them through int_w if tp == "d" or tp == "f": - push_func(add_arg, argdesc, space.float_w(w_arg)) + if tp == "d": + push_func(add_arg, argdesc, space.float_w(w_arg)) + else: + push_func(add_arg, argdesc, rffi.cast(rffi.FLOAT, + space.float_w(w_arg))) elif tp == "s": ll_str = rffi.str2charp(space.str_w(w_arg)) if to_free is not None: @@ -204,7 +206,7 @@ if len(s) != 1: raise OperationError(space.w_ValueError, w( "Expected string of length one as character")) - val = ord(s[0]) + val = s[0] push_func(add_arg, argdesc, val) else: for c, checker in unroll_size_checkers: @@ -221,7 +223,8 @@ checker(val) except FfiValueError, e: raise OperationError(space.w_ValueError, w(e.msg)) - push_func(add_arg, argdesc, val) + TP = LL_TYPEMAP[c] + push_func(add_arg, argdesc, rffi.cast(TP, val)) unwrap_value._annspecialcase_ = 'specialize:arg(1)' From fijal at codespeak.net Fri Oct 19 16:57:16 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Oct 2007 16:57:16 +0200 (CEST) Subject: [pypy-svn] r47594 - pypy/dist/pypy Message-ID: <20071019145716.A407880C2@code0.codespeak.net> Author: fijal Date: Fri Oct 19 16:57:16 2007 New Revision: 47594 Modified: pypy/dist/pypy/conftest.py Log: Be on a safe side regarding dist and trunk versions of py.test Modified: pypy/dist/pypy/conftest.py ============================================================================== --- pypy/dist/pypy/conftest.py (original) +++ pypy/dist/pypy/conftest.py Fri Oct 19 16:57:16 2007 @@ -282,6 +282,9 @@ _pygame_imported = False class IntTestFunction(PyPyTestFunction): + def _haskeyword(self, keyword): + return keyword == 'interplevel' or \ + super(IntTestFunction, self)._haskeyword(keyword) def _keywords(self): return super(IntTestFunction, self)._keywords + ['interplevel'] @@ -312,7 +315,11 @@ assert option.view, ("should not invoke Pygame " "if conftest.option.view is False") -class AppTestFunction(PyPyTestFunction): +class AppTestFunction(PyPyTestFunction): + def _haskeyword(self, keyword): + return keyword == 'applevel' or \ + super(AppTestFunction, self)._haskeyword(keyword) + def _keywords(self): return ['applevel'] + super(AppTestFunction, self)._keywords() @@ -361,6 +368,10 @@ class IntClassCollector(PyPyClassCollector): Function = IntTestFunction + def _haskeyword(self, keyword): + return keyword == 'interplevel' or \ + super(IntClassCollector, self)._haskeyword(keyword) + def _keywords(self): return super(IntClassCollector, self)._keywords() + ['interplevel'] @@ -380,6 +391,10 @@ class AppClassCollector(PyPyClassCollector): Instance = AppClassInstance + def _haskeyword(self, keyword): + return keyword == 'applevel' or \ + super(AppClassCollector, self)._haskeyword(keyword) + def _keywords(self): return super(AppClassCollector, self)._keywords() + ['applevel'] From arigo at codespeak.net Fri Oct 19 17:25:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 17:25:09 +0200 (CEST) Subject: [pypy-svn] r47595 - pypy/dist/pypy/rpython Message-ID: <20071019152509.B8F668107@code0.codespeak.net> Author: arigo Date: Fri Oct 19 17:25:08 2007 New Revision: 47595 Modified: pypy/dist/pypy/rpython/llinterp.py Log: Replace the fragile active_frame logic with an explicit frame_stack and an explicit traceback_frames. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Fri Oct 19 17:25:08 2007 @@ -49,7 +49,7 @@ # 'heap' is module or object that provides malloc, etc for lltype ops self.heap = llheap self.exc_data_ptr = exc_data_ptr - self.active_frame = None + self.frame_stack = [] self.tracer = None self.malloc_check = malloc_check self.frame_class = LLFrame @@ -60,9 +60,12 @@ def eval_graph(self, graph, args=(), recursive=False): llframe = self.frame_class(graph, args, self) if self.tracer and not recursive: + global tracer1 + tracer1 = self.tracer self.tracer.start() retval = None - old_active_frame = self.active_frame + self.traceback_frames = [] + old_frame_stack = self.frame_stack[:] try: try: retval = llframe.eval() @@ -83,7 +86,7 @@ self.tracer.dump(line + '\n') raise finally: - self.active_frame = old_active_frame + assert old_frame_stack == self.frame_stack if self.tracer: if retval is not None: self.tracer.dump(' ---> %r\n' % (retval,)) @@ -92,12 +95,9 @@ return retval def print_traceback(self): - frame = self.active_frame - frames = [] - while frame is not None: - frames.append(frame) - frame = frame.f_back + frames = self.traceback_frames frames.reverse() + self.traceback_frames = [] lines = [] for frame in frames: logline = frame.graph.name @@ -128,12 +128,10 @@ def find_roots(self): """Return a list of the addresses of the roots.""" #log.findroots("starting") - frame = self.active_frame roots = [] - while frame is not None: + for frame in self.frame_stack: #log.findroots("graph", frame.graph.name) frame.find_roots(roots) - frame = frame.f_back return roots def find_exception(self, exc): @@ -141,16 +139,12 @@ klass, inst = exc.args[0], exc.args[1] exdata = self.typer.getexceptiondata() frame = self.frame_class(None, [], self) - old_active_frame = self.active_frame - try: - for cls in enumerate_exceptions_top_down(): - evalue = frame.op_direct_call(exdata.fn_pyexcclass2exc, - lltype.pyobjectptr(cls)) - etype = frame.op_direct_call(exdata.fn_type_of_exc_inst, evalue) - if etype == klass: - return cls - finally: - self.active_frame = old_active_frame + for cls in enumerate_exceptions_top_down(): + evalue = frame.op_direct_call(exdata.fn_pyexcclass2exc, + lltype.pyobjectptr(cls)) + etype = frame.op_direct_call(exdata.fn_type_of_exc_inst, evalue) + if etype == klass: + return cls raise ValueError, "couldn't match exception" def get_transformed_exc_data(self, graph): @@ -181,14 +175,13 @@ class LLFrame(object): - def __init__(self, graph, args, llinterpreter, f_back=None): + def __init__(self, graph, args, llinterpreter): assert not graph or isinstance(graph, FunctionGraph) self.graph = graph self.args = args self.llinterpreter = llinterpreter self.heap = llinterpreter.heap self.bindings = {} - self.f_back = f_back self.curr_block = None self.curr_operation_index = 0 self.alloca_objects = [] @@ -257,24 +250,29 @@ # evaling functions def eval(self): - self.llinterpreter.active_frame = self graph = self.graph tracer = self.llinterpreter.tracer if tracer: tracer.enter(graph) + self.llinterpreter.frame_stack.append(self) try: - nextblock = graph.startblock - args = self.args - while 1: - self.clear() - self.fillvars(nextblock, args) - nextblock, args = self.eval_block(nextblock) - if nextblock is None: - self.llinterpreter.active_frame = self.f_back - for obj in self.alloca_objects: - obj._obj._free() - return args + try: + nextblock = graph.startblock + args = self.args + while 1: + self.clear() + self.fillvars(nextblock, args) + nextblock, args = self.eval_block(nextblock) + if nextblock is None: + for obj in self.alloca_objects: + obj._obj._free() + return args + except Exception: + self.llinterpreter.traceback_frames.append(self) + raise finally: + leavingframe = self.llinterpreter.frame_stack.pop() + assert leavingframe is self if tracer: tracer.leave() @@ -629,7 +627,7 @@ if not lltype.isCompatibleType(T, v.concretetype): raise TypeError("graph with %r args called with wrong func ptr type: %r" % (tuple([v.concretetype for v in args_v]), ARGS)) - frame = self.__class__(graph, args, self.llinterpreter, self) + frame = self.__class__(graph, args, self.llinterpreter) return frame.eval() def op_direct_call(self, f, *args): @@ -656,7 +654,7 @@ args = [] for inarg, arg in zip(inargs, obj.graph.startblock.inputargs): args.append(lltype._cast_whatever(arg.concretetype, inarg)) - frame = self.__class__(graph, args, self.llinterpreter, self) + frame = self.__class__(graph, args, self.llinterpreter) result = frame.eval() from pypy.translator.stackless.frame import storage_type assert storage_type(lltype.typeOf(result)) == TGT From arigo at codespeak.net Fri Oct 19 17:27:10 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 17:27:10 +0200 (CEST) Subject: [pypy-svn] r47596 - pypy/dist/pypy/rpython Message-ID: <20071019152710.A07F080B8@code0.codespeak.net> Author: arigo Date: Fri Oct 19 17:27:10 2007 New Revision: 47596 Modified: pypy/dist/pypy/rpython/llinterp.py Log: Uh? This explodes when seeing a null pointer. Strange that it didn't explode earlier. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Fri Oct 19 17:27:10 2007 @@ -1266,7 +1266,7 @@ self.frame.setvar(self.v, p) def unwrap_possible_weakref(self, addr): # fish fish fish - if isinstance(addr.ptr._obj, llmemory._gctransformed_wref): + if addr and isinstance(addr.ptr._obj, llmemory._gctransformed_wref): return llmemory.fakeaddress(addr.ptr._obj._ptr) return addr From arigo at codespeak.net Fri Oct 19 17:30:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 17:30:35 +0200 (CEST) Subject: [pypy-svn] r47597 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20071019153035.98595812B@code0.codespeak.net> Author: arigo Date: Fri Oct 19 17:30:35 2007 New Revision: 47597 Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py Log: We need support for 'NULL-NULL'. Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Fri Oct 19 17:30:35 2007 @@ -329,6 +329,11 @@ def __sub__(self, other): if isinstance(other, AddressOffset): return self + (-other) + if isinstance(other, fakeaddress): + if self == other: + return 0 + else: + raise TypeError("cannot subtract fakeaddresses in general") if other == 0: return self return NotImplemented Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py Fri Oct 19 17:30:35 2007 @@ -191,6 +191,13 @@ adr2 += ItemOffset(lltype.Char, -1) assert adr1 == adr2 +def test_adr_sub(): + # only (a - a) is supported + assert NULL - NULL == 0 + A = lltype.GcArray(lltype.Char) + a = raw_malloc(sizeof(A, 5)) + assert a - a == 0 + def test_cast_subarray_pointer(): for a in [lltype.malloc(lltype.GcArray(lltype.Signed), 5), lltype.malloc(lltype.FixedSizeArray(lltype.Signed, 5), From arigo at codespeak.net Fri Oct 19 17:33:41 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 17:33:41 +0200 (CEST) Subject: [pypy-svn] r47598 - in pypy/dist/pypy/rpython/memory: gc test Message-ID: <20071019153341.DA05B812B@code0.codespeak.net> Author: arigo Date: Fri Oct 19 17:33:41 2007 New Revision: 47598 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/test/test_gc.py Log: Fix the GenerationGC: the nursery is now allocated out of the current semispace, a bit like another object. This approach is quite simple and fixes various delicate ordering issues. Now test_gc passes. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Fri Oct 19 17:33:41 2007 @@ -27,16 +27,19 @@ max_space_size = max_space_size, get_roots = get_roots) self.nursery_size = nursery_size + assert nursery_size <= space_size // 2 def setup(self): SemiSpaceGC.setup(self) - self.nursery = llarena.arena_malloc(self.nursery_size, True) - debug_assert(bool(self.nursery), "couldn't allocate nursery") - self.nursery_free = self.nursery - self.nursery_top = self.nursery + self.nursery_size + self.reset_nursery() self.old_objects_pointing_to_young = nonnull_endmarker # ^^^ the head of a linked list inside the old objects space + def reset_nursery(self): + self.nursery = llmemory.NULL + self.nursery_top = llmemory.NULL + self.nursery_free = llmemory.NULL + def is_in_nursery(self, addr): return self.nursery <= addr < self.nursery_top @@ -60,6 +63,7 @@ def semispace_collect(self, size_changing=False): self.reset_forwarding() # we are doing a full collection anyway + self.reset_nursery() SemiSpaceGC.semispace_collect(self, size_changing) def reset_forwarding(self): @@ -74,23 +78,26 @@ if self.nursery_size > self.top_of_space - self.free: # the semispace is running out, do a full collect self.obtain_free_space(self.nursery_size) - debug_assert(self.nursery_free == self.nursery, - "nursery not collected") - else: + debug_assert(self.nursery_size <= self.top_of_space - self.free, + "obtain_free_space failed to do its job") + if self.nursery: + # a nursery-only collection scan = self.free self.collect_oldrefs_to_nursery() self.collect_roots_in_nursery() self.scan_objects_just_copied_out_of_nursery(scan) self.notify_objects_just_moved() - return self.nursery_free - - def reset_nursery(self): - llarena.arena_reset(self.nursery, self.nursery_size, True) + # mark the nursery as free and fill it with zeroes again + llarena.arena_reset(self.nursery, self.nursery_size, True) + else: + # no nursery - this occurs after a full collect, triggered either + # just above or by some previous non-nursery-based allocation. + # Grab a piece of the current space for the nursery. + self.nursery = self.free + self.nursery_top = self.nursery + self.nursery_size + self.free = self.nursery_top self.nursery_free = self.nursery - - def notify_objects_just_moved(self): - self.reset_nursery() - SemiSpaceGC.notify_objects_just_moved(self) + return self.nursery_free # NB. we can use self.copy() to move objects out of the nursery, # but only if the object was really in the nursery. Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Fri Oct 19 17:33:41 2007 @@ -276,7 +276,6 @@ def test_finalizer_calls_malloc_during_minor_collect(self): # originally a GenerationGC test, this has also found bugs in other GCs - py.test.skip("in-progress") class B(object): pass b = B() From arigo at codespeak.net Fri Oct 19 17:48:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 17:48:57 +0200 (CEST) Subject: [pypy-svn] r47599 - pypy/dist/pypy/module/_stackless/test Message-ID: <20071019154857.E3A228132@code0.codespeak.net> Author: arigo Date: Fri Oct 19 17:48:57 2007 New Revision: 47599 Modified: pypy/dist/pypy/module/_stackless/test/test_interp_clonable.py Log: Forgotten by the translation.gc option changes. Modified: pypy/dist/pypy/module/_stackless/test/test_interp_clonable.py ============================================================================== --- pypy/dist/pypy/module/_stackless/test/test_interp_clonable.py (original) +++ pypy/dist/pypy/module/_stackless/test/test_interp_clonable.py Fri Oct 19 17:48:57 2007 @@ -11,7 +11,7 @@ class TestClonableCoroutine(test_transformed_gc.GCTest): - gcname = "framework" + gcname = "marksweep" stacklessgc = True class gcpolicy(gc.StacklessFrameworkGcPolicy): class transformerclass(stacklessframework.StacklessFrameworkGCTransformer): From arigo at codespeak.net Fri Oct 19 17:51:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 17:51:53 +0200 (CEST) Subject: [pypy-svn] r47600 - pypy/dist/pypy/config Message-ID: <20071019155153.1E1AA8132@code0.codespeak.net> Author: arigo Date: Fri Oct 19 17:51:52 2007 New Revision: 47600 Modified: pypy/dist/pypy/config/translationoption.py Log: We need default="ref" here, otherwise the option value is None as long as no gc was set, which crashes some tests. Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Fri Oct 19 17:51:52 2007 @@ -57,7 +57,7 @@ cmdline="--gc"), ChoiceOption("gctransformer", "GC transformer that is used - internal", ["boehm", "ref", "framework", "none"], - cmdline=None), + default="ref", cmdline=None), BoolOption("stacklessgc", "Use stackless to find roots in a framework GC", default=False, cmdline="--stacklessgc", From fijal at codespeak.net Fri Oct 19 18:05:10 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Oct 2007 18:05:10 +0200 (CEST) Subject: [pypy-svn] r47601 - in pypy/dist/pypy/module/_ffi: . test Message-ID: <20071019160510.C3D528142@code0.codespeak.net> Author: fijal Date: Fri Oct 19 18:05:10 2007 New Revision: 47601 Added: pypy/dist/pypy/module/_ffi/callback.py (contents, props changed) Modified: pypy/dist/pypy/module/_ffi/__init__.py pypy/dist/pypy/module/_ffi/interp_ffi.py pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: * Begginings of callback - a test and some commented out code * Really call a function even if it's returning Void, C is not functional enough Modified: pypy/dist/pypy/module/_ffi/__init__.py ============================================================================== --- pypy/dist/pypy/module/_ffi/__init__.py (original) +++ pypy/dist/pypy/module/_ffi/__init__.py Fri Oct 19 18:05:10 2007 @@ -15,6 +15,7 @@ 'StructureInstance' : 'structure.W_StructureInstance', 'ArrayInstance' : 'array.W_ArrayInstance', '_get_type' : 'interp_ffi._w_get_type', + #'CallbackPtr' : 'callback.W_CallbackPtr', } appleveldefs = { Added: pypy/dist/pypy/module/_ffi/callback.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/_ffi/callback.py Fri Oct 19 18:05:10 2007 @@ -0,0 +1,31 @@ + +from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable,\ + Arguments +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.module._ffi.structure import unpack_fields + +def stuff(a, b): + print "comparing" + return int(a > b) + +class W_CallbackPtr(Wrappable): + def __init__(self, space, w_callable, w_args, w_result): + self.w_callable = w_callable + self.args = [space.str_w(w_arg) for w_arg in space.unpackiterable( + w_args)] + + def getllfuncptr(space, self): + TP = lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) + ptr = lltype.functionptr(TP, stuff) + return space.wrap(rffi.cast(rffi.INT, ptr)) + +def descr_new_callbackptr(space, w_type, w_callable, w_args, w_result): + return W_CallbackPtr(space, w_callable, w_args, w_result) + +W_CallbackPtr.typedef = TypeDef( + 'CallbackPtr', + buffer = GetSetProperty(W_CallbackPtr.getllfuncptr), + __new__ = interp2app(descr_new_callbackptr), +) Modified: pypy/dist/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/dist/pypy/module/_ffi/interp_ffi.py Fri Oct 19 18:05:10 2007 @@ -51,7 +51,8 @@ 'f' : rffi.FLOAT, 'd' : rffi.DOUBLE, 's' : rffi.CCHARP, - 'P' : rffi.VOIDP, + 'P' : rffi.VOIDP, + 'v' : lltype.Void, } def _get_type(space, key): @@ -194,10 +195,16 @@ mod = space.getbuiltinmodule('_ffi') w_StructureInstance = space.getattr(mod, w('StructureInstance')) w_ArrayInstance = space.getattr(mod, w('ArrayInstance')) + #w_CallbackPtr = space.getattr(mod, w('CallbackPtr')) if space.is_true(space.isinstance(w_arg, w_StructureInstance)) or\ space.is_true(space.isinstance(w_arg, w_ArrayInstance)): ptr = rffi.cast(rffi.VOIDP, space.int_w(space.getattr(w_arg, w('buffer')))) push_func(add_arg, argdesc, ptr) + #elif space.is_true(space.isinstance(w_arg, w_CallbackPtr)): + # TP = lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) + # ptr = rffi.cast(TP, space.int_w(space.getattr(w_arg, w('buffer')))) + # push_func(add_arg, argdesc, ptr) + else: raise OperationError(space.w_TypeError, w( "Expected structure, array or simple type")) @@ -243,6 +250,9 @@ if not res: return space.w_None return space.wrap(rffi.cast(rffi.INT, res)) + elif c == 'v': + func(add_arg, argdesc, ll_type) + return space.w_None elif c == 'q' or c == 'Q' or c == 'L': return space.wrap(func(add_arg, argdesc, ll_type)) elif c == 'f' or c == 'd': @@ -286,9 +296,7 @@ unwrap_value(space, push, self.ptr, i, argtype, w_arg, to_free) i += 1 try: - if self.restype != 'v': - return wrap_value(space, ptr_call, self.ptr, None, self.restype) - return space.w_None + return wrap_value(space, ptr_call, self.ptr, None, self.restype) finally: for elem in to_free: lltype.free(elem, flavor='raw') Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Fri Oct 19 18:05:10 2007 @@ -88,6 +88,7 @@ { return x; } + ''')) compile_c_module([c_file], 'x') return str(udir.join('x.so')) @@ -293,7 +294,7 @@ assert pass_ll(1<<42) == 1<<42 def test_callback(self): - skip("Segfaults") + skip("Not working") import _ffi libc = _ffi.CDLL('libc.so.6') to_sort = "kljhgfa" @@ -302,5 +303,6 @@ def compare(a, b): return a < b qsort(ll_to_sort, len(to_sort), 1, - CallbackPtr(compare, ['i', 'i'], 'i')) - + _ffi.CallbackPtr(compare, ['i', 'i'], 'i')) + res = [ll_to_sort[i] for i in range(len(to_sort))] + assert res == sorted(to_sort) From fijal at codespeak.net Fri Oct 19 18:38:42 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Oct 2007 18:38:42 +0200 (CEST) Subject: [pypy-svn] r47602 - in pypy/dist/pypy/rpython/tool: . test Message-ID: <20071019163842.552938125@code0.codespeak.net> Author: fijal Date: Fri Oct 19 18:38:40 2007 New Revision: 47602 Modified: pypy/dist/pypy/rpython/tool/rfficache.py pypy/dist/pypy/rpython/tool/test/test_rfficache.py Log: Simplify rfficache, to use it only for bootstrapping Modified: pypy/dist/pypy/rpython/tool/rfficache.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rfficache.py (original) +++ pypy/dist/pypy/rpython/tool/rfficache.py Fri Oct 19 18:38:40 2007 @@ -11,12 +11,10 @@ from pypy.tool.autopath import pypydir from pypy.rlib import rarithmetic from pypy.rpython.lltypesystem import lltype +from pypy.tool.gcc_cache import build_executable_cache -def ask_gcc(question, includes=[], add_source="", include_dirs=[], - compiler_exe=None): - from py.compat.subprocess import PIPE, Popen - includes.append('stdio.h') - includes.append('sys' + os.path.sep + 'types.h') +def ask_gcc(question, add_source="", compiler_exe=None): + includes = ['stdlib.h', 'sys/types.h'] include_string = "\n".join(["#include <%s>" % i for i in includes]) c_source = py.code.Source(''' // includes @@ -33,114 +31,26 @@ ''' % (include_string, add_source, str(question))) c_file = udir.join("gcctest.c") c_file.write(c_source) - - # always include pypy include dir - pypypath = py.path.local(pypydir) - include_dirs = include_dirs[:] - include_dirs.append(str(pypypath.join('translator', 'c'))) - - c_exec = build_executable([str(c_file)], include_dirs=include_dirs, - compiler_exe=compiler_exe) - pipe = Popen(c_exec, stdout=PIPE) - pipe.wait() - return pipe.stdout.read() + return build_executable_cache([c_file], compiler_exe=compiler_exe) def sizeof_c_type(c_typename, **kwds): question = 'printf("%%d", sizeof(%s));' % (c_typename,); return int(ask_gcc(question, **kwds)) -def c_ifdefined(c_def, **kwds): - question = py.code.Source(""" - #ifdef %s - printf("0"); - #endif - """ % (c_def,)) - return ask_gcc(question, **kwds) == '0' - -def c_defined_int(c_def, **kwds): - question = 'printf("%%d", %s);' % (c_def,) - return int(ask_gcc(question, **kwds)) - -def have_c_obj(c_obj, **kwds): - question = c_obj + ';' - try: - ask_gcc(question, **kwds) - return True - except distutils.errors.CompileError: - # parsing errors here and trying to deduce whether - # it's this or not sounds like an overkill - return False - -def create_cache_access_method(acc_func, meth_name): - def method(self, name, **kwds): - try: - return self.cache[meth_name, name] - except KeyError: - res = acc_func(name, **kwds) - self.cache[meth_name, name] = res - self._store_cache() - return res - method.func_name = meth_name - return method - -class RffiCache(object): - """ Class holding all of the c-level caches, eventually loaded from - the file, like #ifdefs, typesizes, int-level #defines - """ - def __init__(self, filename): - self.filename = filename - self.numbertype_to_rclass = {} +class Platform: + def __init__(self): self.types = {} - try: - mod = {} - exec py.path.local(filename).read() in mod - self.cache = mod['cache'] - self.type_names = mod['type_names'] - self._build_types() - except (py.error.ENOENT, KeyError): - self.cache = {} - self.type_names = {} - - def _build_types(self): - for name, (c_name, signed) in self.type_names.items(): - bits = self.cache['bits', c_name] - inttype = rarithmetic.build_int('r_' + name, signed, bits) - tp = lltype.build_number(name, inttype) - self.numbertype_to_rclass[tp] = inttype - self.types[name] = tp - + self.numbertype_to_rclass = {} + def inttype(self, name, c_name, signed, **kwds): - # XXX sign should be inferred somehow automatically try: return self.types[name] except KeyError: bits = sizeof_c_type(c_name, **kwds) * 8 inttype = rarithmetic.build_int('r_' + name, signed, bits) - self.cache['bits', c_name] = bits - self.type_names[name] = (c_name, signed) tp = lltype.build_number(name, inttype) self.numbertype_to_rclass[tp] = inttype self.types[name] = tp - self._store_cache() return tp - defined = create_cache_access_method(c_ifdefined, 'defined') - intdefined = create_cache_access_method(c_defined_int, 'intdefined') - sizeof = create_cache_access_method(sizeof_c_type, 'sizeof') - has = create_cache_access_method(have_c_obj, 'has') - - # optimal way of caching it, would be to store file on __del__, - # but since we cannot rely on __del__ having all modules, let's - # do it after each change :-( - def _store_cache(self): - types = 'type_names = ' + repr(self.type_names) + '\n' - py.path.local(self.filename).write('cache = ' + repr(self.cache) - + '\n' + types) - -import pypy -import py -py.path.local(pypy.__file__).new(basename='_cache').ensure(dir=1) -from pypy.tool import autopath -CACHE = py.magic.autopath()/'..'/'..'/'..'/'_cache'/'stdtypes.py' -platform = RffiCache(CACHE) - +platform = Platform() Modified: pypy/dist/pypy/rpython/tool/test/test_rfficache.py ============================================================================== --- pypy/dist/pypy/rpython/tool/test/test_rfficache.py (original) +++ pypy/dist/pypy/rpython/tool/test/test_rfficache.py Fri Oct 19 18:38:40 2007 @@ -7,34 +7,6 @@ sizeofchar = sizeof_c_type('char') assert sizeofchar == 1 -def test_c_ifdefined(): - assert c_ifdefined('X', add_source='#define X') - assert not c_ifdefined('X') - -def test_c_defined_int(): - assert c_defined_int('X', add_source='#define X 3') == 3 - -def test_c_existing_symbol(): - assert have_c_obj('X', add_source='int X(int, int, int);') - assert not have_c_obj('xxx') - -def test_rfficache(): - cache = RffiCache(udir.join('cache.py')) - assert cache.inttype('uchar', 'unsigned char', False)._type.BITS == 8 - assert cache.inttype('uchar', 'unsigned char', False, compiler_exe='xxx')._type.BITS == 8 - assert cache.defined('STUFF', add_source='#define STUFF') - assert cache.defined('STUFF') - assert cache.intdefined('STUFFI', add_source='#define STUFFI 3') == 3 - assert cache.intdefined('STUFFI') == 3 - assert cache.sizeof('short') == 2 - cache = RffiCache(udir.join('cache.py')) - assert cache.intdefined('STUFFI') == 3 - assert cache.defined('STUFF') - assert cache.inttype('uchar', 'unsigned char', False, compiler_exe='xxx')._type.BITS == 8 - assert cache.sizeof('short', compiler_exe='xxx') == 2 - assert cache.intdefined('STUFFZ', add_source='#define STUFFZ 0') == 0 - assert cache.defined('STUFFZ', add_source='#define STUFFZ 0') - def test_types_present(): for name in rffi.TYPES: if name.startswith('unsigned'): From fijal at codespeak.net Fri Oct 19 18:46:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Oct 2007 18:46:54 +0200 (CEST) Subject: [pypy-svn] r47603 - pypy/dist/pypy/rpython/module Message-ID: <20071019164654.7D0D38132@code0.codespeak.net> Author: fijal Date: Fri Oct 19 18:46:54 2007 New Revision: 47603 Modified: pypy/dist/pypy/rpython/module/ll_termios.py Log: Kill (hopefully last) occurence of old rffi.platform Modified: pypy/dist/pypy/rpython/module/ll_termios.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_termios.py (original) +++ pypy/dist/pypy/rpython/module/ll_termios.py Fri Oct 19 18:46:54 2007 @@ -14,13 +14,18 @@ from pypy.annotation import model as annmodel from pypy.rpython import rclass from pypy.rlib import rtermios -from pypy.rpython.lltypesystem.rffi import platform +from pypy.rpython.tool import rffi_platform includes = ['termios.h', 'unistd.h'] +class CConfig: + _includes_ = includes + NCCS = rffi_platform.DefinedConstantInteger('NCCS') + +NCCS = rffi_platform.configure(CConfig)['NCCS'] + TCFLAG_T = rffi.UINT CC_T = rffi.UCHAR -NCCS = platform.intdefined('NCCS', includes=includes) SPEED_T = rffi.UINT INT = rffi.INT From fijal at codespeak.net Fri Oct 19 18:53:18 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Oct 2007 18:53:18 +0200 (CEST) Subject: [pypy-svn] r47604 - in pypy/dist/pypy/rpython/tool: . test Message-ID: <20071019165318.BE3788136@code0.codespeak.net> Author: fijal Date: Fri Oct 19 18:53:18 2007 New Revision: 47604 Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py Log: Implement rffi_platform.SizeOf Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Fri Oct 19 18:53:18 2007 @@ -42,6 +42,14 @@ HAS = Has(name) return configure(CConfig)['HAS'] +def sizeof(name, c_header_source, **kwds): + class CConfig: + _header_ = c_header_source + SIZE = SizeOf(name) + for k, v in kwds.items(): + setattr(CConfig, k, v) + return configure(CConfig)['SIZE'] + # ____________________________________________________________ # # General interface @@ -270,6 +278,19 @@ kwds = {'hints': hints} return rffi.CStruct(name, *fields, **kwds) +class SizeOf(CConfigEntry): + """An entry in a CConfig class that stands for sizeof + of some external opaque type + """ + def __init__(self, name): + self.name = name + + def prepare_code(self): + yield 'dump("size", sizeof(%s));' % self.name + + def build_result(self, info, config_result): + return info['size'] + class SimpleType(CConfigEntry): """An entry in a CConfig class that stands for an externally defined simple numeric type. Modified: pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py Fri Oct 19 18:53:18 2007 @@ -191,3 +191,6 @@ assert not rffi_platform.has("x", "") # has() should also not crash if it is given an invalid #include assert not rffi_platform.has("x", "#include ") + +def test_sizeof(): + assert rffi_platform.sizeof("char", "") == 1 From fijal at codespeak.net Fri Oct 19 18:54:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Oct 2007 18:54:46 +0200 (CEST) Subject: [pypy-svn] r47605 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071019165446.881338136@code0.codespeak.net> Author: fijal Date: Fri Oct 19 18:54:46 2007 New Revision: 47605 Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: Use new rffi_platform.sizeof Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Fri Oct 19 18:54:46 2007 @@ -11,6 +11,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rlib.unroll import unrolling_iterable from pypy.tool.sourcetools import func_with_new_name +from pypy.rpython.tool.rfficache import platform import os class CConstant(Symbolic): @@ -136,8 +137,6 @@ # ____________________________________________________________ -from pypy.rpython.tool.rfficache import platform - TYPES = [] for _name in 'short int long'.split(): for name in (_name, 'unsigned ' + _name): @@ -232,7 +231,8 @@ hints['c_name'] = name def lazy_getsize(result=[]): if not result: - size = platform.sizeof(name, **kwds) + from pypy.rpython.tool import rffi_platform + size = rffi_platform.sizeof(name, "", **kwds) result.append(size) return result[0] hints['getsize'] = lazy_getsize From fijal at codespeak.net Fri Oct 19 18:58:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Oct 2007 18:58:04 +0200 (CEST) Subject: [pypy-svn] r47606 - pypy/dist/pypy/rlib Message-ID: <20071019165804.4B65D8136@code0.codespeak.net> Author: fijal Date: Fri Oct 19 18:58:03 2007 New Revision: 47606 Modified: pypy/dist/pypy/rlib/libffi.py Log: Parameters for COpaquePtr. Not sure that this is the right way, probably we should rather have OpaquePtr directly as an attribute of CConfig. but at least test passes now on wyvern Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Fri Oct 19 18:58:03 2007 @@ -111,7 +111,9 @@ FFI_OK = cConfig.FFI_OK FFI_BAD_TYPEDEF = cConfig.FFI_BAD_TYPEDEF FFI_DEFAULT_ABI = rffi.cast(rffi.USHORT, cConfig.FFI_DEFAULT_ABI) -FFI_CIFP = rffi.COpaquePtr('ffi_cif', includes=includes) +FFI_CIFP = rffi.COpaquePtr('ffi_cif', _includes_=includes, + _include_dirs_=include_dirs, + _library_dirs_=lib_dirs) VOIDPP = rffi.CArrayPtr(rffi.VOIDP) From arigo at codespeak.net Fri Oct 19 19:04:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 19:04:32 +0200 (CEST) Subject: [pypy-svn] r47607 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071019170432.74C738136@code0.codespeak.net> Author: arigo Date: Fri Oct 19 19:04:32 2007 New Revision: 47607 Modified: pypy/dist/pypy/rpython/memory/gc/base.py Log: This nursery size is probably saner. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Fri Oct 19 19:04:32 2007 @@ -192,7 +192,7 @@ return SemiSpaceGC, GC_PARAMS elif config.translation.gc == "generation": GC_PARAMS = {'space_size': 8*1024*1024, # XXX adjust - 'nursery_size': 12*1024} + 'nursery_size': 896*1024} from pypy.rpython.memory.gc.generation import GenerationGC return GenerationGC, GC_PARAMS else: From arigo at codespeak.net Fri Oct 19 19:14:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 19:14:20 +0200 (CEST) Subject: [pypy-svn] r47608 - pypy/dist/dotviewer Message-ID: <20071019171420.78956812E@code0.codespeak.net> Author: arigo Date: Fri Oct 19 19:14:18 2007 New Revision: 47608 Modified: pypy/dist/dotviewer/graphclient.py pypy/dist/dotviewer/msgstruct.py Log: Windows doesn't support this way of using popen. Replace it with an explicit flush(). Modified: pypy/dist/dotviewer/graphclient.py ============================================================================== --- pypy/dist/dotviewer/graphclient.py (original) +++ pypy/dist/dotviewer/graphclient.py Fri Oct 19 19:14:18 2007 @@ -130,6 +130,6 @@ else: python = sys.executable cmdline = '"%s" -u "%s" --stdio' % (python, GRAPHSERVER) - child_in, child_out = os.popen2(cmdline, 'tb', 0) + child_in, child_out = os.popen2(cmdline, 'tb') io = msgstruct.FileIO(child_out, child_in) return io Modified: pypy/dist/dotviewer/msgstruct.py ============================================================================== --- pypy/dist/dotviewer/msgstruct.py (original) +++ pypy/dist/dotviewer/msgstruct.py Fri Oct 19 19:14:18 2007 @@ -93,6 +93,7 @@ def sendall(self, data): self.f_out.write(data) + self.f_out.flush() def recv(self): fd = self.f_in.fileno() From arigo at codespeak.net Fri Oct 19 19:30:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 19:30:15 +0200 (CEST) Subject: [pypy-svn] r47609 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071019173015.76A4D8133@code0.codespeak.net> Author: arigo Date: Fri Oct 19 19:30:15 2007 New Revision: 47609 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py Log: Allocate varsized objects in the nursery as well, if they are not too large. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Fri Oct 19 19:30:15 2007 @@ -61,6 +61,29 @@ self.nursery_free = result + totalsize return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) + def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, + can_collect, has_finalizer=False): + # only use the nursery if there are not too many items + if (has_finalizer or not can_collect or + length > self.nursery_size // 4 // raw_malloc_usage(itemsize) or + raw_malloc_usage(size) > self.nursery_size // 4): + return SemiSpaceGC.malloc_varsize(self, typeid, length, size, + itemsize, offset_to_length, + can_collect, has_finalizer) + # with the above checks we know now that totalsize cannot be more + # than about half of the nursery size; in particular, the + and * + # cannot overflow + size_gc_header = self.gcheaderbuilder.size_gc_header + totalsize = size_gc_header + size + itemsize * length + result = self.nursery_free + if raw_malloc_usage(totalsize) > self.nursery_top - result: + result = self.collect_nursery() + llarena.arena_reserve(result, totalsize) + self.init_gc_object(result, typeid) + (result + size_gc_header + offset_to_length).signed[0] = length + self.nursery_free = result + llarena.round_up_for_allocation(totalsize) + return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) + def semispace_collect(self, size_changing=False): self.reset_forwarding() # we are doing a full collection anyway self.reset_nursery() From arigo at codespeak.net Fri Oct 19 21:12:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 21:12:12 +0200 (CEST) Subject: [pypy-svn] r47610 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071019191212.0F64E813F@code0.codespeak.net> Author: arigo Date: Fri Oct 19 21:12:11 2007 New Revision: 47610 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: Register myself. Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Fri Oct 19 21:12:11 2007 @@ -15,6 +15,7 @@ Laura Creighton lives there Christian Tismer Laura and Jacob's Maciej Fijalkowski 19-26 Laura and Jacob's +Armin Rigo lives thereish ==================== ============== ===================== @@ -23,7 +24,6 @@ ==================== ============== ===================== Name Arrive/Depart Accomodation ==================== ============== ===================== -Armin Rigo ? ? Michael Hudson ? ? Antonio Cuni ? ? Anders Chrigstroem ? ? From arigo at codespeak.net Fri Oct 19 22:01:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 22:01:26 +0200 (CEST) Subject: [pypy-svn] r47611 - in pypy/dist/pypy/rpython/memory: gc test Message-ID: <20071019200126.BA27C8121@code0.codespeak.net> Author: arigo Date: Fri Oct 19 22:01:25 2007 New Revision: 47611 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/test/test_gc.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: (cfbolz, arigo) Fix weakrefs for the GenerationGC. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Fri Oct 19 22:01:25 2007 @@ -34,6 +34,7 @@ self.reset_nursery() self.old_objects_pointing_to_young = nonnull_endmarker # ^^^ the head of a linked list inside the old objects space + self.young_objects_with_weakrefs = self.AddressLinkedList() def reset_nursery(self): self.nursery = llmemory.NULL @@ -45,8 +46,9 @@ def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): - if (has_finalizer or contains_weakptr or not can_collect or + if (has_finalizer or not can_collect or raw_malloc_usage(size) >= self.nursery_size // 2): + debug_assert(not contains_weakptr, "wrong case for mallocing weakref") # "non-simple" case or object too big: don't use the nursery return SemiSpaceGC.malloc_fixedsize(self, typeid, size, can_collect, has_finalizer, @@ -59,6 +61,8 @@ llarena.arena_reserve(result, totalsize) self.init_gc_object(result, typeid) self.nursery_free = result + totalsize + if contains_weakptr: + self.young_objects_with_weakrefs.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, @@ -86,6 +90,7 @@ def semispace_collect(self, size_changing=False): self.reset_forwarding() # we are doing a full collection anyway + self.weakrefs_grow_older() self.reset_nursery() SemiSpaceGC.semispace_collect(self, size_changing) @@ -97,6 +102,11 @@ hdr.forw = llmemory.NULL self.old_objects_pointing_to_young = nonnull_endmarker + def weakrefs_grow_older(self): + while self.young_objects_with_weakrefs.non_empty(): + obj = self.young_objects_with_weakrefs.pop() + self.objects_with_weakrefs.append(obj) + def collect_nursery(self): if self.nursery_size > self.top_of_space - self.free: # the semispace is running out, do a full collect @@ -109,6 +119,8 @@ self.collect_oldrefs_to_nursery() self.collect_roots_in_nursery() self.scan_objects_just_copied_out_of_nursery(scan) + if self.young_objects_with_weakrefs.non_empty(): + self.invalidate_young_weakrefs() self.notify_objects_just_moved() # mark the nursery as free and fill it with zeroes again llarena.arena_reset(self.nursery, self.nursery_size, True) @@ -185,6 +197,25 @@ j += 1 i += 1 + def invalidate_young_weakrefs(self): + # walk over the list of objects that contain weakrefs and are in the + # nursery. if the object it references survives then update the + # weakref; otherwise invalidate the weakref + while self.young_objects_with_weakrefs.non_empty(): + obj = self.young_objects_with_weakrefs.pop() + if not self.is_forwarded(obj): + continue # weakref itself dies + obj = self.get_forwarding_address(obj) + offset = self.weakpointer_offset(self.header(obj).typeid) + pointing_to = (obj + offset).address[0] + if self.is_in_nursery(pointing_to): + if self.is_forwarded(pointing_to): + (obj + offset).address[0] = self.get_forwarding_address( + pointing_to) + self.objects_with_weakrefs.append(obj) + else: + (obj + offset).address[0] = NULL + def write_barrier(self, addr, addr_to, addr_struct): if not self.is_in_nursery(addr_struct) and self.is_in_nursery(addr): oldhdr = self.header(addr_struct) Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Fri Oct 19 22:01:25 2007 @@ -307,6 +307,25 @@ res = self.interpret(f, [500]) assert res == 1 + 500 + def test_weakref_across_minor_collection(self): + import weakref + class A: + pass + def f(x): + a = A() + a.foo = x + ref = weakref.ref(a) + all = [None] * x + i = 0 + while i < x: + all[i] = [i] * i + i += 1 + assert ref() is a + llop.gc__collect(lltype.Void) + assert ref() is a + return a.foo + len(all) + res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection + assert res == 20 + 20 class TestMarkSweepGC(GCTest): from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Fri Oct 19 22:01:25 2007 @@ -736,3 +736,25 @@ GC_PARAMS = {'space_size': 2048, 'nursery_size': 128} root_stack_depth = 200 + + def test_weakref_across_minor_collection(self): + import weakref + class A: + pass + def f(): + x = 20 # for GenerationGC, enough for a minor collection + a = A() + a.foo = x + ref = weakref.ref(a) + all = [None] * x + i = 0 + while i < x: + all[i] = [i] * i + i += 1 + assert ref() is a + llop.gc__collect(lltype.Void) + assert ref() is a + return a.foo + len(all) + run = self.runner(f) + res = run([]) + assert res == 20 + 20 From cfbolz at codespeak.net Fri Oct 19 22:58:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Oct 2007 22:58:47 +0200 (CEST) Subject: [pypy-svn] r47612 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071019205847.8C2578134@code0.codespeak.net> Author: cfbolz Date: Fri Oct 19 22:58:46 2007 New Revision: 47612 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/semispace.py Log: consistently use a method to get the type id. makes it possible to introduce flag bits in the type it field later. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Fri Oct 19 22:58:46 2007 @@ -171,8 +171,7 @@ """obj must not be in the nursery. This copies all the young objects it references out of the nursery. """ - gc_info = self.header(obj) - typeid = gc_info.typeid + typeid = self.get_type_id(obj) offsets = self.offsets_to_gc_pointers(typeid) i = 0 while i < len(offsets): Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Fri Oct 19 22:58:46 2007 @@ -227,8 +227,7 @@ return newobj def trace_and_copy(self, obj): - gc_info = self.header(obj) - typeid = gc_info.typeid + typeid = self.get_type_id(obj) offsets = self.offsets_to_gc_pointers(typeid) i = 0 while i < len(offsets): @@ -264,7 +263,7 @@ gc_info.forw = newobj def get_size(self, obj): - typeid = self.header(obj).typeid + typeid = self.get_type_id(obj) size = self.fixed_size(typeid) if self.is_varsize(typeid): lenaddr = obj + self.varsize_offset_to_length(typeid) @@ -277,6 +276,9 @@ addr -= self.gcheaderbuilder.size_gc_header return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) + def get_type_id(self, addr): + return self.header(addr).typeid + def init_gc_object(self, addr, typeid): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) #hdr.forw = NULL -- unneeded, the space is initially filled with zero @@ -340,6 +342,7 @@ self.finalizer_lock_count = 1 try: while self.run_finalizers.non_empty(): + #print "finalizer" obj = self.run_finalizers.pop() hdr = self.header(obj) finalizer = self.getfinalizer(hdr.typeid) From arigo at codespeak.net Fri Oct 19 23:37:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Oct 2007 23:37:59 +0200 (CEST) Subject: [pypy-svn] r47613 - pypy/dist/pypy/translator/c Message-ID: <20071019213759.96F8D811F@code0.codespeak.net> Author: arigo Date: Fri Oct 19 23:37:58 2007 New Revision: 47613 Modified: pypy/dist/pypy/translator/c/genc.py Log: Generate a better Makefile. This one supports commands like 'make -j3 debug'. Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Fri Oct 19 23:37:58 2007 @@ -901,23 +901,23 @@ \trm -f $(OBJECTS) $(TARGET) debug: -\tmake CFLAGS="-g -DRPY_ASSERT" +\t$(MAKE) CFLAGS="-g -DRPY_ASSERT" debug_exc: -\tmake CFLAGS="-g -DRPY_ASSERT -DDO_LOG_EXC" +\t$(MAKE) CFLAGS="-g -DRPY_ASSERT -DDO_LOG_EXC" debug_mem: -\tmake CFLAGS="-g -DRPY_ASSERT -DNO_OBMALLOC" +\t$(MAKE) CFLAGS="-g -DRPY_ASSERT -DNO_OBMALLOC" llsafer: -\tmake CFLAGS="-O2 -DRPY_LL_ASSERT" +\t$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" profile: -\tmake CFLAGS="-g -pg $(CFLAGS)" LDFLAGS="-pg $(LDFLAGS)" +\t$(MAKE) CFLAGS="-g -pg $(CFLAGS)" LDFLAGS="-pg $(LDFLAGS)" profopt: -\tmake CFLAGS="-fprofile-generate $(CFLAGS)" LDFLAGS="-fprofile-generate $(LDFLAGS)" +\t$(MAKE) CFLAGS="-fprofile-generate $(CFLAGS)" LDFLAGS="-fprofile-generate $(LDFLAGS)" \t./$(TARGET) $(PROFOPT) \trm -f $(OBJECTS) $(TARGET) -\tmake CFLAGS="-fprofile-use $(CFLAGS)" LDFLAGS="-fprofile-use $(LDFLAGS)" +\t$(MAKE) CFLAGS="-fprofile-use $(CFLAGS)" LDFLAGS="-fprofile-use $(LDFLAGS)" ''' From cfbolz at codespeak.net Fri Oct 19 23:38:03 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Oct 2007 23:38:03 +0200 (CEST) Subject: [pypy-svn] r47614 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071019213803.5C758812D@code0.codespeak.net> Author: cfbolz Date: Fri Oct 19 23:38:02 2007 New Revision: 47614 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: write a log line about the number of write barrier calls too. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Fri Oct 19 23:38:02 2007 @@ -122,6 +122,7 @@ StackRootIterator = self.build_stack_root_iterator() gcdata.gc = GCClass(AddressLinkedList, get_roots=StackRootIterator, **GC_PARAMS) self.num_pushs = 0 + self.write_barrier_calls = 0 def frameworkgc_setup(): # run-time initialization code @@ -379,6 +380,8 @@ log.info("assigned %s typeids" % (len(table), )) log.info("added %s push/pop stack root instructions" % ( self.num_pushs, )) + log.info("inserted %s write barrier calls" % ( + self.write_barrier_calls, )) # replace the type_info_table pointer in gcdata -- at this point, # the database is in principle complete, so it has already seen @@ -580,6 +583,7 @@ if self.write_barrier_ptr is None or isinstance(v_newvalue, Constant): super(FrameworkGCTransformer, self).transform_generic_set(hop) else: + self.write_barrier_calls += 1 assert isinstance(v_newvalue.concretetype, lltype.Ptr) opname = hop.spaceop.opname assert opname in ('setfield', 'setarrayitem', 'setinteriorfield') From cfbolz at codespeak.net Sat Oct 20 01:40:49 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 20 Oct 2007 01:40:49 +0200 (CEST) Subject: [pypy-svn] r47616 - in pypy/dist/pypy/rpython/memory: . gc gctransform Message-ID: <20071019234049.A30A88134@code0.codespeak.net> Author: cfbolz Date: Sat Oct 20 01:40:48 2007 New Revision: 47616 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gcwrapper.py Log: check in an idea that will likely not work as easily to not have it get lost (I will revert it afterwards): don't walk the full list of static roots at every minor collection, instead keep a list of static structures that had young pointers stored into them since the last collection. This does not work, because it ignores the static roots that are part of a nongc-struct/arrays. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Sat Oct 20 01:40:48 2007 @@ -1,5 +1,5 @@ import sys -from pypy.rpython.memory.gc.semispace import SemiSpaceGC +from pypy.rpython.memory.gc.semispace import SemiSpaceGC, GCFLAG_IMMORTAL from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object, debug_assert @@ -7,6 +7,7 @@ nonnull_endmarker = llmemory.raw_malloc(llmemory.sizeof(lltype.Char)) llmemory.raw_memclear(nonnull_endmarker, llmemory.sizeof(lltype.Char)) +GCFLAG_REMEMBERED = 2 class GenerationGC(SemiSpaceGC): """A basic generational GC: it's a SemiSpaceGC with an additional @@ -35,6 +36,7 @@ self.old_objects_pointing_to_young = nonnull_endmarker # ^^^ the head of a linked list inside the old objects space self.young_objects_with_weakrefs = self.AddressLinkedList() + self.static_to_young_pointer = self.AddressLinkedList() def reset_nursery(self): self.nursery = llmemory.NULL @@ -90,6 +92,7 @@ def semispace_collect(self, size_changing=False): self.reset_forwarding() # we are doing a full collection anyway + self.reset_static() self.weakrefs_grow_older() self.reset_nursery() SemiSpaceGC.semispace_collect(self, size_changing) @@ -102,6 +105,12 @@ hdr.forw = llmemory.NULL self.old_objects_pointing_to_young = nonnull_endmarker + def reset_static(self): + while self.static_to_young_pointer.non_empty(): + obj = self.static_to_young_pointer.pop() + hdr = self.header(obj) + hdr.typeid &= ~GCFLAG_REMEMBERED + def weakrefs_grow_older(self): while self.young_objects_with_weakrefs.non_empty(): obj = self.young_objects_with_weakrefs.pop() @@ -116,6 +125,7 @@ if self.nursery: # a nursery-only collection scan = self.free + self.collect_static_to_nursery() self.collect_oldrefs_to_nursery() self.collect_roots_in_nursery() self.scan_objects_just_copied_out_of_nursery(scan) @@ -150,7 +160,7 @@ self.old_objects_pointing_to_young = nonnull_endmarker def collect_roots_in_nursery(self): - roots = self.get_roots() + roots = self.get_roots(with_static=False) while 1: root = roots.pop() if root == NULL: @@ -160,6 +170,13 @@ root.address[0] = self.copy(obj) free_non_gc_object(roots) + def collect_static_to_nursery(self): + while self.static_to_young_pointer.non_empty(): + obj = self.static_to_young_pointer.pop() + hdr = self.header(obj) + hdr.typeid &= ~GCFLAG_REMEMBERED + self.trace_and_drag_out_of_nursery(obj) + def scan_objects_just_copied_out_of_nursery(self, scan): while scan < self.free: curr = scan + self.size_gc_header() @@ -217,8 +234,17 @@ def write_barrier(self, addr, addr_to, addr_struct): if not self.is_in_nursery(addr_struct) and self.is_in_nursery(addr): - oldhdr = self.header(addr_struct) - if oldhdr.forw == NULL: - oldhdr.forw = self.old_objects_pointing_to_young - self.old_objects_pointing_to_young = addr_struct + self.remember_young_pointer(addr_struct, addr) addr_to.address[0] = addr + + + def remember_young_pointer(self, addr_struct, addr): + oldhdr = self.header(addr_struct) + if oldhdr.forw == NULL: + oldhdr.forw = self.old_objects_pointing_to_young + self.old_objects_pointing_to_young = addr_struct + elif (oldhdr.typeid >> 16) & (GCFLAG_IMMORTAL | GCFLAG_REMEMBERED) == ( + GCFLAG_IMMORTAL): + self.static_to_young_pointer.append(addr_struct) + oldhdr.typeid |= GCFLAG_REMEMBERED + remember_young_pointer.dont_inline = True Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Sat Oct 20 01:40:48 2007 @@ -14,6 +14,8 @@ memoryError = MemoryError() +GCFLAG_IMMORTAL = 1 + class SemiSpaceGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True @@ -277,7 +279,7 @@ return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) def get_type_id(self, addr): - return self.header(addr).typeid + return self.header(addr).typeid & 0xff def init_gc_object(self, addr, typeid): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) @@ -288,7 +290,7 @@ # immortal objects always have forward to themselves hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) hdr.forw = addr + self.gcheaderbuilder.size_gc_header - hdr.typeid = typeid + hdr.typeid = typeid | (GCFLAG_IMMORTAL << 16) def deal_with_objects_with_finalizers(self): # walk over list of objects with finalizers Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Sat Oct 20 01:40:48 2007 @@ -339,9 +339,12 @@ return top.address[0] pop_root = staticmethod(pop_root) - def __init__(self): + def __init__(self, with_static=True): self.stack_current = gcdata.root_stack_top - self.static_current = gcdata.static_root_start + if with_static: + self.static_current = gcdata.static_root_start + else: + self.static_current = gcdata.static_root_end def pop(self): while self.static_current != gcdata.static_root_end: Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Sat Oct 20 01:40:48 2007 @@ -27,12 +27,13 @@ self.constantroots = layoutbuilder.addresses_of_static_ptrs - def get_roots_from_llinterp(self): + def get_roots_from_llinterp(self, with_static=True): sizeofaddr = llmemory.sizeof(llmemory.Address) ll = [llmemory.NULL] # end marker - for addrofaddr in self.constantroots: - if addrofaddr.address[0]: - ll.append(addrofaddr) + if with_static: + for addrofaddr in self.constantroots: + if addrofaddr.address[0]: + ll.append(addrofaddr) for addrofaddr in self.llinterp.find_roots(): if addrofaddr.address[0]: ll.append(addrofaddr) From cfbolz at codespeak.net Sat Oct 20 01:42:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 20 Oct 2007 01:42:47 +0200 (CEST) Subject: [pypy-svn] r47617 - pypy/dist/pypy/rpython/memory Message-ID: <20071019234247.B35608134@code0.codespeak.net> Author: cfbolz Date: Sat Oct 20 01:42:47 2007 New Revision: 47617 Added: pypy/dist/pypy/rpython/memory/ - copied from r47615, pypy/dist/pypy/rpython/memory/ Log: revert last commit From cfbolz at codespeak.net Sat Oct 20 09:59:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 20 Oct 2007 09:59:38 +0200 (CEST) Subject: [pypy-svn] r47618 - in pypy/extradoc/talk: . agile2006 Message-ID: <20071020075938.E905E8146@code0.codespeak.net> Author: cfbolz Date: Sat Oct 20 09:59:37 2007 New Revision: 47618 Added: pypy/extradoc/talk/agile2006/during_sprintdriven.odp - copied unchanged from r47617, pypy/extradoc/talk/during_sprintdriven.odp Removed: pypy/extradoc/talk/during_sprintdriven.odp Log: move this talk into the proper folder From cfbolz at codespeak.net Sat Oct 20 10:21:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 20 Oct 2007 10:21:44 +0200 (CEST) Subject: [pypy-svn] r47619 - pypy/extradoc/talk/bern2007 Message-ID: <20071020082144.0EB718144@code0.codespeak.net> Author: cfbolz Date: Sat Oct 20 10:21:43 2007 New Revision: 47619 Added: pypy/extradoc/talk/bern2007/ pypy/extradoc/talk/bern2007/sprint-facts.txt (contents, props changed) Log: try to do a two-slide summary of sprint-driven development for the bern sprint. Added: pypy/extradoc/talk/bern2007/sprint-facts.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/bern2007/sprint-facts.txt Sat Oct 20 10:21:43 2007 @@ -0,0 +1,29 @@ +============================================= +Two page summary of sprint-driven development +============================================= + +Overview +======== + +A sprint is a very focused coding session to drive forward development quickly +in an explorative style. + + - Structure: + + - 6-20 developers in one room + - 5-7 days + - predefined goals + +Details +======= + + - daily planning session in the morning + + - discuss status of the work done on the previous day + - assign tasks to pairs of people + - produce a status file, which is updated during the day + + - development + + - Pair-programming + - Test-driven From fijal at codespeak.net Sat Oct 20 11:59:05 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 11:59:05 +0200 (CEST) Subject: [pypy-svn] r47620 - in pypy/dist/pypy/rpython/tool: . test Message-ID: <20071020095905.17CC68150@code0.codespeak.net> Author: fijal Date: Sat Oct 20 11:59:04 2007 New Revision: 47620 Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py Log: Support for proper COpaqueType Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Sat Oct 20 11:59:04 2007 @@ -42,13 +42,13 @@ HAS = Has(name) return configure(CConfig)['HAS'] -def sizeof(name, c_header_source, **kwds): +def copaque(name, c_header_source, **kwds): class CConfig: _header_ = c_header_source - SIZE = SizeOf(name) + C = COpaquePtr(name) for k, v in kwds.items(): setattr(CConfig, k, v) - return configure(CConfig)['SIZE'] + return configure(CConfig)['C'] # ____________________________________________________________ # @@ -278,9 +278,9 @@ kwds = {'hints': hints} return rffi.CStruct(name, *fields, **kwds) -class SizeOf(CConfigEntry): - """An entry in a CConfig class that stands for sizeof - of some external opaque type +class COpaquePtr(CConfigEntry): + """An entry in a CConfig class that stands for + some external opaque type """ def __init__(self, name): self.name = name @@ -289,7 +289,7 @@ yield 'dump("size", sizeof(%s));' % self.name def build_result(self, info, config_result): - return info['size'] + return rffi.COpaquePtr(self.name, info['size']) class SimpleType(CConfigEntry): """An entry in a CConfig class that stands for an externally Modified: pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py Sat Oct 20 11:59:04 2007 @@ -193,4 +193,4 @@ assert not rffi_platform.has("x", "#include ") def test_sizeof(): - assert rffi_platform.sizeof("char", "") == 1 + assert rffi_platform.copaque("char", "").TO.hints['size'] == 1 From lac at codespeak.net Sat Oct 20 11:59:32 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Sat, 20 Oct 2007 11:59:32 +0200 (CEST) Subject: [pypy-svn] r47621 - pypy/extradoc Message-ID: <20071020095932.AD4538151@code0.codespeak.net> Author: lac Date: Sat Oct 20 11:59:32 2007 New Revision: 47621 Modified: pypy/extradoc/confrest.py Log: fix link that went nowhere Modified: pypy/extradoc/confrest.py ============================================================================== --- pypy/extradoc/confrest.py (original) +++ pypy/extradoc/confrest.py Sat Oct 20 11:59:32 2007 @@ -9,7 +9,7 @@ html.a("doc", href="/pypy/dist/pypy/doc/index.html", class_="menu"), " ", html.a("contact", href="/pypy/dist/pypy/doc/contact.html", class_="menu"), " ", html.a("getting-started", - href="/pypy/dist/doc/getting-started.html", class_="menu"), " ", + href="/pypy/dist/pypy/doc/getting-started.html", class_="menu"), " ", html.a("issue", href="https://codespeak.net/issue/pypy-dev/", class_="menu"), From fijal at codespeak.net Sat Oct 20 12:05:26 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 12:05:26 +0200 (CEST) Subject: [pypy-svn] r47622 - pypy/dist/pypy/rpython/tool Message-ID: <20071020100526.93C3D813F@code0.codespeak.net> Author: fijal Date: Sat Oct 20 12:05:26 2007 New Revision: 47622 Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py Log: Support for proper argument passing Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Sat Oct 20 12:05:26 2007 @@ -289,7 +289,12 @@ yield 'dump("size", sizeof(%s));' % self.name def build_result(self, info, config_result): - return rffi.COpaquePtr(self.name, info['size']) + # XXX this is strange mapping, but well, I've got no + # better idea + kwds = {} + for item in ['includes', 'include_dirs', 'libraries']: + kwds[item] = getattr(config_result, '_%s_' % item, []) + return rffi.COpaquePtr(self.name, info['size'], **kwds) class SimpleType(CConfigEntry): """An entry in a CConfig class that stands for an externally From fijal at codespeak.net Sat Oct 20 12:05:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 12:05:46 +0200 (CEST) Subject: [pypy-svn] r47623 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20071020100546.09064813F@code0.codespeak.net> Author: fijal Date: Sat Oct 20 12:05:45 2007 New Revision: 47623 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Log: Adapt rffi to new interface of rffi_platform. Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Sat Oct 20 12:05:45 2007 @@ -172,7 +172,7 @@ elif isinstance(T, lltype.OpaqueType): if T.hints.get('external', None) != 'C': raise TypeError("%s is not external" % T) - return ctypes.c_char * T.hints['getsize']() + return ctypes.c_char * T.hints['size'] else: _setup_ctypes_cache() if T in _ctypes_cache: @@ -410,7 +410,7 @@ elif isinstance(T.TO, lltype.Array): convert_array(container) elif isinstance(T.TO, lltype.OpaqueType): - cbuf = ctypes.create_string_buffer(T.TO.hints['getsize']()) + cbuf = ctypes.create_string_buffer(T.TO.hints['size']) add_storage(container, _parentable_mixin, cbuf) else: raise NotImplementedError(T) Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Sat Oct 20 12:05:45 2007 @@ -222,20 +222,14 @@ return lltype.Ptr(CArray(tp)) CArray._annspecialcase_ = 'specialize:memo' -def COpaque(name, hints=None, **kwds): +def COpaque(name, size, hints=None, **kwds): if hints is None: hints = {} else: hints = hints.copy() hints['external'] = 'C' hints['c_name'] = name - def lazy_getsize(result=[]): - if not result: - from pypy.rpython.tool import rffi_platform - size = rffi_platform.sizeof(name, "", **kwds) - result.append(size) - return result[0] - hints['getsize'] = lazy_getsize + hints['size'] = size return lltype.OpaqueType(name, hints) def COpaquePtr(*args, **kwds): Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sat Oct 20 12:05:45 2007 @@ -2,6 +2,7 @@ import sys, struct import ctypes from pypy.rpython.lltypesystem import lltype, rffi, llmemory +from pypy.rpython.tool import rffi_platform from pypy.rpython.lltypesystem.ll2ctypes import lltype2ctypes, ctypes2lltype from pypy.rpython.lltypesystem.ll2ctypes import standard_c_lib from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes @@ -274,8 +275,8 @@ def test_opaque_obj(self): includes = ['sys/time.h', 'time.h'] - TIMEVALP = rffi.COpaquePtr('struct timeval', includes=includes) - TIMEZONEP = rffi.COpaquePtr('struct timezone', includes=includes) + TIMEVALP = rffi_platform.copaque('struct timeval', '', _includes_=includes) + TIMEZONEP = rffi_platform.copaque('struct timezone', '', _includes_=includes) gettimeofday = rffi.llexternal('gettimeofday', [TIMEVALP, TIMEZONEP], rffi.INT, includes=includes) ll_timevalp = lltype.malloc(TIMEVALP.TO, flavor='raw') Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Sat Oct 20 12:05:45 2007 @@ -223,8 +223,9 @@ h_file = udir.join("opaque.h") h_file.write(h_source) - STUFFP = COpaquePtr('struct stuff', includes=['opaque.h'], - include_dirs=[str(udir)]) + from pypy.rpython.tool import rffi_platform + STUFFP = rffi_platform.copaque('struct stuff', '', _includes_=['opaque.h'], + _include_dirs_=[str(udir)]) ll_get = llexternal('get', [STUFFP], lltype.Char, includes=['opaque.h'], include_dirs=[str(udir)]) From fijal at codespeak.net Sat Oct 20 12:08:09 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 12:08:09 +0200 (CEST) Subject: [pypy-svn] r47624 - pypy/dist/pypy/rpython/module Message-ID: <20071020100809.84E608150@code0.codespeak.net> Author: fijal Date: Sat Oct 20 12:08:08 2007 New Revision: 47624 Modified: pypy/dist/pypy/rpython/module/ll_os.py Log: Fix COpaquePtr reference Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Sat Oct 20 12:08:08 2007 @@ -621,11 +621,12 @@ _includes_ = ['sys/types.h', 'dirent.h'] DIRENT = platform.Struct('struct dirent', [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]) + DIRP = platform.COpaquePtr('DIR') config = platform.configure(CConfig) DIRENT = config['DIRENT'] DIRENTP = lltype.Ptr(DIRENT) - DIRP = rffi.COpaquePtr('DIR') + DIRP = config['DIRP'] os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP, includes=CConfig._includes_) os_readdir = self.llexternal('readdir', [DIRP], DIRENTP, From fijal at codespeak.net Sat Oct 20 12:11:37 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 12:11:37 +0200 (CEST) Subject: [pypy-svn] r47625 - pypy/dist/pypy/rpython/module Message-ID: <20071020101137.2ECD78150@code0.codespeak.net> Author: fijal Date: Sat Oct 20 12:11:36 2007 New Revision: 47625 Modified: pypy/dist/pypy/rpython/module/ll_os.py Log: Typo Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Sat Oct 20 12:11:36 2007 @@ -621,7 +621,7 @@ _includes_ = ['sys/types.h', 'dirent.h'] DIRENT = platform.Struct('struct dirent', [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]) - DIRP = platform.COpaquePtr('DIR') + DIRP = platform.COpaquePtr('DIRP') config = platform.configure(CConfig) DIRENT = config['DIRENT'] From fijal at codespeak.net Sat Oct 20 12:14:56 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 12:14:56 +0200 (CEST) Subject: [pypy-svn] r47626 - in pypy/dist/pypy/module: bz2 thread Message-ID: <20071020101456.4831E8150@code0.codespeak.net> Author: fijal Date: Sat Oct 20 12:14:55 2007 New Revision: 47626 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/thread/ll_thread.py Log: Make this modules as well to use rffi_platform.COpaque Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Oct 20 12:14:55 2007 @@ -34,6 +34,9 @@ ('bzfree', lltype.Ptr(_free_type)), ('opaque', rffi.VOIDP), ]) + FILE = platform.COpaquePtr('FILE') + BZFILE = platform.COpaquePtr('BZFILE') + constants = {} constant_names = ['BZ_RUN', 'BZ_FLUSH', 'BZ_FINISH', 'BZ_OK', @@ -69,6 +72,8 @@ BZ_MEM_ERROR = cConfig.BZ_MEM_ERROR BZ_UNEXPECTED_EOF = cConfig.BZ_UNEXPECTED_EOF BZ_SEQUENCE_ERROR = cConfig.BZ_SEQUENCE_ERROR +BZFILE = cConfig.BZFILE +FILE = cConfig.FILE if BUFSIZ < 8192: SMALLCHUNK = 8192 @@ -100,9 +105,6 @@ return rffi.llexternal(name, args, result, includes=CConfig._includes_, libraries=['bz2']) -FILE = rffi.COpaquePtr('FILE') -BZFILE = rffi.COpaquePtr('BZFILE') - # the least but one parameter should be rffi.VOIDP but it's not used # so I trick the compiler to not complain about constanst pointer passed # to void* arg Modified: pypy/dist/pypy/module/thread/ll_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/ll_thread.py (original) +++ pypy/dist/pypy/module/thread/ll_thread.py Sat Oct 20 12:14:55 2007 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype -from pypy.rpython.lltypesystem.rffi import platform +from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.extfunc import genericcallable from pypy.rpython.annlowlevel import cast_instance_to_base_ptr from pypy.translator.tool.cbuild import cache_c_module @@ -18,6 +18,7 @@ pypydir = py.path.local(pypydir) c_dir = pypydir.join('translator', 'c') includes = ['unistd.h', 'src/thread.h'] +include_dirs = [str(c_dir)] def setup_thread_so(): # XXX this is quiiiiiiiite a hack! @@ -36,7 +37,9 @@ c_thread_start = llexternal('RPyThreadStart', [CALLBACK, rffi.VOIDP], rffi.INT) c_thread_get_ident = llexternal('RPyThreadGetIdent', [], rffi.INT) -TLOCKP = rffi.COpaquePtr('struct RPyOpaque_ThreadLock', includes=includes) +TLOCKP = platform.copaque('struct RPyOpaque_ThreadLock', '', + _includes_=includes, + _include_dirs_=include_dirs) c_thread_lock_init = llexternal('RPyThreadLockInit', [TLOCKP], lltype.Void) c_thread_acquirelock = llexternal('RPyThreadAcquireLock', [TLOCKP, rffi.INT], From fijal at codespeak.net Sat Oct 20 12:23:55 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 12:23:55 +0200 (CEST) Subject: [pypy-svn] r47627 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071020102355.D3D328150@code0.codespeak.net> Author: fijal Date: Sat Oct 20 12:23:55 2007 New Revision: 47627 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: Cleanup this file (there was like 2 armins, 2 samueles and 3 arre's) Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Sat Oct 20 12:23:55 2007 @@ -26,29 +26,23 @@ ==================== ============== ===================== Michael Hudson ? ? Antonio Cuni ? ? -Anders Chrigstroem ? ? Andrew Thompson ? ? Stephan Diehl ? ? Eric van Riet paap ? ? Niko Matsakis ? ? Leonardo Santagada ? ? Richard Emslie ? ? -Christian Tismer ? ? Alexander Schremmer ? ? -Jacob Hallen ? ? Aurelien Campeas ? ? Alexandre Fayolle ? ? Lene Wagner ? ? Amaury Forgeot d'Arc ? ? Valentino Volonghi ? ? -Boris Feigin ? ? +Boris Feigin ? ? Bert Freudenberg ? ? -Laura Creighton ? ? Beatrice Duering ? ? Johan Hahn ? ? Holger Krekel ? ? -Anders Chrigstroem ? ? -Samuele Pedroni ? ? Anders Lehmann ? ? Niklaus Haldimann ? ? ==================== ============== ===================== From arigo at codespeak.net Sat Oct 20 12:33:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Oct 2007 12:33:51 +0200 (CEST) Subject: [pypy-svn] r47628 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071020103351.8054E8150@code0.codespeak.net> Author: arigo Date: Sat Oct 20 12:33:51 2007 New Revision: 47628 Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py Log: Fix clear_large_memory_chunk() to avoid indirectly using malloc at all costs. This code runs during GC initialization, so calling malloc is unlikely to work! Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llarena.py Sat Oct 20 12:33:51 2007 @@ -280,28 +280,40 @@ if os.name == 'posix': READ_MAX = (sys.maxint//4) + 1 # upper bound on reads to avoid surprises - os_read = rffi.llexternal('read', - [rffi.INT, llmemory.Address, rffi.SIZE_T], - rffi.SIZE_T, - sandboxsafe=True) + raw_os_open = rffi.llexternal('open', + [rffi.CCHARP, rffi.INT, rffi.MODE_T], + rffi.INT, + sandboxsafe=True, _nowrapper=True) + raw_os_read = rffi.llexternal('read', + [rffi.INT, llmemory.Address, rffi.SIZE_T], + rffi.SIZE_T, + sandboxsafe=True, _nowrapper=True) + raw_os_close = rffi.llexternal('close', + [rffi.INT], + rffi.INT, + sandboxsafe=True, _nowrapper=True) + _dev_zero = rffi.str2charp('/dev/zero') # prebuilt def clear_large_memory_chunk(baseaddr, size): # on Linux at least, reading from /dev/zero is the fastest way # to clear arenas, because the kernel knows that it doesn't - # need to even allocate the pages before they are used - try: - fd = os.open('/dev/zero', os.O_RDONLY, 0644) - except OSError: - pass - else: + # need to even allocate the pages before they are used. + + # NB.: careful, don't do anything that could malloc here! + # this code is called during GC initialization. + fd = raw_os_open(_dev_zero, + rffi.cast(rffi.INT, os.O_RDONLY), + rffi.cast(rffi.MODE_T, 0644)) + if rffi.cast(lltype.Signed, fd) != -1: while size > 0: - count = os_read(fd, baseaddr, min(READ_MAX, size)) + size1 = rffi.cast(rffi.SIZE_T, min(READ_MAX, size)) + count = raw_os_read(fd, baseaddr, size1) count = rffi.cast(lltype.Signed, count) - if count < 0: + if count <= 0: break size -= count baseaddr += count - os.close(fd) + raw_os_close(fd) if size > 0: # reading from /dev/zero failed, fallback llmemory.raw_memclear(baseaddr, size) From fijal at codespeak.net Sat Oct 20 12:41:34 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 12:41:34 +0200 (CEST) Subject: [pypy-svn] r47629 - pypy/dist/pypy/rpython/module Message-ID: <20071020104134.B4259814E@code0.codespeak.net> Author: fijal Date: Sat Oct 20 12:41:34 2007 New Revision: 47629 Modified: pypy/dist/pypy/rpython/module/ll_os.py Log: This wasn't typo after all. Still CConfig explodes (no idea why) Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Sat Oct 20 12:41:34 2007 @@ -621,7 +621,7 @@ _includes_ = ['sys/types.h', 'dirent.h'] DIRENT = platform.Struct('struct dirent', [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]) - DIRP = platform.COpaquePtr('DIRP') + DIRP = platform.COpaquePtr('DIR') config = platform.configure(CConfig) DIRENT = config['DIRENT'] From arigo at codespeak.net Sat Oct 20 12:45:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Oct 2007 12:45:51 +0200 (CEST) Subject: [pypy-svn] r47630 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071020104551.22DBE814F@code0.codespeak.net> Author: arigo Date: Sat Oct 20 12:45:50 2007 New Revision: 47630 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/semispace.py Log: A few forgotten direct references to 'typeid'. Renamed the field to a shorter 'tid' to make sure no one uses it directly any more. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Sat Oct 20 12:45:50 2007 @@ -205,7 +205,7 @@ if not self.is_forwarded(obj): continue # weakref itself dies obj = self.get_forwarding_address(obj) - offset = self.weakpointer_offset(self.header(obj).typeid) + offset = self.weakpointer_offset(self.get_type_id(obj)) pointing_to = (obj + offset).address[0] if self.is_in_nursery(pointing_to): if self.is_forwarded(pointing_to): Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Sat Oct 20 12:45:50 2007 @@ -19,7 +19,7 @@ inline_simple_malloc = True HDR = lltype.Struct('header', ('forw', llmemory.Address), - ('typeid', lltype.Signed)) + ('tid', lltype.Signed)) def __init__(self, AddressLinkedList, space_size=4096, max_space_size=sys.maxint//2+1, @@ -277,18 +277,18 @@ return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) def get_type_id(self, addr): - return self.header(addr).typeid + return self.header(addr).tid def init_gc_object(self, addr, typeid): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) #hdr.forw = NULL -- unneeded, the space is initially filled with zero - hdr.typeid = typeid + hdr.tid = typeid def init_gc_object_immortal(self, addr, typeid): # immortal objects always have forward to themselves hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) hdr.forw = addr + self.gcheaderbuilder.size_gc_header - hdr.typeid = typeid + hdr.tid = typeid def deal_with_objects_with_finalizers(self): # walk over list of objects with finalizers @@ -314,7 +314,7 @@ if not self.is_forwarded(obj): continue # weakref itself dies obj = self.get_forwarding_address(obj) - offset = self.weakpointer_offset(self.header(obj).typeid) + offset = self.weakpointer_offset(self.get_type_id(obj)) pointing_to = (obj + offset).address[0] if pointing_to: if self.is_forwarded(pointing_to): @@ -344,8 +344,7 @@ while self.run_finalizers.non_empty(): #print "finalizer" obj = self.run_finalizers.pop() - hdr = self.header(obj) - finalizer = self.getfinalizer(hdr.typeid) + finalizer = self.getfinalizer(self.get_type_id(obj)) finalizer(obj) finally: self.finalizer_lock_count = 0 From fijal at codespeak.net Sat Oct 20 13:00:19 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 13:00:19 +0200 (CEST) Subject: [pypy-svn] r47631 - pypy/dist/pypy Message-ID: <20071020110019.1B1A88152@code0.codespeak.net> Author: fijal Date: Sat Oct 20 13:00:18 2007 New Revision: 47631 Modified: pypy/dist/pypy/conftest.py Log: Ick Modified: pypy/dist/pypy/conftest.py ============================================================================== --- pypy/dist/pypy/conftest.py (original) +++ pypy/dist/pypy/conftest.py Sat Oct 20 13:00:18 2007 @@ -286,7 +286,7 @@ return keyword == 'interplevel' or \ super(IntTestFunction, self)._haskeyword(keyword) def _keywords(self): - return super(IntTestFunction, self)._keywords + ['interplevel'] + return super(IntTestFunction, self)._keywords() + ['interplevel'] def execute(self, target, *args): co = target.func_code From fijal at codespeak.net Sat Oct 20 13:01:22 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 13:01:22 +0200 (CEST) Subject: [pypy-svn] r47632 - in pypy/dist/pypy/rpython: lltypesystem lltypesystem/test module tool tool/test Message-ID: <20071020110122.537A78153@code0.codespeak.net> Author: fijal Date: Sat Oct 20 13:01:21 2007 New Revision: 47632 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/tool/rffi_platform.py pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py Log: "Revert" changes from the morning - make COpaquePtr count it's size lazy, but still use rffi_platform for that Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Sat Oct 20 13:01:21 2007 @@ -172,7 +172,7 @@ elif isinstance(T, lltype.OpaqueType): if T.hints.get('external', None) != 'C': raise TypeError("%s is not external" % T) - return ctypes.c_char * T.hints['size'] + return ctypes.c_char * T.hints['getsize']() else: _setup_ctypes_cache() if T in _ctypes_cache: @@ -410,7 +410,7 @@ elif isinstance(T.TO, lltype.Array): convert_array(container) elif isinstance(T.TO, lltype.OpaqueType): - cbuf = ctypes.create_string_buffer(T.TO.hints['size']) + cbuf = ctypes.create_string_buffer(T.TO.hints['getsize']()) add_storage(container, _parentable_mixin, cbuf) else: raise NotImplementedError(T) Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Sat Oct 20 13:01:21 2007 @@ -222,14 +222,22 @@ return lltype.Ptr(CArray(tp)) CArray._annspecialcase_ = 'specialize:memo' -def COpaque(name, size, hints=None, **kwds): +def COpaque(name, hints=None, **kwds): if hints is None: hints = {} else: hints = hints.copy() hints['external'] = 'C' hints['c_name'] = name - hints['size'] = size + def lazy_getsize(): + from pypy.rpython.tool import rffi_platform + k = {} + for _name, value in kwds.items(): + if _name in ['includes', 'include_dirs', 'libraries']: + k['_%s_' % _name] = value + return rffi_platform.sizeof(name, '', **k) + + hints['getsize'] = lazy_getsize return lltype.OpaqueType(name, hints) def COpaquePtr(*args, **kwds): Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sat Oct 20 13:01:21 2007 @@ -275,8 +275,8 @@ def test_opaque_obj(self): includes = ['sys/time.h', 'time.h'] - TIMEVALP = rffi_platform.copaque('struct timeval', '', _includes_=includes) - TIMEZONEP = rffi_platform.copaque('struct timezone', '', _includes_=includes) + TIMEVALP = rffi.COpaquePtr('struct timeval', includes=includes) + TIMEZONEP = rffi.COpaquePtr('struct timezone', includes=includes) gettimeofday = rffi.llexternal('gettimeofday', [TIMEVALP, TIMEZONEP], rffi.INT, includes=includes) ll_timevalp = lltype.malloc(TIMEVALP.TO, flavor='raw') Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Sat Oct 20 13:01:21 2007 @@ -224,8 +224,8 @@ h_file.write(h_source) from pypy.rpython.tool import rffi_platform - STUFFP = rffi_platform.copaque('struct stuff', '', _includes_=['opaque.h'], - _include_dirs_=[str(udir)]) + STUFFP = COpaquePtr('struct stuff', includes=['opaque.h'], + include_dirs=[str(udir)]) ll_get = llexternal('get', [STUFFP], lltype.Char, includes=['opaque.h'], include_dirs=[str(udir)]) Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Sat Oct 20 13:01:21 2007 @@ -621,12 +621,11 @@ _includes_ = ['sys/types.h', 'dirent.h'] DIRENT = platform.Struct('struct dirent', [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]) - DIRP = platform.COpaquePtr('DIR') + DIRP = rffi.COpaquePtr('DIR') config = platform.configure(CConfig) DIRENT = config['DIRENT'] DIRENTP = lltype.Ptr(DIRENT) - DIRP = config['DIRP'] os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP, includes=CConfig._includes_) os_readdir = self.llexternal('readdir', [DIRP], DIRENTP, Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Sat Oct 20 13:01:21 2007 @@ -42,13 +42,13 @@ HAS = Has(name) return configure(CConfig)['HAS'] -def copaque(name, c_header_source, **kwds): +def sizeof(name, c_header_source, **kwds): class CConfig: _header_ = c_header_source - C = COpaquePtr(name) + SIZE = SizeOf(name) for k, v in kwds.items(): setattr(CConfig, k, v) - return configure(CConfig)['C'] + return configure(CConfig)['SIZE'] # ____________________________________________________________ # @@ -278,24 +278,6 @@ kwds = {'hints': hints} return rffi.CStruct(name, *fields, **kwds) -class COpaquePtr(CConfigEntry): - """An entry in a CConfig class that stands for - some external opaque type - """ - def __init__(self, name): - self.name = name - - def prepare_code(self): - yield 'dump("size", sizeof(%s));' % self.name - - def build_result(self, info, config_result): - # XXX this is strange mapping, but well, I've got no - # better idea - kwds = {} - for item in ['includes', 'include_dirs', 'libraries']: - kwds[item] = getattr(config_result, '_%s_' % item, []) - return rffi.COpaquePtr(self.name, info['size'], **kwds) - class SimpleType(CConfigEntry): """An entry in a CConfig class that stands for an externally defined simple numeric type. @@ -440,6 +422,19 @@ def question(self, ask_gcc): return ask_gcc(self.name + ';') +class SizeOf(CConfigEntry): + """An entry in a CConfig class that stands for + some external opaque type + """ + def __init__(self, name): + self.name = name + + def prepare_code(self): + yield 'dump("size", sizeof(%s));' % self.name + + def build_result(self, info, config_result): + return info['size'] + # ____________________________________________________________ # # internal helpers Modified: pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py Sat Oct 20 13:01:21 2007 @@ -193,4 +193,4 @@ assert not rffi_platform.has("x", "#include ") def test_sizeof(): - assert rffi_platform.copaque("char", "").TO.hints['size'] == 1 + assert rffi_platform.sizeof("char", "") == 1 From mwh at codespeak.net Sat Oct 20 13:02:46 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sat, 20 Oct 2007 13:02:46 +0200 (CEST) Subject: [pypy-svn] r47633 - pypy/dist/pypy/translator/goal Message-ID: <20071020110246.6E2848156@code0.codespeak.net> Author: mwh Date: Sat Oct 20 13:02:46 2007 New Revision: 47633 Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py Log: optimism: test the generational gc too Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py ============================================================================== --- pypy/dist/pypy/translator/goal/bench-cronjob.py (original) +++ pypy/dist/pypy/translator/goal/bench-cronjob.py Sat Oct 20 13:02:46 2007 @@ -185,6 +185,7 @@ c--_allworkingmodules c--gc=marksweep--_faassen c--gc=semispace--_faassen + c--gc=generation--_faassen c--_objspace-std-withrope cli """.split('\n') if backend.strip() and not backend.strip().startswith('#')] From cfbolz at codespeak.net Sat Oct 20 15:17:20 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 20 Oct 2007 15:17:20 +0200 (CEST) Subject: [pypy-svn] r47642 - in pypy/dist/pypy/rpython/memory: . gc gctransform Message-ID: <20071020131720.2CD5A8158@code0.codespeak.net> Author: cfbolz Date: Sat Oct 20 15:17:18 2007 New Revision: 47642 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctypelayout.py pypy/dist/pypy/rpython/memory/gcwrapper.py Log: (cfbolz, arigo review) In minor collections of the GenerationGC, avoid scanning all static roots. It is enough to scan the static roots not living inside a GcStruct (there are only 3 of them in PyPy!) and remember which prebuilt GcStructs have got a pointer to a young object. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Sat Oct 20 15:17:18 2007 @@ -1,5 +1,6 @@ import sys -from pypy.rpython.memory.gc.semispace import SemiSpaceGC +from pypy.rpython.memory.gc.semispace import SemiSpaceGC, GCFLAGSHIFT, \ + GCFLAG_IMMORTAL from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object, debug_assert @@ -7,6 +8,7 @@ nonnull_endmarker = llmemory.raw_malloc(llmemory.sizeof(lltype.Char)) llmemory.raw_memclear(nonnull_endmarker, llmemory.sizeof(lltype.Char)) +GCFLAG_REMEMBERED = 2 << GCFLAGSHIFT class GenerationGC(SemiSpaceGC): """A basic generational GC: it's a SemiSpaceGC with an additional @@ -35,6 +37,7 @@ self.old_objects_pointing_to_young = nonnull_endmarker # ^^^ the head of a linked list inside the old objects space self.young_objects_with_weakrefs = self.AddressLinkedList() + self.static_to_young_pointer = self.AddressLinkedList() def reset_nursery(self): self.nursery = llmemory.NULL @@ -90,6 +93,7 @@ def semispace_collect(self, size_changing=False): self.reset_forwarding() # we are doing a full collection anyway + self.reset_static() self.weakrefs_grow_older() self.reset_nursery() SemiSpaceGC.semispace_collect(self, size_changing) @@ -102,6 +106,12 @@ hdr.forw = llmemory.NULL self.old_objects_pointing_to_young = nonnull_endmarker + def reset_static(self): + while self.static_to_young_pointer.non_empty(): + obj = self.static_to_young_pointer.pop() + hdr = self.header(obj) + hdr.tid &= ~GCFLAG_REMEMBERED + def weakrefs_grow_older(self): while self.young_objects_with_weakrefs.non_empty(): obj = self.young_objects_with_weakrefs.pop() @@ -117,6 +127,7 @@ # a nursery-only collection scan = self.free self.collect_oldrefs_to_nursery() + self.collect_static_to_nursery() self.collect_roots_in_nursery() self.scan_objects_just_copied_out_of_nursery(scan) if self.young_objects_with_weakrefs.non_empty(): @@ -149,8 +160,15 @@ hdr.forw = llmemory.NULL self.old_objects_pointing_to_young = nonnull_endmarker + def collect_static_to_nursery(self): + while self.static_to_young_pointer.non_empty(): + obj = self.static_to_young_pointer.pop() + hdr = self.header(obj) + hdr.tid &= ~GCFLAG_REMEMBERED + self.trace_and_drag_out_of_nursery(obj) + def collect_roots_in_nursery(self): - roots = self.get_roots() + roots = self.get_roots(with_static=False) while 1: root = roots.pop() if root == NULL: @@ -217,8 +235,17 @@ def write_barrier(self, addr, addr_to, addr_struct): if not self.is_in_nursery(addr_struct) and self.is_in_nursery(addr): - oldhdr = self.header(addr_struct) - if oldhdr.forw == NULL: - oldhdr.forw = self.old_objects_pointing_to_young - self.old_objects_pointing_to_young = addr_struct + self.remember_young_pointer(addr_struct, addr) addr_to.address[0] = addr + + def remember_young_pointer(self, addr_struct, addr): + oldhdr = self.header(addr_struct) + if oldhdr.forw == NULL: + oldhdr.forw = self.old_objects_pointing_to_young + self.old_objects_pointing_to_young = addr_struct + elif (oldhdr.tid & (GCFLAG_IMMORTAL | GCFLAG_REMEMBERED) == + GCFLAG_IMMORTAL): + self.static_to_young_pointer.append(addr_struct) + oldhdr.tid |= GCFLAG_REMEMBERED + remember_young_pointer.dont_inline = True + Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Sat Oct 20 15:17:18 2007 @@ -9,9 +9,12 @@ from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.memory.gc.base import MovingGCBase - import sys, os +TYPEID_MASK = 0xffff +GCFLAGSHIFT = 16 +GCFLAG_IMMORTAL = 1 << GCFLAGSHIFT + memoryError = MemoryError() class SemiSpaceGC(MovingGCBase): @@ -288,7 +291,7 @@ # immortal objects always have forward to themselves hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) hdr.forw = addr + self.gcheaderbuilder.size_gc_header - hdr.tid = typeid + hdr.tid = typeid | GCFLAG_IMMORTAL def deal_with_objects_with_finalizers(self): # walk over list of objects with finalizers Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Sat Oct 20 15:17:18 2007 @@ -112,8 +112,9 @@ # to make the annotator happy. The fields are patched in finish() # to point to a real array (not 'static_roots', another one). a_random_address = llmemory.cast_ptr_to_adr(gcdata.type_info_table) - gcdata.static_root_start = a_random_address # patched in finish() - gcdata.static_root_end = a_random_address # patched in finish() + gcdata.static_root_start = a_random_address # patched in finish() + gcdata.static_root_nongcstart = a_random_address # patched in finish() + gcdata.static_root_end = a_random_address # patched in finish() self.gcdata = gcdata self.malloc_fnptr_cache = {} @@ -154,6 +155,9 @@ 'static_root_start', annmodel.SomeAddress()) data_classdef.generalize_attr( + 'static_root_nongcstart', + annmodel.SomeAddress()) + data_classdef.generalize_attr( 'static_root_end', annmodel.SomeAddress()) @@ -339,9 +343,12 @@ return top.address[0] pop_root = staticmethod(pop_root) - def __init__(self): + def __init__(self, with_static=True): self.stack_current = gcdata.root_stack_top - self.static_current = gcdata.static_root_start + if with_static: + self.static_current = gcdata.static_root_start + else: + self.static_current = gcdata.static_root_nongcstart def pop(self): while self.static_current != gcdata.static_root_end: @@ -400,7 +407,9 @@ ll_instance.inst_type_info_table = table #self.gcdata.type_info_table = table - addresses_of_static_ptrs = self.layoutbuilder.addresses_of_static_ptrs + addresses_of_static_ptrs = ( + self.layoutbuilder.addresses_of_static_ptrs + + self.layoutbuilder.addresses_of_static_ptrs_in_nongc) log.info("found %s static roots" % (len(addresses_of_static_ptrs), )) ll_static_roots_inside = lltype.malloc(lltype.Array(llmemory.Address), len(addresses_of_static_ptrs), @@ -408,6 +417,7 @@ for i in range(len(addresses_of_static_ptrs)): ll_static_roots_inside[i] = addresses_of_static_ptrs[i] ll_instance.inst_static_root_start = llmemory.cast_ptr_to_adr(ll_static_roots_inside) + llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address)) + ll_instance.inst_static_root_nongcstart = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(self.layoutbuilder.addresses_of_static_ptrs) ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(ll_static_roots_inside) newgcdependencies = [] Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Sat Oct 20 15:17:18 2007 @@ -10,12 +10,13 @@ self.type_info_list = [dummy] # don't use typeid 0, helps debugging self.id_of_type = {} # {LLTYPE: type_id} self.seen_roots = {} - # the following is a list of addresses of gc pointers living inside - # the prebuilt structures (independently on whether the prebuilt - # structures are themselves GcStruct or plain Struct). It should - # list all the locations that could possibly point to a GC heap - # object. + # the following are lists of addresses of gc pointers living inside the + # prebuilt structures. It should list all the locations that could + # possibly point to a GC heap object. + # this lists contains pointers in GcStructs and GcArrays self.addresses_of_static_ptrs = [] + # this lists contains pointers in raw Structs and Arrays + self.addresses_of_static_ptrs_in_nongc = [] self.finalizer_funcptrs = {} def get_type_id(self, TYPE): @@ -27,6 +28,7 @@ # Record the new type_id description as a small dict for now. # It will be turned into a Struct("type_info") in finish() type_id = len(self.type_info_list) + assert type_id & 0xffff == type_id # make sure it fits into 2 bytes info = {} self.type_info_list.append(info) self.id_of_type[TYPE] = type_id @@ -151,8 +153,12 @@ # fields are potential roots: unless the structure is immutable, # they could be changed later to point to GC heap objects. adr = llmemory.cast_ptr_to_adr(value._as_ptr()) + if TYPE._gckind == "gc": + appendto = self.addresses_of_static_ptrs + else: + appendto = self.addresses_of_static_ptrs_in_nongc for a in mutable_gc_pointers_inside(value, adr): - self.addresses_of_static_ptrs.append(a) + appendto.append(a) # ____________________________________________________________ # Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Sat Oct 20 15:17:18 2007 @@ -26,11 +26,16 @@ layoutbuilder.consider_constant(TYPE, obj, self.gc) self.constantroots = layoutbuilder.addresses_of_static_ptrs + self.constantrootsnongc = layoutbuilder.addresses_of_static_ptrs_in_nongc - def get_roots_from_llinterp(self): + def get_roots_from_llinterp(self, with_static=True): sizeofaddr = llmemory.sizeof(llmemory.Address) ll = [llmemory.NULL] # end marker - for addrofaddr in self.constantroots: + if with_static: + for addrofaddr in self.constantroots: + if addrofaddr.address[0]: + ll.append(addrofaddr) + for addrofaddr in self.constantrootsnongc: if addrofaddr.address[0]: ll.append(addrofaddr) for addrofaddr in self.llinterp.find_roots(): From arigo at codespeak.net Sat Oct 20 16:09:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Oct 2007 16:09:51 +0200 (CEST) Subject: [pypy-svn] r47643 - in pypy/dist/pypy/rpython/memory: gc gctransform Message-ID: <20071020140951.20A0B8158@code0.codespeak.net> Author: arigo Date: Sat Oct 20 16:09:49 2007 New Revision: 47643 Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: Two obvious bug fixes. Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Sat Oct 20 16:09:49 2007 @@ -280,7 +280,7 @@ return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) def get_type_id(self, addr): - return self.header(addr).tid + return self.header(addr).tid & TYPEID_MASK def init_gc_object(self, addr, typeid): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Sat Oct 20 16:09:49 2007 @@ -590,7 +590,8 @@ v_newvalue = hop.spaceop.args[-1] # XXX for some GCs the skipping if the newvalue is a constant won't be # ok - if self.write_barrier_ptr is None or isinstance(v_newvalue, Constant): + if (self.write_barrier_ptr is None or isinstance(v_newvalue, Constant) + or v_struct.concretetype.TO._gckind != "gc"): super(FrameworkGCTransformer, self).transform_generic_set(hop) else: self.write_barrier_calls += 1 From arigo at codespeak.net Sat Oct 20 16:14:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Oct 2007 16:14:22 +0200 (CEST) Subject: [pypy-svn] r47644 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071020141422.D7708812F@code0.codespeak.net> Author: arigo Date: Sat Oct 20 16:14:22 2007 New Revision: 47644 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/semispace.py Log: Unify the two lists of objects that contain pointers to young objects. Change the GCFLAG to allow for a version of the write_barrier() whose fast path is (probably) faster. The GCFLAG_NO_YOUNG_PTRS is chosen in preference to a GCFLAG_YOUNG_PTRS, so that young objects start with the flag cleared - i.e. we avoid an instruction to set the flag in malloc. Warning, the correct handling of this flag is slightly delicate. See documented invariants. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Sat Oct 20 16:14:22 2007 @@ -5,10 +5,12 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object, debug_assert -nonnull_endmarker = llmemory.raw_malloc(llmemory.sizeof(lltype.Char)) -llmemory.raw_memclear(nonnull_endmarker, llmemory.sizeof(lltype.Char)) +# The following flag is never set on young objects, i.e. the ones living +# in the nursery. It is initially set on all prebuilt and old objects, +# and gets cleared by the write_barrier() when we write in them a +# pointer to a young object. +GCFLAG_NO_YOUNG_PTRS = 2 << GCFLAGSHIFT -GCFLAG_REMEMBERED = 2 << GCFLAGSHIFT class GenerationGC(SemiSpaceGC): """A basic generational GC: it's a SemiSpaceGC with an additional @@ -18,6 +20,7 @@ """ inline_simple_malloc = True needs_write_barrier = True + default_gcflags = GCFLAG_NO_YOUNG_PTRS # for old and static objects def __init__(self, AddressLinkedList, nursery_size=128, @@ -34,15 +37,20 @@ def setup(self): SemiSpaceGC.setup(self) self.reset_nursery() - self.old_objects_pointing_to_young = nonnull_endmarker - # ^^^ the head of a linked list inside the old objects space + self.old_objects_pointing_to_young = NULL + # ^^^ the head of a linked list inside the old objects space; it + # may contain static prebuilt objects as well. More precisely, + # it lists exactly the old and static objects whose + # GCFLAG_NO_YOUNG_PTRS bit is not set. The 'forw' header field + # of such objects is abused for this linked list; it needs to be + # reset to its correct value when GCFLAG_NO_YOUNG_PTRS is set + # again at the start of a collection. self.young_objects_with_weakrefs = self.AddressLinkedList() - self.static_to_young_pointer = self.AddressLinkedList() def reset_nursery(self): - self.nursery = llmemory.NULL - self.nursery_top = llmemory.NULL - self.nursery_free = llmemory.NULL + self.nursery = NULL + self.nursery_top = NULL + self.nursery_free = NULL def is_in_nursery(self, addr): return self.nursery <= addr < self.nursery_top @@ -62,7 +70,7 @@ if raw_malloc_usage(totalsize) > self.nursery_top - result: result = self.collect_nursery() llarena.arena_reserve(result, totalsize) - self.init_gc_object(result, typeid) + self.init_young_gc_object(result, typeid) self.nursery_free = result + totalsize if contains_weakptr: self.young_objects_with_weakrefs.append(result + size_gc_header) @@ -86,31 +94,31 @@ if raw_malloc_usage(totalsize) > self.nursery_top - result: result = self.collect_nursery() llarena.arena_reserve(result, totalsize) - self.init_gc_object(result, typeid) + self.init_young_gc_object(result, typeid) (result + size_gc_header + offset_to_length).signed[0] = length self.nursery_free = result + llarena.round_up_for_allocation(totalsize) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) + def init_young_gc_object(self, addr, typeid): + hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) + #hdr.forw = NULL -- unneeded, the space is initially filled with zero + hdr.tid = typeid # GCFLAG_NO_YOUNG_PTRS is never set on young objs + def semispace_collect(self, size_changing=False): - self.reset_forwarding() # we are doing a full collection anyway - self.reset_static() + self.reset_young_gcflags() # we are doing a full collection anyway self.weakrefs_grow_older() self.reset_nursery() SemiSpaceGC.semispace_collect(self, size_changing) - def reset_forwarding(self): + def reset_young_gcflags(self): obj = self.old_objects_pointing_to_young - while obj != nonnull_endmarker: - hdr = self.header(obj) - obj = hdr.forw - hdr.forw = llmemory.NULL - self.old_objects_pointing_to_young = nonnull_endmarker - - def reset_static(self): - while self.static_to_young_pointer.non_empty(): - obj = self.static_to_young_pointer.pop() + while obj: hdr = self.header(obj) - hdr.tid &= ~GCFLAG_REMEMBERED + hdr.tid |= GCFLAG_NO_YOUNG_PTRS + nextobj = hdr.forw + self.init_forwarding(obj) + obj = nextobj + self.old_objects_pointing_to_young = NULL def weakrefs_grow_older(self): while self.young_objects_with_weakrefs.non_empty(): @@ -127,9 +135,10 @@ # a nursery-only collection scan = self.free self.collect_oldrefs_to_nursery() - self.collect_static_to_nursery() self.collect_roots_in_nursery() self.scan_objects_just_copied_out_of_nursery(scan) + # at this point, all static and old objects have got their + # GCFLAG_NO_YOUNG_PTRS set again by trace_and_drag_out_of_nursery if self.young_objects_with_weakrefs.non_empty(): self.invalidate_young_weakrefs() self.notify_objects_just_moved() @@ -151,21 +160,14 @@ def collect_oldrefs_to_nursery(self): # Follow the old_objects_pointing_to_young list and move the # young objects they point to out of the nursery. The 'forw' - # fields are reset to NULL along the way. + # fields are reset to their correct value along the way. obj = self.old_objects_pointing_to_young - while obj != nonnull_endmarker: - self.trace_and_drag_out_of_nursery(obj) - hdr = self.header(obj) - obj = hdr.forw - hdr.forw = llmemory.NULL - self.old_objects_pointing_to_young = nonnull_endmarker - - def collect_static_to_nursery(self): - while self.static_to_young_pointer.non_empty(): - obj = self.static_to_young_pointer.pop() - hdr = self.header(obj) - hdr.tid &= ~GCFLAG_REMEMBERED + while obj: + nextobj = self.header(obj).forw + self.init_forwarding(obj) self.trace_and_drag_out_of_nursery(obj) + obj = nextobj + self.old_objects_pointing_to_young = NULL def collect_roots_in_nursery(self): roots = self.get_roots(with_static=False) @@ -189,6 +191,7 @@ """obj must not be in the nursery. This copies all the young objects it references out of the nursery. """ + self.header(obj).tid |= GCFLAG_NO_YOUNG_PTRS typeid = self.get_type_id(obj) offsets = self.offsets_to_gc_pointers(typeid) i = 0 @@ -234,18 +237,16 @@ (obj + offset).address[0] = NULL def write_barrier(self, addr, addr_to, addr_struct): - if not self.is_in_nursery(addr_struct) and self.is_in_nursery(addr): + if self.header(addr_struct).tid & GCFLAG_NO_YOUNG_PTRS: self.remember_young_pointer(addr_struct, addr) addr_to.address[0] = addr def remember_young_pointer(self, addr_struct, addr): - oldhdr = self.header(addr_struct) - if oldhdr.forw == NULL: + debug_assert(not self.is_in_nursery(addr_struct), + "nursery object with GCFLAG_NO_YOUNG_PTRS") + if self.is_in_nursery(addr): + oldhdr = self.header(addr_struct) oldhdr.forw = self.old_objects_pointing_to_young self.old_objects_pointing_to_young = addr_struct - elif (oldhdr.tid & (GCFLAG_IMMORTAL | GCFLAG_REMEMBERED) == - GCFLAG_IMMORTAL): - self.static_to_young_pointer.append(addr_struct) - oldhdr.tid |= GCFLAG_REMEMBERED + oldhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS remember_young_pointer.dont_inline = True - Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Sat Oct 20 16:14:22 2007 @@ -20,6 +20,7 @@ class SemiSpaceGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True + default_gcflags = 0 HDR = lltype.Struct('header', ('forw', llmemory.Address), ('tid', lltype.Signed)) @@ -285,13 +286,21 @@ def init_gc_object(self, addr, typeid): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) #hdr.forw = NULL -- unneeded, the space is initially filled with zero - hdr.tid = typeid + hdr.tid = typeid | self.default_gcflags def init_gc_object_immortal(self, addr, typeid): # immortal objects always have forward to themselves hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.forw = addr + self.gcheaderbuilder.size_gc_header - hdr.tid = typeid | GCFLAG_IMMORTAL + hdr.tid = typeid | self.default_gcflags | GCFLAG_IMMORTAL + self.init_forwarding(addr + self.gcheaderbuilder.size_gc_header) + + def init_forwarding(self, obj): + hdr = self.header(obj) + if hdr.tid & GCFLAG_IMMORTAL: + hdr.forw = obj # prebuilt objects point to themselves, + # so that a collection does not move them + else: + hdr.forw = NULL def deal_with_objects_with_finalizers(self): # walk over list of objects with finalizers From fijal at codespeak.net Sat Oct 20 16:52:08 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 16:52:08 +0200 (CEST) Subject: [pypy-svn] r47645 - in pypy/dist/pypy/module: bz2 thread Message-ID: <20071020145208.4B5448153@code0.codespeak.net> Author: fijal Date: Sat Oct 20 16:52:07 2007 New Revision: 47645 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/thread/ll_thread.py Log: Update API Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Oct 20 16:52:07 2007 @@ -34,8 +34,9 @@ ('bzfree', lltype.Ptr(_free_type)), ('opaque', rffi.VOIDP), ]) - FILE = platform.COpaquePtr('FILE') - BZFILE = platform.COpaquePtr('BZFILE') + +FILE = rffi.COpaquePtr('FILE') +BZFILE = rffi.COpaquePtr('BZFILE') constants = {} @@ -72,8 +73,6 @@ BZ_MEM_ERROR = cConfig.BZ_MEM_ERROR BZ_UNEXPECTED_EOF = cConfig.BZ_UNEXPECTED_EOF BZ_SEQUENCE_ERROR = cConfig.BZ_SEQUENCE_ERROR -BZFILE = cConfig.BZFILE -FILE = cConfig.FILE if BUFSIZ < 8192: SMALLCHUNK = 8192 Modified: pypy/dist/pypy/module/thread/ll_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/ll_thread.py (original) +++ pypy/dist/pypy/module/thread/ll_thread.py Sat Oct 20 16:52:07 2007 @@ -37,9 +37,9 @@ c_thread_start = llexternal('RPyThreadStart', [CALLBACK, rffi.VOIDP], rffi.INT) c_thread_get_ident = llexternal('RPyThreadGetIdent', [], rffi.INT) -TLOCKP = platform.copaque('struct RPyOpaque_ThreadLock', '', - _includes_=includes, - _include_dirs_=include_dirs) +TLOCKP = rffi.COpaquePtr('struct RPyOpaque_ThreadLock', + includes=includes, + include_dirs=include_dirs) c_thread_lock_init = llexternal('RPyThreadLockInit', [TLOCKP], lltype.Void) c_thread_acquirelock = llexternal('RPyThreadAcquireLock', [TLOCKP, rffi.INT], From fijal at codespeak.net Sat Oct 20 17:18:25 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 17:18:25 +0200 (CEST) Subject: [pypy-svn] r47646 - pypy/dist/pypy/translator/c/src Message-ID: <20071020151825.91D6B814C@code0.codespeak.net> Author: fijal Date: Sat Oct 20 17:18:24 2007 New Revision: 47646 Modified: pypy/dist/pypy/translator/c/src/g_prerequisite.h Log: Mute gcc warnings Modified: pypy/dist/pypy/translator/c/src/g_prerequisite.h ============================================================================== --- pypy/dist/pypy/translator/c/src/g_prerequisite.h (original) +++ pypy/dist/pypy/translator/c/src/g_prerequisite.h Sat Oct 20 17:18:24 2007 @@ -2,8 +2,10 @@ /**************************************************************/ /*** this is included before any code produced by genc.py ***/ +#ifndef #define _GNU_SOURCE /* this must be defined before including other headers in order to get a few extra functions like mremap() */ +#endif /* XXX for now we always include Python.h even to produce stand-alone * executables (which are *not* linked against CPython then), From fijal at codespeak.net Sat Oct 20 17:19:05 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 17:19:05 +0200 (CEST) Subject: [pypy-svn] r47647 - pypy/dist/pypy/translator/c/src Message-ID: <20071020151905.DDC06814C@code0.codespeak.net> Author: fijal Date: Sat Oct 20 17:19:05 2007 New Revision: 47647 Modified: pypy/dist/pypy/translator/c/src/g_prerequisite.h Log: Premature commit :-/ Modified: pypy/dist/pypy/translator/c/src/g_prerequisite.h ============================================================================== --- pypy/dist/pypy/translator/c/src/g_prerequisite.h (original) +++ pypy/dist/pypy/translator/c/src/g_prerequisite.h Sat Oct 20 17:19:05 2007 @@ -2,7 +2,7 @@ /**************************************************************/ /*** this is included before any code produced by genc.py ***/ -#ifndef +#ifndef _GNU_SOURCE #define _GNU_SOURCE /* this must be defined before including other headers in order to get a few extra functions like mremap() */ #endif From cfbolz at codespeak.net Sat Oct 20 19:30:19 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 20 Oct 2007 19:30:19 +0200 (CEST) Subject: [pypy-svn] r47648 - in pypy/dist/pypy/rpython/memory: . test Message-ID: <20071020173019.76E248107@code0.codespeak.net> Author: cfbolz Date: Sat Oct 20 19:30:18 2007 New Revision: 47648 Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py pypy/dist/pypy/rpython/memory/test/test_gc.py Log: don't invoke the write barrier for nongc (outer) pointers in the gcwrapper either. Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Sat Oct 20 19:30:18 2007 @@ -72,7 +72,8 @@ self.setinterior(array, addr, ARRAY.OF, newitem) def setinterior(self, toplevelcontainer, inneraddr, INNERTYPE, newvalue): - if isinstance(INNERTYPE, lltype.Ptr) and INNERTYPE.TO._gckind == 'gc': + if (lltype.typeOf(toplevelcontainer).TO._gckind == 'gc' and + isinstance(INNERTYPE, lltype.Ptr) and INNERTYPE.TO._gckind == 'gc'): self.gc.write_barrier(llmemory.cast_ptr_to_adr(newvalue), inneraddr, llmemory.cast_ptr_to_adr(toplevelcontainer)) Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Sat Oct 20 19:30:18 2007 @@ -327,6 +327,21 @@ res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection assert res == 20 + 20 + def test_nongc_static_root(self): + from pypy.rpython.lltypesystem import lltype + T1 = lltype.GcStruct("C", ('x', lltype.Signed)) + T2 = lltype.Struct("C", ('p', lltype.Ptr(T1))) + static = lltype.malloc(T2, immortal=True) + def f(): + t1 = lltype.malloc(T1) + t1.x = 42 + static.p = t1 + llop.gc__collect(lltype.Void) + return static.p.x + res = self.interpret(f, []) + assert res == 42 + + class TestMarkSweepGC(GCTest): from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass From cfbolz at codespeak.net Sat Oct 20 19:33:49 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 20 Oct 2007 19:33:49 +0200 (CEST) Subject: [pypy-svn] r47649 - pypy/dist/pypy/rpython/memory/test Message-ID: <20071020173349.488DC8093@code0.codespeak.net> Author: cfbolz Date: Sat Oct 20 19:33:48 2007 New Revision: 47649 Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: same test (passing) for transformed GCs Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Sat Oct 20 19:33:48 2007 @@ -178,6 +178,21 @@ heap_size = self.heap_usage(statistics) assert heap_size < 16000 * INT_SIZE / 4 # xxx + def test_nongc_static_root(self): + from pypy.rpython.lltypesystem import lltype + T1 = lltype.GcStruct("C", ('x', lltype.Signed)) + T2 = lltype.Struct("C", ('p', lltype.Ptr(T1))) + static = lltype.malloc(T2, immortal=True) + def f(): + t1 = lltype.malloc(T1) + t1.x = 42 + static.p = t1 + llop.gc__collect(lltype.Void) + return static.p.x + run = self.runner(f, nbargs=0) + res = run([]) + assert res == 42 + def test_finalizer(self): class B(object): pass From cfbolz at codespeak.net Sat Oct 20 19:34:15 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 20 Oct 2007 19:34:15 +0200 (CEST) Subject: [pypy-svn] r47650 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071020173415.E9332810E@code0.codespeak.net> Author: cfbolz Date: Sat Oct 20 19:34:15 2007 New Revision: 47650 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: only log number of write barrier calls when the GC uses a write barrier Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Sat Oct 20 19:34:15 2007 @@ -387,8 +387,9 @@ log.info("assigned %s typeids" % (len(table), )) log.info("added %s push/pop stack root instructions" % ( self.num_pushs, )) - log.info("inserted %s write barrier calls" % ( - self.write_barrier_calls, )) + if self.write_barrier_ptr: + log.info("inserted %s write barrier calls" % ( + self.write_barrier_calls, )) # replace the type_info_table pointer in gcdata -- at this point, # the database is in principle complete, so it has already seen From fijal at codespeak.net Sat Oct 20 19:44:33 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Oct 2007 19:44:33 +0200 (CEST) Subject: [pypy-svn] r47651 - in pypy/dist/pypy: rlib rpython/lltypesystem Message-ID: <20071020174433.E330C8143@code0.codespeak.net> Author: fijal Date: Sat Oct 20 19:44:33 2007 New Revision: 47651 Modified: pypy/dist/pypy/rlib/libffi.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: Mess part 321443223... Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Sat Oct 20 19:44:33 2007 @@ -111,9 +111,9 @@ FFI_OK = cConfig.FFI_OK FFI_BAD_TYPEDEF = cConfig.FFI_BAD_TYPEDEF FFI_DEFAULT_ABI = rffi.cast(rffi.USHORT, cConfig.FFI_DEFAULT_ABI) -FFI_CIFP = rffi.COpaquePtr('ffi_cif', _includes_=includes, - _include_dirs_=include_dirs, - _library_dirs_=lib_dirs) +FFI_CIFP = rffi.COpaquePtr('ffi_cif', includes=includes, + include_dirs=include_dirs, + library_dirs=lib_dirs) VOIDPP = rffi.CArrayPtr(rffi.VOIDP) Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Sat Oct 20 19:44:33 2007 @@ -233,7 +233,8 @@ from pypy.rpython.tool import rffi_platform k = {} for _name, value in kwds.items(): - if _name in ['includes', 'include_dirs', 'libraries']: + if _name in ['includes', 'include_dirs', 'libraries', + 'library_dirs']: k['_%s_' % _name] = value return rffi_platform.sizeof(name, '', **k) From cfbolz at codespeak.net Sat Oct 20 19:53:45 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 20 Oct 2007 19:53:45 +0200 (CEST) Subject: [pypy-svn] r47652 - pypy/dist/pypy/rpython/memory/test Message-ID: <20071020175345.C81D08145@code0.codespeak.net> Author: cfbolz Date: Sat Oct 20 19:53:45 2007 New Revision: 47652 Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: two more passing tests for the generational GC :-(. Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Sat Oct 20 19:53:45 2007 @@ -773,3 +773,51 @@ run = self.runner(f) res = run([]) assert res == 20 + 20 + + def test_nongc_static_root_minor_collect(self): + from pypy.rpython.lltypesystem import lltype + T1 = lltype.GcStruct("C", ('x', lltype.Signed)) + T2 = lltype.Struct("C", ('p', lltype.Ptr(T1))) + static = lltype.malloc(T2, immortal=True) + def f(): + t1 = lltype.malloc(T1) + t1.x = 42 + static.p = t1 + x = 20 + all = [None] * x + i = 0 + while i < x: # enough to cause a minor collect + all[i] = [i] * i + i += 1 + i = static.p.x + llop.gc__collect(lltype.Void) + return static.p.x + i + run = self.runner(f, nbargs=0) + res = run([]) + assert res == 84 + + + def test_static_root_minor_collect(self): + from pypy.rpython.lltypesystem import lltype + class A: + pass + class B: + pass + static = A() + static.p = None + def f(): + t1 = B() + t1.x = 42 + static.p = t1 + x = 20 + all = [None] * x + i = 0 + while i < x: # enough to cause a minor collect + all[i] = [i] * i + i += 1 + i = static.p.x + llop.gc__collect(lltype.Void) + return static.p.x + i + run = self.runner(f, nbargs=0) + res = run([]) + assert res == 84 From cfbolz at codespeak.net Sat Oct 20 20:06:05 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 20 Oct 2007 20:06:05 +0200 (CEST) Subject: [pypy-svn] r47653 - pypy/dist/pypy/translator/c/test Message-ID: <20071020180605.A7B798146@code0.codespeak.net> Author: cfbolz Date: Sat Oct 20 20:06:04 2007 New Revision: 47653 Modified: pypy/dist/pypy/translator/c/test/test_newgc.py Log: more passing tests. one of the rare times where you don't want to see an F. Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Sat Oct 20 20:06:04 2007 @@ -858,3 +858,8 @@ fn = self.getcompiled(f) res = fn() assert res == -2 + + +class TestGenerationalGC(TestUsingFramework): + gcpolicy = "generation" + should_be_moving = True From cfbolz at codespeak.net Sat Oct 20 20:07:48 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 20 Oct 2007 20:07:48 +0200 (CEST) Subject: [pypy-svn] r47654 - pypy/dist/pypy/translator/c/test Message-ID: <20071020180748.6F24D8143@code0.codespeak.net> Author: cfbolz Date: Sat Oct 20 20:07:46 2007 New Revision: 47654 Modified: pypy/dist/pypy/translator/c/test/test_newgc.py Log: hah, this one fails Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Sat Oct 20 20:07:46 2007 @@ -860,6 +860,8 @@ assert res == -2 -class TestGenerationalGC(TestUsingFramework): +class TestGenerationalGC(TestSemiSpaceGC): gcpolicy = "generation" should_be_moving = True + def test_many_ids(self): + py.test.skip("fails!") From cfbolz at codespeak.net Sat Oct 20 21:24:11 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 20 Oct 2007 21:24:11 +0200 (CEST) Subject: [pypy-svn] r47655 - pypy/dist/pypy/interpreter Message-ID: <20071020192411.5B6C98143@code0.codespeak.net> Author: cfbolz Date: Sat Oct 20 21:24:09 2007 New Revision: 47655 Modified: pypy/dist/pypy/interpreter/typedef.py Log: fix typo Modified: pypy/dist/pypy/interpreter/typedef.py ============================================================================== --- pypy/dist/pypy/interpreter/typedef.py (original) +++ pypy/dist/pypy/interpreter/typedef.py Sat Oct 20 21:24:09 2007 @@ -50,7 +50,7 @@ # Hash support def get_default_hash_function(cls): - # go to the first parent class of 'cls' that as a typedef + # go to the first parent class of 'cls' that has a typedef while 'typedef' not in cls.__dict__: cls = cls.__bases__[0] if cls is object: From arigo at codespeak.net Sat Oct 20 23:14:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Oct 2007 23:14:02 +0200 (CEST) Subject: [pypy-svn] r47656 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071020211402.2DA128149@code0.codespeak.net> Author: arigo Date: Sat Oct 20 23:14:01 2007 New Revision: 47656 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py Log: Puh. This bug took some time to find. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Sat Oct 20 23:14:01 2007 @@ -110,6 +110,17 @@ self.reset_nursery() SemiSpaceGC.semispace_collect(self, size_changing) + def trace_and_copy(self, obj): + # during a full collect, all objects copied might come from the nursery and + # so must have this flag set: + self.header(obj).tid |= GCFLAG_NO_YOUNG_PTRS + SemiSpaceGC.trace_and_copy(self, obj) + # history: this was missing and caused an object to become old but without the + # flag set. Such an object is bogus in the sense that the write_barrier doesn't + # work on it. So it can eventually contain a ptr to a young object but we didn't + # know about it. That ptr was not updated in the next minor collect... boom at + # the next usage. + def reset_young_gcflags(self): obj = self.old_objects_pointing_to_young while obj: From cfbolz at codespeak.net Sat Oct 20 23:21:14 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 20 Oct 2007 23:21:14 +0200 (CEST) Subject: [pypy-svn] r47657 - pypy/dist/pypy/translator/goal Message-ID: <20071020212114.754778149@code0.codespeak.net> Author: cfbolz Date: Sat Oct 20 23:21:14 2007 New Revision: 47657 Added: pypy/dist/pypy/translator/goal/targetcrashstandalone.py (contents, props changed) Log: somewhat rough target that shows a bug in semispace and generation GCs: if this is called with big arguments (>300000), some weakrefs are not properly updated. Added: pypy/dist/pypy/translator/goal/targetcrashstandalone.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/goal/targetcrashstandalone.py Sat Oct 20 23:21:14 2007 @@ -0,0 +1,59 @@ +""" +A simple standalone target. + +The target below specifies None as the argument types list. +This is a case treated specially in driver.py . If the list +of input types is empty, it is meant to be a list of strings, +actually implementing argv of the executable. +""" + +import os, sys, weakref, gc +import pypy.rlib.rgc +from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython.lltypesystem import lltype + +def debug(msg): + os.write(2, "debug: " + msg + '\n') + +# __________ Entry point __________ + +class A: + def __init__(self, i): + self.i = i +def f(x): + alist = [A(i) for i in range(x)] + refarray = [None] * len(alist) + # Compute the id of all elements of the list. The goal is + # to not allocate memory, so that if the GC needs memory to + # remember the ids, it will trigger some collections itself + i = 0 + while i < len(alist): + refarray[i] = weakref.ref(alist[i]) + i += 1 + j = 0 + gc.collect() + while j < 100: + i = 0 + while i < len(alist): + if refarray[i]() is not alist[i]: + print "mismatch", j, i + print refarray[i]().i + llop.debug_print(lltype.Void, refarray[i], refarray[i](), type(refarray[i]())) + print alist[i].i + llop.debug_print(lltype.Void, alist[i], type(alist[i])) + return + i += 1 + j += 1 + print "match" + return + + +def entry_point(argv): + debug("hello world") + f(int(argv[1])) + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None From arigo at codespeak.net Sat Oct 20 23:22:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Oct 2007 23:22:29 +0200 (CEST) Subject: [pypy-svn] r47658 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071020212229.9BF648149@code0.codespeak.net> Author: arigo Date: Sat Oct 20 23:22:29 2007 New Revision: 47658 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/semispace.py Log: Replace the default_gcflags class attribute with method overriding that changes the default value of an argument. I think none of the two approaches is much stranger than the other, and this version might allow better constant-propagation. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Sat Oct 20 23:22:29 2007 @@ -20,7 +20,6 @@ """ inline_simple_malloc = True needs_write_barrier = True - default_gcflags = GCFLAG_NO_YOUNG_PTRS # for old and static objects def __init__(self, AddressLinkedList, nursery_size=128, @@ -70,7 +69,8 @@ if raw_malloc_usage(totalsize) > self.nursery_top - result: result = self.collect_nursery() llarena.arena_reserve(result, totalsize) - self.init_young_gc_object(result, typeid) + # GCFLAG_NO_YOUNG_PTRS is never set on young objs + self.init_gc_object(result, typeid, flags=0) self.nursery_free = result + totalsize if contains_weakptr: self.young_objects_with_weakrefs.append(result + size_gc_header) @@ -94,15 +94,20 @@ if raw_malloc_usage(totalsize) > self.nursery_top - result: result = self.collect_nursery() llarena.arena_reserve(result, totalsize) - self.init_young_gc_object(result, typeid) + # GCFLAG_NO_YOUNG_PTRS is never set on young objs + self.init_gc_object(result, typeid, flags=0) (result + size_gc_header + offset_to_length).signed[0] = length self.nursery_free = result + llarena.round_up_for_allocation(totalsize) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) - def init_young_gc_object(self, addr, typeid): - hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - #hdr.forw = NULL -- unneeded, the space is initially filled with zero - hdr.tid = typeid # GCFLAG_NO_YOUNG_PTRS is never set on young objs + # override the init_gc_object methods to change the default value of 'flags', + # used by objects that are directly created outside the nursery by the SemiSpaceGC. + # These objects must have the GCFLAG_NO_YOUNG_PTRS flag set immediately. + def init_gc_object(self, addr, typeid, flags=GCFLAG_NO_YOUNG_PTRS): + SemiSpaceGC.init_gc_object(self, addr, typeid, flags) + + def init_gc_object_immortal(self, addr, typeid, flags=GCFLAG_NO_YOUNG_PTRS): + SemiSpaceGC.init_gc_object_immortal(self, addr, typeid, flags) def semispace_collect(self, size_changing=False): self.reset_young_gcflags() # we are doing a full collection anyway Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Sat Oct 20 23:22:29 2007 @@ -20,7 +20,6 @@ class SemiSpaceGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True - default_gcflags = 0 HDR = lltype.Struct('header', ('forw', llmemory.Address), ('tid', lltype.Signed)) @@ -215,9 +214,8 @@ # Objects not living the GC heap have all been initialized by # setting their 'forw' address so that it points to themselves. # The logic below will thus simply return 'obj' if 'obj' is prebuilt. -## print "copying regularly", obj, if self.is_forwarded(obj): -## print "already copied to", self.get_forwarding_address(obj) + #llop.debug_print(lltype.Void, obj, "already copied to", self.get_forwarding_address(obj)) return self.get_forwarding_address(obj) else: newaddr = self.free @@ -226,7 +224,9 @@ raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize) self.free += totalsize newobj = newaddr + self.size_gc_header() -## print "to", newobj + #llop.debug_print(lltype.Void, obj, "copied to", newobj, + # "tid", self.header(obj).tid, + # "size", totalsize) self.set_forwarding_address(obj, newobj) return newobj @@ -283,15 +283,15 @@ def get_type_id(self, addr): return self.header(addr).tid & TYPEID_MASK - def init_gc_object(self, addr, typeid): + def init_gc_object(self, addr, typeid, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) #hdr.forw = NULL -- unneeded, the space is initially filled with zero - hdr.tid = typeid | self.default_gcflags + hdr.tid = typeid | flags - def init_gc_object_immortal(self, addr, typeid): + def init_gc_object_immortal(self, addr, typeid, flags=0): # immortal objects always have forward to themselves hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.tid = typeid | self.default_gcflags | GCFLAG_IMMORTAL + hdr.tid = typeid | flags | GCFLAG_IMMORTAL self.init_forwarding(addr + self.gcheaderbuilder.size_gc_header) def init_forwarding(self, obj): From elmom at codespeak.net Sun Oct 21 00:11:34 2007 From: elmom at codespeak.net (elmom at codespeak.net) Date: Sun, 21 Oct 2007 00:11:34 +0200 (CEST) Subject: [pypy-svn] r47661 - pypy/dist/pypy/config Message-ID: <20071020221134.DDA918142@code0.codespeak.net> Author: elmom Date: Sun Oct 21 00:11:33 2007 New Revision: 47661 Modified: pypy/dist/pypy/config/pypyoption.py Log: Forgot this one, should fix the failed tests in test_listmultiobject Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Sun Oct 21 00:11:33 2007 @@ -229,6 +229,10 @@ "only overallocate O(sqrt(n)) elements for lists", default=False, requires=[("objspace.std.withmultilist", True)]), + BoolOption("withblist", + "good asymptotic performance for very large lists", + default=False, + requires=[("objspace.std.withmultilist", True)]), BoolOption("optimized_int_add", "special case the addition of two integers in BINARY_ADD", default=False), From elmom at codespeak.net Sun Oct 21 00:35:05 2007 From: elmom at codespeak.net (elmom at codespeak.net) Date: Sun, 21 Oct 2007 00:35:05 +0200 (CEST) Subject: [pypy-svn] r47662 - pypy/dist/pypy/doc/config Message-ID: <20071020223505.CBE3E80F8@code0.codespeak.net> Author: elmom Date: Sun Oct 21 00:35:04 2007 New Revision: 47662 Added: pypy/dist/pypy/doc/config/objspace.std.withblist.txt (contents, props changed) Log: Add the doc for the withblist option Added: pypy/dist/pypy/doc/config/objspace.std.withblist.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/config/objspace.std.withblist.txt Sun Oct 21 00:35:04 2007 @@ -0,0 +1,2 @@ +Use a list implementation that has good asymptotic performance for very +large lists. From elmom at codespeak.net Sun Oct 21 00:45:30 2007 From: elmom at codespeak.net (elmom at codespeak.net) Date: Sun, 21 Oct 2007 00:45:30 +0200 (CEST) Subject: [pypy-svn] r47663 - pypy/dist/pypy/doc/config Message-ID: <20071020224530.309418145@code0.codespeak.net> Author: elmom Date: Sun Oct 21 00:45:29 2007 New Revision: 47663 Modified: pypy/dist/pypy/doc/config/objspace.std.withblist.txt Log: Note that the option isn't yet useful Modified: pypy/dist/pypy/doc/config/objspace.std.withblist.txt ============================================================================== --- pypy/dist/pypy/doc/config/objspace.std.withblist.txt (original) +++ pypy/dist/pypy/doc/config/objspace.std.withblist.txt Sun Oct 21 00:45:29 2007 @@ -1,2 +1,2 @@ Use a list implementation that has good asymptotic performance for very -large lists. +large lists (not yet working). From arigo at codespeak.net Sun Oct 21 11:17:58 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 21 Oct 2007 11:17:58 +0200 (CEST) Subject: [pypy-svn] r47666 - in pypy/dist/pypy/rpython/memory: . gc gctransform Message-ID: <20071021091758.352358162@code0.codespeak.net> Author: arigo Date: Sun Oct 21 11:17:56 2007 New Revision: 47666 Modified: pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gcwrapper.py Log: For now, reduce the write_barrier() interface. I don't know if I'm making some GCs impossible to write, but at least it fixes a problematic crash -- and I could kill a page of obscure code, probably not too bad :-) The issue is that if the write barrier has to write the new field value itself via the address, we get strange C code manipulating "void* + offset". The C compiler's strict aliasing rules don't realize that it's a write to a field of a structure and may reorder a subsequent read before the write... see gcc option -fno-strict-aliasing. With this change, the write barrier is no longer responsible for actually writing the field - we still emit a bare_setfield which the C compiler understands correctly. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Sun Oct 21 11:17:56 2007 @@ -23,8 +23,8 @@ self.varsize_offsets_to_gcpointers_in_var_part = varsize_offsets_to_gcpointers_in_var_part self.weakpointer_offset = weakpointer_offset - def write_barrier(self, addr, addr_to, addr_struct): - addr_to.address[0] = addr + def write_barrier(self, oldvalue, newvalue, addr_struct): + pass def setup(self): pass Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Sun Oct 21 11:17:56 2007 @@ -252,10 +252,9 @@ else: (obj + offset).address[0] = NULL - def write_barrier(self, addr, addr_to, addr_struct): + def write_barrier(self, oldvalue, newvalue, addr_struct): if self.header(addr_struct).tid & GCFLAG_NO_YOUNG_PTRS: - self.remember_young_pointer(addr_struct, addr) - addr_to.address[0] = addr + self.remember_young_pointer(addr_struct, newvalue) def remember_young_pointer(self, addr_struct, addr): debug_assert(not self.is_in_nursery(addr_struct), Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Sun Oct 21 11:17:56 2007 @@ -587,58 +587,32 @@ def transform_generic_set(self, hop): from pypy.objspace.flow.model import Constant + opname = hop.spaceop.opname v_struct = hop.spaceop.args[0] v_newvalue = hop.spaceop.args[-1] + assert opname in ('setfield', 'setarrayitem', 'setinteriorfield') + assert isinstance(v_newvalue.concretetype, lltype.Ptr) # XXX for some GCs the skipping if the newvalue is a constant won't be # ok - if (self.write_barrier_ptr is None or isinstance(v_newvalue, Constant) - or v_struct.concretetype.TO._gckind != "gc"): - super(FrameworkGCTransformer, self).transform_generic_set(hop) - else: + if (self.write_barrier_ptr is not None + and not isinstance(v_newvalue, Constant) + and v_struct.concretetype.TO._gckind == "gc"): self.write_barrier_calls += 1 - assert isinstance(v_newvalue.concretetype, lltype.Ptr) - opname = hop.spaceop.opname - assert opname in ('setfield', 'setarrayitem', 'setinteriorfield') - offsets = hop.spaceop.args[1:-1] - CURTYPE = v_struct.concretetype.TO - v_currentofs = None - for ofs in offsets: - if ofs.concretetype is lltype.Void: - # a field in a structure - fieldname = ofs.value - fieldofs = llmemory.offsetof(CURTYPE, fieldname) - v_offset = rmodel.inputconst(lltype.Signed, fieldofs) - CURTYPE = getattr(CURTYPE, fieldname) - else: - # an index in an array - assert ofs.concretetype is lltype.Signed - firstitem = llmemory.ArrayItemsOffset(CURTYPE) - itemsize = llmemory.sizeof(CURTYPE.OF) - c_firstitem = rmodel.inputconst(lltype.Signed, firstitem) - c_itemsize = rmodel.inputconst(lltype.Signed, itemsize) - v_index = hop.spaceop.args[1] - v_offset = hop.genop("int_mul", [c_itemsize, v_index], - resulttype = lltype.Signed) - v_offset = hop.genop("int_add", [c_firstitem, v_offset], - resulttype = lltype.Signed) - CURTYPE = CURTYPE.OF - if v_currentofs is None: - v_currentofs = v_offset - else: - v_currentofs = hop.genop("int_add", - [v_currentofs, v_offset], - resulttype = lltype.Signed) + v_oldvalue = hop.genop('g' + opname[1:], + hop.inputargs()[:-1], + resulttype=v_newvalue.concretetype) + v_oldvalue = hop.genop("cast_ptr_to_adr", [v_oldvalue], + resulttype = llmemory.Address) v_newvalue = hop.genop("cast_ptr_to_adr", [v_newvalue], resulttype = llmemory.Address) v_structaddr = hop.genop("cast_ptr_to_adr", [v_struct], resulttype = llmemory.Address) - v_fieldaddr = hop.genop("adr_add", [v_structaddr, v_currentofs], - resulttype = llmemory.Address) hop.genop("direct_call", [self.write_barrier_ptr, self.c_const_gc, + v_oldvalue, v_newvalue, - v_fieldaddr, v_structaddr]) + hop.rename('bare_' + opname) def var_needs_set_transform(self, var): return var_needsgc(var) Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Sun Oct 21 11:17:56 2007 @@ -74,12 +74,11 @@ def setinterior(self, toplevelcontainer, inneraddr, INNERTYPE, newvalue): if (lltype.typeOf(toplevelcontainer).TO._gckind == 'gc' and isinstance(INNERTYPE, lltype.Ptr) and INNERTYPE.TO._gckind == 'gc'): - self.gc.write_barrier(llmemory.cast_ptr_to_adr(newvalue), - inneraddr, + oldvalue = inneraddr.address[0] + self.gc.write_barrier(oldvalue, + llmemory.cast_ptr_to_adr(newvalue), llmemory.cast_ptr_to_adr(toplevelcontainer)) - else: - llheap.setinterior(toplevelcontainer, inneraddr, - INNERTYPE, newvalue) + llheap.setinterior(toplevelcontainer, inneraddr, INNERTYPE, newvalue) def collect(self): self.gc.collect() From arigo at codespeak.net Sun Oct 21 13:22:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 21 Oct 2007 13:22:17 +0200 (CEST) Subject: [pypy-svn] r47668 - pypy/dist/pypy/translator/c Message-ID: <20071021112217.14CB28149@code0.codespeak.net> Author: arigo Date: Sun Oct 21 13:22:16 2007 New Revision: 47668 Modified: pypy/dist/pypy/translator/c/genc.py Log: Add a Makefile target: "make lldebug" which enables both RPython-level assertion and low-level NULL-pointer and array-out-of-bounds detection. Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Sun Oct 21 13:22:16 2007 @@ -912,6 +912,9 @@ llsafer: \t$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" +lldebug: +\t$(MAKE) CFLAGS="-g -DRPY_ASSERT -DRPY_LL_ASSERT" + profile: \t$(MAKE) CFLAGS="-g -pg $(CFLAGS)" LDFLAGS="-pg $(LDFLAGS)" From antocuni at codespeak.net Sun Oct 21 17:13:13 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 21 Oct 2007 17:13:13 +0200 (CEST) Subject: [pypy-svn] r47671 - pypy/extradoc/talk/dls2007 Message-ID: <20071021151313.F04668175@code0.codespeak.net> Author: antocuni Date: Sun Oct 21 17:13:11 2007 New Revision: 47671 Modified: pypy/extradoc/talk/dls2007/rpython-talk.latex pypy/extradoc/talk/dls2007/rpython-talk.pdf Log: typo Modified: pypy/extradoc/talk/dls2007/rpython-talk.latex ============================================================================== --- pypy/extradoc/talk/dls2007/rpython-talk.latex (original) +++ pypy/extradoc/talk/dls2007/rpython-talk.latex Sun Oct 21 17:13:11 2007 @@ -331,7 +331,7 @@ \begin{frame} \frametitle{About PyPy (1)} - \begin{alertblock}{Pyton in (R)Python} + \begin{alertblock}{Python in (R)Python} \begin{itemize} \item High level interpreter written in RPython \item Easy to understand Modified: pypy/extradoc/talk/dls2007/rpython-talk.pdf ============================================================================== Binary files. No diff available. From arigo at codespeak.net Sun Oct 21 17:44:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 21 Oct 2007 17:44:23 +0200 (CEST) Subject: [pypy-svn] r47672 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071021154423.94B5A8177@code0.codespeak.net> Author: arigo Date: Sun Oct 21 17:44:23 2007 New Revision: 47672 Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py Log: This operation cannot be arbitrarily moved around in the presence of a moving GC. Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Sun Oct 21 17:44:23 2007 @@ -369,7 +369,7 @@ 'adr_gt': LLOp(canfold=True), 'adr_ge': LLOp(canfold=True), 'adr_call': LLOp(canraise=(Exception,)), - 'cast_ptr_to_adr': LLOp(canfold=True), + 'cast_ptr_to_adr': LLOp(sideeffects=False), 'cast_adr_to_ptr': LLOp(canfold=True), 'cast_adr_to_int': LLOp(canfold=True), 'cast_int_to_adr': LLOp(canfold=True), # not implemented in llinterp From arigo at codespeak.net Sun Oct 21 17:52:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 21 Oct 2007 17:52:49 +0200 (CEST) Subject: [pypy-svn] r47673 - in pypy/dist/pypy/rpython: . lltypesystem memory memory/gc memory/test Message-ID: <20071021155249.2DE7E817B@code0.codespeak.net> Author: arigo Date: Sun Oct 21 17:52:48 2007 New Revision: 47673 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/llheap.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/gcwrapper.py pypy/dist/pypy/rpython/memory/test/test_gc.py Log: in gcwrapper, we have to be lazy in reading the llinterp variable containing the 'obj' pointer, because the gc.malloc() call could move it around. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Sun Oct 21 17:52:48 2007 @@ -721,8 +721,11 @@ return lltype.cast_opaque_ptr(RESTYPE, obj) op_cast_opaque_ptr.need_result_type = True - def op_weakref_create(self, obj): - return self.heap.weakref_create(obj) + def op_weakref_create(self, v_obj): + def objgetter(): # special support for gcwrapper.py + return self.getval(v_obj) + return self.heap.weakref_create_getlazy(objgetter) + op_weakref_create.specialform = True def op_weakref_deref(self, PTRTYPE, obj): return self.heap.weakref_deref(PTRTYPE, obj) Modified: pypy/dist/pypy/rpython/lltypesystem/llheap.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llheap.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llheap.py Sun Oct 21 17:52:48 2007 @@ -16,3 +16,6 @@ inneraddr.ref()[0] = newvalue from pypy.rpython.lltypesystem.lltype import cast_ptr_to_int as gc_id + +def weakref_create_getlazy(objgetter): + return weakref_create(objgetter()) Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Sun Oct 21 17:52:48 2007 @@ -328,6 +328,7 @@ obj = self.get_forwarding_address(obj) offset = self.weakpointer_offset(self.get_type_id(obj)) pointing_to = (obj + offset).address[0] + # XXX I think that pointing_to cannot be NULL here if pointing_to: if self.is_forwarded(pointing_to): (obj + offset).address[0] = self.get_forwarding_address( Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Sun Oct 21 17:52:48 2007 @@ -83,11 +83,14 @@ def collect(self): self.gc.collect() - def weakref_create(self, obj): + def weakref_create_getlazy(self, objgetter): + # we have to be lazy in reading the llinterp variable containing + # the 'obj' pointer, because the gc.malloc() call below could + # move it around type_id = self.get_type_id(gctypelayout.WEAKREF) addr = self.gc.malloc(type_id, None, zero=False) result = llmemory.cast_adr_to_ptr(addr, gctypelayout.WEAKREFPTR) - result.weakptr = llmemory.cast_ptr_to_adr(obj) + result.weakptr = llmemory.cast_ptr_to_adr(objgetter()) return llmemory.cast_ptr_to_weakrefptr(result) def weakref_deref(self, PTRTYPE, obj): Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Sun Oct 21 17:52:48 2007 @@ -327,6 +327,21 @@ res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection assert res == 20 + 20 + def test_many_weakrefs(self): + # test for the case where allocating the weakref itself triggers + # a collection + import weakref + class A: + pass + def f(x): + a = A() + i = 0 + while i < x: + ref = weakref.ref(a) + assert ref() is a + i += 1 + self.interpret(f, [1100]) + def test_nongc_static_root(self): from pypy.rpython.lltypesystem import lltype T1 = lltype.GcStruct("C", ('x', lltype.Signed)) From arigo at codespeak.net Sun Oct 21 17:55:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 21 Oct 2007 17:55:44 +0200 (CEST) Subject: [pypy-svn] r47674 - in pypy/dist/pypy/rpython: . lltypesystem Message-ID: <20071021155544.8927B817B@code0.codespeak.net> Author: arigo Date: Sun Oct 21 17:55:44 2007 New Revision: 47674 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/opimpl.py Log: Fix test_lloperation. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Sun Oct 21 17:55:44 2007 @@ -721,6 +721,10 @@ return lltype.cast_opaque_ptr(RESTYPE, obj) op_cast_opaque_ptr.need_result_type = True + def op_cast_ptr_to_adr(self, ptr): + checkptr(ptr) + return llmemory.cast_ptr_to_adr(ptr) + def op_weakref_create(self, v_obj): def objgetter(): # special support for gcwrapper.py return self.getval(v_obj) Modified: pypy/dist/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/opimpl.py Sun Oct 21 17:55:44 2007 @@ -297,10 +297,6 @@ return lltype.cast_pointer(RESTYPE, obj) op_cast_pointer.need_result_type = True -def op_cast_ptr_to_adr(ptr): - checkptr(ptr) - return llmemory.cast_ptr_to_adr(ptr) - def op_cast_adr_to_ptr(TYPE, adr): checkadr(adr) return llmemory.cast_adr_to_ptr(adr, TYPE) From arigo at codespeak.net Sun Oct 21 18:00:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 21 Oct 2007 18:00:06 +0200 (CEST) Subject: [pypy-svn] r47675 - pypy/dist/pypy/translator/goal Message-ID: <20071021160006.CFF718182@code0.codespeak.net> Author: arigo Date: Sun Oct 21 18:00:06 2007 New Revision: 47675 Added: pypy/dist/pypy/translator/goal/targetgcbench2.py (contents, props changed) Modified: pypy/dist/pypy/translator/goal/gcbench.py Log: A target similar to targetgcbench, but where the command-line arguments give the depths of the trees to create. Modified: pypy/dist/pypy/translator/goal/gcbench.py ============================================================================== --- pypy/dist/pypy/translator/goal/gcbench.py (original) +++ pypy/dist/pypy/translator/goal/gcbench.py Sun Oct 21 18:00:06 2007 @@ -107,7 +107,7 @@ t_finish = time.time() println("\tBottom up constrution took %f ms" % ((t_finish-t_start)*1000.)) -def main(): +def main(depths=range(kMinTreeDepth, kMaxTreeDepth+1, 2)): println("Garbage Collector Test") println(" Stretching memory with a binary tree of depth %d" % kStretchTreeDepth) print_diagnostics() @@ -129,7 +129,7 @@ i += 1 print_diagnostics() - for d in range(kMinTreeDepth, kMaxTreeDepth+1, 2): + for d in depths: time_construction(d) if long_lived_tree is None or array[1000] != 1.0/1000: Added: pypy/dist/pypy/translator/goal/targetgcbench2.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/goal/targetgcbench2.py Sun Oct 21 18:00:06 2007 @@ -0,0 +1,24 @@ +import os, sys +from pypy.translator.goal import gcbench + +def entry_point(argv): + if len(argv) > 1: + depths = [int(s) for s in argv[1:]] + gcbench.main(depths) + else: + gcbench.main() + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None + +""" +Why is this a stand-alone target? + +The above target specifies None as the argument types list. +This is a case treated specially in the driver.py . If the list +of input types is empty, it is meant to be a list of strings, +actually implementing argv of the executable. +""" From arigo at codespeak.net Sun Oct 21 18:00:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 21 Oct 2007 18:00:47 +0200 (CEST) Subject: [pypy-svn] r47676 - in pypy/dist/pypy: rpython/memory/gctransform rpython/memory/test translator/goal Message-ID: <20071021160047.E96C48183@code0.codespeak.net> Author: arigo Date: Sun Oct 21 18:00:47 2007 New Revision: 47676 Removed: pypy/dist/pypy/translator/goal/targetcrashstandalone.py Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: Test and fix for the issue of weakrefs getting bogus object references, part 2: the gc transformer. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Sun Oct 21 18:00:47 2007 @@ -551,15 +551,15 @@ args = [self.c_const_gc, c_type_id, c_size, c_can_collect, c_has_finalizer, c_has_weakptr] - v_instance, = op.args - v_addr = hop.genop("cast_ptr_to_adr", [v_instance], - resulttype=llmemory.Address) - livevars = self.push_roots(hop) + livevars = self.push_roots(hop, keep_current_args=True) v_result = hop.genop("direct_call", [malloc_ptr] + args, resulttype=llmemory.GCREF) v_result = hop.genop("cast_opaque_ptr", [v_result], resulttype=WEAKREFPTR) self.pop_roots(hop, livevars) + v_instance, = op.args + v_addr = hop.genop("cast_ptr_to_adr", [v_instance], + resulttype=llmemory.Address) hop.genop("bare_setfield", [v_result, rmodel.inputconst(lltype.Void, "weakptr"), v_addr]) v_weakref = hop.genop("cast_ptr_to_weakrefptr", [v_result], @@ -623,10 +623,10 @@ def pop_alive_nopyobj(self, var, llops): pass - def push_roots(self, hop): + def push_roots(self, hop, keep_current_args=False): if self.incr_stack_ptr is None: return - if self.gcdata.gc.moving_gc: + if self.gcdata.gc.moving_gc and not keep_current_args: # moving GCs don't borrow, so the caller does not need to keep # the arguments alive livevars = [var for var in hop.livevars_after_op() Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Sun Oct 21 18:00:47 2007 @@ -821,3 +821,20 @@ run = self.runner(f, nbargs=0) res = run([]) assert res == 84 + + + def test_many_weakrefs(self): + # test for the case where allocating the weakref itself triggers + # a collection + import weakref + class A: + pass + def f(): + a = A() + i = 0 + while i < 17: + ref = weakref.ref(a) + assert ref() is a + i += 1 + run = self.runner(f, nbargs=0) + run([]) From arigo at codespeak.net Sun Oct 21 18:03:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 21 Oct 2007 18:03:32 +0200 (CEST) Subject: [pypy-svn] r47677 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071021160332.7E16A80B3@code0.codespeak.net> Author: arigo Date: Sun Oct 21 18:03:32 2007 New Revision: 47677 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: Some comments. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Sun Oct 21 18:03:32 2007 @@ -551,12 +551,17 @@ args = [self.c_const_gc, c_type_id, c_size, c_can_collect, c_has_finalizer, c_has_weakptr] + # push and pop the current live variables *including* the argument + # to the weakref_create operation, which must be kept alive and + # moved if the GC needs to collect livevars = self.push_roots(hop, keep_current_args=True) v_result = hop.genop("direct_call", [malloc_ptr] + args, resulttype=llmemory.GCREF) v_result = hop.genop("cast_opaque_ptr", [v_result], resulttype=WEAKREFPTR) self.pop_roots(hop, livevars) + # cast_ptr_to_adr must be done after malloc, as the GC pointer + # might have moved just now. v_instance, = op.args v_addr = hop.genop("cast_ptr_to_adr", [v_instance], resulttype=llmemory.Address) From fijal at codespeak.net Sun Oct 21 23:55:39 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 21 Oct 2007 23:55:39 +0200 (CEST) Subject: [pypy-svn] r47689 - pypy/dist/pypy/module/__builtin__ Message-ID: <20071021215539.AA6BD817A@code0.codespeak.net> Author: fijal Date: Sun Oct 21 23:55:39 2007 New Revision: 47689 Modified: pypy/dist/pypy/module/__builtin__/__init__.py Log: Startup hook for zipimport. No tests as zipimport is not here. Modified: pypy/dist/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/__init__.py (original) +++ pypy/dist/pypy/module/__builtin__/__init__.py Sun Oct 21 23:55:39 2007 @@ -166,3 +166,20 @@ # xxx hide the installer space.delitem(self.w_dict, space.wrap(name)) del self.loaders[name] + + def startup(self, space): + # install zipimport hook + try: + w_import = space.builtin.get('__import__') + w_zipimport = space.call(w_import, space.newlist( + [space.wrap('zipimport')])) + except OperationError, e: + if not e.match(space, space.w_ImportError): + raise + else: + w_sys = space.getbuiltinmodule('sys') + w_path_hooks = space.getattr(w_sys, space.wrap('path_hooks')) + w_append = space.getattr(w_path_hooks, space.wrap('append')) + w_zipimporter = space.getattr(w_zipimport, + space.wrap('zipimporter')) + space.call(w_append, space.newlist([w_zipimporter])) From fijal at codespeak.net Mon Oct 22 10:51:24 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Oct 2007 10:51:24 +0200 (CEST) Subject: [pypy-svn] r47701 - in pypy/dist/pypy: rpython/memory/gctransform translator/backendopt Message-ID: <20071022085124.481F5816F@code0.codespeak.net> Author: fijal Date: Mon Oct 22 10:51:21 2007 New Revision: 47701 Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py pypy/dist/pypy/translator/backendopt/inline.py Log: Speed up inlining (especially important when inlining helpers) Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Mon Oct 22 10:51:21 2007 @@ -19,6 +19,7 @@ from pypy.rlib.rarithmetic import ovfcheck import sets, os, sys from pypy.rpython.lltypesystem.lloperation import llop +from pypy.translator.simplify import join_blocks, cleanup_graph def var_ispyobj(var): if hasattr(var, 'concretetype'): @@ -141,11 +142,12 @@ # XXX quite inefficient: we go over the function lots of times inline.inline_function(self.translator, inline_graph, graph, self.lltype_to_classdef, - raise_analyzer) + raise_analyzer, + cleanup=False) except inline.CannotInline, e: print 'CANNOT INLINE:', e print '\t%s into %s' % (inline_graph, graph) - checkgraph(graph) + cleanup_graph(graph) def compute_borrowed_vars(self, graph): # the input args are borrowed, and stay borrowed for as long as they Modified: pypy/dist/pypy/translator/backendopt/inline.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/inline.py (original) +++ pypy/dist/pypy/translator/backendopt/inline.py Mon Oct 22 10:51:21 2007 @@ -100,10 +100,10 @@ return False def inline_function(translator, inline_func, graph, lltype_to_classdef, - raise_analyzer, call_count_pred=None): + raise_analyzer, call_count_pred=None, cleanup=True): inliner = Inliner(translator, graph, inline_func, lltype_to_classdef, raise_analyzer = raise_analyzer, - call_count_pred=call_count_pred) + call_count_pred=call_count_pred, cleanup=cleanup) return inliner.inline_all() def simple_inline_function(translator, inline_func, graph): @@ -178,9 +178,11 @@ inline_guarded_calls=False, inline_guarded_calls_no_matter_what=False, raise_analyzer=None, - call_count_pred=None): + call_count_pred=None, + cleanup=True): self.translator = translator self.graph = graph + self.do_cleanup = cleanup self.inline_guarded_calls = inline_guarded_calls # if this argument is set, the inliner will happily produce wrong code! # it is used by the exception transformation @@ -213,7 +215,8 @@ operation = block.operations[index_operation] self.inline_once(block, index_operation) count += 1 - self.cleanup() + if self.do_cleanup: + self.cleanup() return count def get_graph_from_op(self, op): @@ -512,12 +515,14 @@ inline_guarded_calls=False, inline_guarded_calls_no_matter_what=False, raise_analyzer=None, - call_count_pred=None): + call_count_pred=None, + cleanup=True): BaseInliner.__init__(self, translator, graph, lltype_to_classdef, inline_guarded_calls, inline_guarded_calls_no_matter_what, raise_analyzer, - call_count_pred) + call_count_pred, + cleanup) self.inline_func = inline_func # to simplify exception matching join_blocks(graph) @@ -683,7 +688,9 @@ callgraph=None, call_count_pred=None, heuristic=inlining_heuristic): + assert threshold is not None and threshold != 1 + to_cleanup = {} from heapq import heappush, heappop, heapreplace, heapify callers = {} # {graph: {graphs-that-call-it}} callees = {} # {graph: {graphs-that-it-calls}} @@ -699,7 +706,6 @@ lltype_to_classdef = translator.rtyper.lltype_to_classdef_mapping() raise_analyzer = RaiseAnalyzer(translator) count = 0 - while heap: weight, _, graph = heap[0] if not valid_weight.get(graph): @@ -744,7 +750,8 @@ try: subcount = inline_function(translator, graph, parentgraph, lltype_to_classdef, raise_analyzer, - call_count_pred) + call_count_pred, cleanup=False) + to_cleanup[graph] = True res = bool(subcount) except CannotInline: try_again[graph] = True @@ -763,6 +770,8 @@ del try_again[parentgraph] heappush(heap, (0.0, -len(callers[parentgraph]), parentgraph)) valid_weight[parentgraph] = False + for graph in to_cleanup: + cleanup_graph(graph) return count def auto_inline_graphs(translator, graphs, threshold, call_count_pred=None, From cfbolz at codespeak.net Mon Oct 22 11:51:07 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 22 Oct 2007 11:51:07 +0200 (CEST) Subject: [pypy-svn] r47702 - pypy/extradoc/talk/bern2007 Message-ID: <20071022095107.3AC048170@code0.codespeak.net> Author: cfbolz Date: Mon Oct 22 11:51:06 2007 New Revision: 47702 Added: pypy/extradoc/talk/bern2007/rpython-overview.txt (contents, props changed) Log: add an rpython overview talk Added: pypy/extradoc/talk/bern2007/rpython-overview.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/bern2007/rpython-overview.txt Mon Oct 22 11:51:06 2007 @@ -0,0 +1,154 @@ +===================== +Overview over RPython +===================== + +RPython +======= + + - implementation language of the PyPy Python interpreter + - restrictions necessary to allow efficient translation + - quite close to Python: e.g. fully garbage collected + - fully testable on top of CPython + - feels a bit like Java without the type annotations + - "being RPython" is a whole-program property + + + +Guide to writing RPython programs +================================= + + - *Important*: First make it work on top of CPython, then worry about RPython + + +Type restrictions +================= + +You cannot arbitrarily mix types. The following is not allowed:: + + # NOT RPYTHON + if condition: + x = 42 + else: + x = "fortytwo" + +This, however is fine:: + + x = 42 + x = "fortytwo" + +(not that it's useful) + + +Control Flow +============ + + - all control flow statements allowed + - no generators (yield) and generator expressions + - global variables are constant + - closures are not supported + +Classes +======= + + - single inheritance:: + + class A(object): + def __init__(self, a, b): + self.a = a + self.b = b + def method(self): + return self.a + self.b + + class B(A): + def __init__(self, a, b, c): + A.__init__(self, a, b) + self.c = c + def method(self): + return A.method(self) + self.c + + + +Data Types +========== + + - immutable: + + - ``int``, ``float``, ``bool`` + - tuples: (1, 2, 3) + - ``string`` + + - mutable: + + - ``list`` + - ``dict`` + + +Numerical Types +=============== + + - Python ``int``s are C ``long``s + - have kind of the obvious C/Java-like behaviour + - difference between Python and RPython: RPython ``int``s wrap around silently + + +Strings +======= + +Fully supported in RPython:: + + >>> a = "hello" + >>> b = " world" + >>> print a + b * 2 + hello world world + +Most string methods work fine + + +Lists +===== + +Restriction: Lists must be homogeneous:: + + >>> l = [] + >>> l.append(1) + >>> l.extend([5, 7, 9]) + >>> l[0] = "this is wrong" # NOT RPYTHON + +Otherwise, most list methods supported (apart from sorting) + +Dicts +===== + +Restriction: Dictionary keys and values must each be homogeneous:: + + >>> d = {"hello": 1, "world": 2} + >>> d["moon"] = 3 + >>> del d[5] # NOT RPYTHON + +The backends typically create specialized versions suited for the keys and +values. + + +Init-time metaprogramming +========================= + + - analysis starts from live objects + - before analysis starts, you can do all sorts of non-RPython thing + - can use init-time metaprogramming:: + + def adder(x): # NOT RPYTHON + def add(y): + return x + y + return add + + add4 = adder(4) + # now add4 is RPython + + +Things you also can do (but usually don't) +========================================== + + - manipulate raw memory addresses (to write GCs) + - explicit specialization of functions + - explicit loop unrolling + From arigo at codespeak.net Mon Oct 22 13:42:16 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 22 Oct 2007 13:42:16 +0200 (CEST) Subject: [pypy-svn] r47703 - pypy/dist/pypy/objspace/std Message-ID: <20071022114216.EB69D817F@code0.codespeak.net> Author: arigo Date: Mon Oct 22 13:42:15 2007 New Revision: 47703 Modified: pypy/dist/pypy/objspace/std/formatting.py Log: Pointless Python 2.3 compatibility fix. There are tons of decorators in ll_os.py anyway. Modified: pypy/dist/pypy/objspace/std/formatting.py ============================================================================== --- pypy/dist/pypy/objspace/std/formatting.py (original) +++ pypy/dist/pypy/objspace/std/formatting.py Mon Oct 22 13:42:15 2007 @@ -4,6 +4,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import ovfcheck, formatd_overflow from pypy.interpreter.error import OperationError +from pypy.tool.sourcetools import func_with_new_name class BaseStringFormatter(object): @@ -480,8 +481,8 @@ raise num = space.bigint_w(w_value) return num.format(digits) - format_num_helper.func_name = 'base%d_num_helper' % len(digits) - return format_num_helper + return func_with_new_name(format_num_helper, + 'base%d_num_helper' % len(digits)) int_num_helper = format_num_helper_generator('%d', '0123456789') oct_num_helper = format_num_helper_generator('%o', '01234567') From arigo at codespeak.net Mon Oct 22 17:25:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 22 Oct 2007 17:25:13 +0200 (CEST) Subject: [pypy-svn] r47706 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071022152513.1D3AC8164@code0.codespeak.net> Author: arigo Date: Mon Oct 22 17:25:12 2007 New Revision: 47706 Added: pypy/dist/pypy/lang/smalltalk/ (props changed) pypy/dist/pypy/lang/smalltalk/__init__.py (contents, props changed) pypy/dist/pypy/lang/smalltalk/model.py (contents, props changed) pypy/dist/pypy/lang/smalltalk/test/ (props changed) pypy/dist/pypy/lang/smalltalk/test/__init__.py (contents, props changed) pypy/dist/pypy/lang/smalltalk/test/test_model.py (contents, props changed) Log: (all) Started the basic object model of Smalltalk. Added: pypy/dist/pypy/lang/smalltalk/__init__.py ============================================================================== Added: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/model.py Mon Oct 22 17:25:12 2007 @@ -0,0 +1,13 @@ + +class W_Object: + def __init__(self, w_class): + self.w_class = w_class + +class W_Class(W_Object): + def __init__(self, w_class, w_superclass): + W_Object.__init__(self, w_class) + self.w_superclass = w_superclass + self.methoddict = {} + + def new(self): + return W_Object(self) Added: pypy/dist/pypy/lang/smalltalk/test/__init__.py ============================================================================== Added: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Mon Oct 22 17:25:12 2007 @@ -0,0 +1,7 @@ +from pypy.lang.smalltalk import model + +def test_new(): + w_mycls = model.W_Class(None, None) + w_myinstance = w_mycls.new() + assert isinstance(w_myinstance, model.W_Object) + assert w_myinstance.w_class is w_mycls From arigo at codespeak.net Mon Oct 22 17:52:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 22 Oct 2007 17:52:31 +0200 (CEST) Subject: [pypy-svn] r47707 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071022155231.031B38169@code0.codespeak.net> Author: arigo Date: Mon Oct 22 17:52:31 2007 New Revision: 47707 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_model.py Log: (all) More of the basic object model: NamedVarsObject, for objects with a fixed-length list of slots (i.e. most Squeak objects). Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Mon Oct 22 17:52:31 2007 @@ -3,11 +3,34 @@ def __init__(self, w_class): self.w_class = w_class +class W_NamedVarsObject(W_Object): + def __init__(self, w_class): + W_Object.__init__(self, w_class) + self.named_vars = [None] * w_class.namedvarssize + + def getnamedvar(self, index): + return self.named_vars[index] + + def setnamedvar(self, index, w_value): + self.named_vars[index] = w_value + + +NO_INDEXED = 0 +OBJ_INDEXED = 1 +BYTE_INDEXED = 2 +WORD_INDEXED = 3 + class W_Class(W_Object): - def __init__(self, w_class, w_superclass): + def __init__(self, w_class, w_superclass, namedvarssize=0, + indexed=NO_INDEXED): W_Object.__init__(self, w_class) self.w_superclass = w_superclass self.methoddict = {} + self.namedvarssize = namedvarssize + self.indexed = indexed def new(self): - return W_Object(self) + if self.indexed == NO_INDEXED: + return W_NamedVarsObject(self) + else: + raise NotImplementedError(self.indexed) Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Mon Oct 22 17:52:31 2007 @@ -1,3 +1,4 @@ +import py from pypy.lang.smalltalk import model def test_new(): @@ -5,3 +6,13 @@ w_myinstance = w_mycls.new() assert isinstance(w_myinstance, model.W_Object) assert w_myinstance.w_class is w_mycls + +def test_new_namedvars(): + w_mycls = model.W_Class(None, None, 3) + w_myinstance = w_mycls.new() + assert isinstance(w_myinstance, model.W_NamedVarsObject) + assert w_myinstance.w_class is w_mycls + assert w_myinstance.getnamedvar(0) is None + py.test.raises(IndexError, lambda: w_myinstance.getnamedvar(3)) + w_myinstance.setnamedvar(1, w_myinstance) + assert w_myinstance.getnamedvar(1) is w_myinstance From arigo at codespeak.net Mon Oct 22 18:02:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 22 Oct 2007 18:02:31 +0200 (CEST) Subject: [pypy-svn] r47708 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071022160231.1883B8169@code0.codespeak.net> Author: arigo Date: Mon Oct 22 18:02:31 2007 New Revision: 47708 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_model.py Log: (all) Implemented objects with indexed variables (i.e. var-sized, with the size choosen at instantiation time). Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Mon Oct 22 18:02:31 2007 @@ -14,6 +14,18 @@ def setnamedvar(self, index, w_value): self.named_vars[index] = w_value +class W_IndexedNamedVarsObject(W_NamedVarsObject): + def __init__(self, w_class, size): + W_NamedVarsObject.__init__(self, w_class) + self.indexed_vars = [None] * size + + def getindexedvar(self, index): + return self.indexed_vars[index] + + def setindexedvar(self, index, w_value): + self.indexed_vars[index] = w_value + +# ____________________________________________________________ NO_INDEXED = 0 OBJ_INDEXED = 1 @@ -29,8 +41,11 @@ self.namedvarssize = namedvarssize self.indexed = indexed - def new(self): + def new(self, size=0): if self.indexed == NO_INDEXED: + assert size == 0 return W_NamedVarsObject(self) + elif self.indexed == OBJ_INDEXED: + return W_IndexedNamedVarsObject(self, size) else: raise NotImplementedError(self.indexed) Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Mon Oct 22 18:02:31 2007 @@ -16,3 +16,17 @@ py.test.raises(IndexError, lambda: w_myinstance.getnamedvar(3)) w_myinstance.setnamedvar(1, w_myinstance) assert w_myinstance.getnamedvar(1) is w_myinstance + +def test_new_indexednamedvars(): + w_mycls = model.W_Class(None, None, 3, model.OBJ_INDEXED) + w_myinstance = w_mycls.new(2) + assert isinstance(w_myinstance, model.W_IndexedNamedVarsObject) + assert w_myinstance.w_class is w_mycls + assert w_myinstance.getnamedvar(0) is None + py.test.raises(IndexError, lambda: w_myinstance.getnamedvar(3)) + w_myinstance.setnamedvar(1, w_myinstance) + assert w_myinstance.getnamedvar(1) is w_myinstance + assert w_myinstance.getindexedvar(1) is None + py.test.raises(IndexError, lambda: w_myinstance.getindexedvar(2)) + w_myinstance.setindexedvar(0, w_myinstance) + assert w_myinstance.getindexedvar(0) is w_myinstance From antocuni at codespeak.net Mon Oct 22 20:15:26 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 22 Oct 2007 20:15:26 +0200 (CEST) Subject: [pypy-svn] r47709 - pypy/extradoc/talk/dls2007 Message-ID: <20071022181526.08AE780FA@code0.codespeak.net> Author: antocuni Date: Mon Oct 22 20:15:25 2007 New Revision: 47709 Modified: pypy/extradoc/talk/dls2007/rpython-talk.latex pypy/extradoc/talk/dls2007/rpython-talk.pdf Log: last minute's changes Modified: pypy/extradoc/talk/dls2007/rpython-talk.latex ============================================================================== --- pypy/extradoc/talk/dls2007/rpython-talk.latex (original) +++ pypy/extradoc/talk/dls2007/rpython-talk.latex Mon Oct 22 20:15:25 2007 @@ -16,18 +16,32 @@ \usetheme{Boadilla} %\usetheme{Warsaw} \setbeamercovered{transparent} +\setbeamertemplate{navigation symbols}{} \title[RPython, a dynamic static language]{RPython\\A Step Towards Reconciling Dynamically and Statically Typed OO Languages} -\author[Antonio Cuni]{Antonio Cuni -- DISI, Universit? degli Studi di Genova} +\author[Antonio Cuni]{Antonio Cuni -- DISI, Universit? degli Studi di Genova\\\small{joint work with D. Ancona, M. Ancona, N. D. Matsakis}} \institute[DSL'07 OOPSLA]{DLS'07 OOPSLA Montreal CA} \date{October 22, 2007} + % outline +\AtBeginSection[] +{ + \begin{frame} + \frametitle{Outline} + \small + \tableofcontents[currentsection,hideothersubsections] + \normalsize + \end{frame} +} + \begin{document} \begin{frame} \titlepage \end{frame} +\section{Introduction to RPython} + \begin{frame} \frametitle{Dynamic languages for .NET and JVM} \begin{itemize} @@ -63,12 +77,12 @@ \frametitle{RPython Quick Facts} \begin{itemize} - \item Restricted subset of Python - \item Statically typed (type inference) + \item \textbf{Restricted subset} of Python + \item Statically typed (\textbf{type inference}) \item Still allows metaprogramming \item RPython programs still run under {\{}C,J,Iron{\}}Python \item Three backends: C, .NET, JVM - \item Almost as fast as C, C{\#}, Java + \item \textbf{Almost as fast} as C, C{\#}, Java \end{itemize} \end{frame} @@ -117,13 +131,13 @@ \frametitle{Other restrictions} \begin{itemize} - \item Globals are assumed to be constant + \item Globals are assumed to be \textbf{constant} \item \texttt{yield} and generators not supported \item No special \_\_methods\_\_ (except \_\_init\_\_ and \_\_del\_\_) \item No run-time definition of new functions and classes \item Cannot modify classes at run-time \item Cannot change the {\_}{\_}class{\_}{\_} of an object - \item Single inheritance, with limited support for mixins + \item \textbf{Single inheritance}, with limited support for mixins \end{itemize} \end{frame} @@ -171,6 +185,7 @@ \end{itemize} \end{frame} +\section{RPython idioms} \begin{frame}[fragile] \frametitle{Compute complex constants} @@ -231,6 +246,7 @@ \end{exampleblock} \end{frame} +\section{Implementation notes and benchmarks} \begin{frame} \frametitle{The Translation Toolchain} @@ -241,7 +257,7 @@ \begin{itemize} \item High level Python types (\texttt{List(Integer)}) \end{itemize} - \item \textbf{RTyper}: high level types -{\textgreater} low level types + \item \textbf{RTyper}: high level types -{}-{\textgreater} low level types \begin{itemize} \item lltype for C, ootype for CLI and JVM \end{itemize} @@ -301,6 +317,7 @@ \end{itemize} \end{frame} +\section{Conclusions} \begin{frame} \frametitle{What's good about RPython} @@ -329,7 +346,7 @@ \begin{frame} - \frametitle{About PyPy (1)} + \frametitle{About PyPy} \begin{alertblock}{Python in (R)Python} \begin{itemize} @@ -349,28 +366,28 @@ \end{alertblock} \end{frame} -\begin{frame} - \frametitle{About PyPy (2)} - - \begin{alertblock}{Low-level aspects inserted by the TT} - \begin{itemize} - \item Garbage collector - \item Threading model/Stackless - \item Additional language features - \item JIT compiler (only for the C backend so far) - \end{itemize} - \end{alertblock} +%\begin{frame} +% \frametitle{About PyPy (2)} - \pause - \begin{alertblock}{PyPy you can get} - \begin{itemize} - \item pypy-c (about 2x slower than CPython) - \item pypy-c-jit (up to 60x \textbf{faster} than CPython) - \item pypy.NET (about 6x slower than IronPython) - \item pypy-jvm (about 30\% \textbf{faster} than Jython) - \end{itemize} - \end{alertblock} -\end{frame} +% \begin{alertblock}{Low-level aspects inserted by the TT} +% \begin{itemize} +% \item Garbage collector +% \item Threading model/Stackless +% \item Additional language features +% \item JIT compiler (only for the C backend so far) +% \end{itemize} +% \end{alertblock} + +% \pause +% \begin{alertblock}{PyPy you can get} +% \begin{itemize} +% \item pypy-c (about 2x slower than CPython) +% \item pypy-c-jit (up to 60x \textbf{faster} than CPython) +% \item pypy.NET (about 6x slower than IronPython) +% \item pypy-jvm (about 30\% \textbf{faster} than Jython) +% \end{itemize} +% \end{alertblock} +%\end{frame} \begin{frame} @@ -379,11 +396,11 @@ \item The whole PyPy Team \begin{itemize} \item RPython is \textbf{not} mine :-) + \item Our contributions: CLI and JVM backends \end{itemize} - \item Davide Ancona - \item Massimo Ancona - \item Nicholas D. Matsakis + \item Thanks for the attention + \item Any question? \end{itemize} \end{frame} Modified: pypy/extradoc/talk/dls2007/rpython-talk.pdf ============================================================================== Binary files. No diff available. From fijal at codespeak.net Mon Oct 22 21:10:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Oct 2007 21:10:41 +0200 (CEST) Subject: [pypy-svn] r47713 - in pypy/dist/pypy/translator: . goal goal/test2 Message-ID: <20071022191041.027308139@code0.codespeak.net> Author: fijal Date: Mon Oct 22 21:10:40 2007 New Revision: 47713 Added: pypy/dist/pypy/translator/goal/test2/test_timing.py (contents, props changed) pypy/dist/pypy/translator/goal/timing.py (contents, props changed) Modified: pypy/dist/pypy/translator/driver.py pypy/dist/pypy/translator/goal/translate.py Log: A very fancy timing for annotation parts. If someone is interested in not seeing that, I might make an option. Also the test, I've managed to not have it working first time (what's not tested is broken) Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Mon Oct 22 21:10:40 2007 @@ -3,6 +3,7 @@ from pypy.translator.translator import TranslationContext, graphof from pypy.translator.tool.taskengine import SimpleTaskEngine from pypy.translator.goal import query +from pypy.translator.goal.timing import Timer from pypy.annotation import model as annmodel from pypy.annotation.listdef import s_list_of_strings from pypy.annotation import policy as annpolicy @@ -14,7 +15,6 @@ log = py.log.Producer("translation") py.log.setconsumer("translation", ansi_log) - DEFAULTS = { 'translation.gc': 'ref', 'translation.cc': None, @@ -82,6 +82,7 @@ disable=[], exe_name=None, extmod_name=None, config=None, overrides=None): + self.timer = Timer() SimpleTaskEngine.__init__(self) self.log = log @@ -260,16 +261,20 @@ return else: self.log.info("%s..." % title) - instrument = False + self.timer.start_event(goal) try: - res = func() - except Instrument: - instrument = True - if not func.task_idempotent: - self.done[goal] = True - if instrument: - self.proceed('compile') - assert False, 'we should not get here' + instrument = False + try: + res = func() + except Instrument: + instrument = True + if not func.task_idempotent: + self.done[goal] = True + if instrument: + self.proceed('compile') + assert False, 'we should not get here' + finally: + self.timer.end_event(goal) return res def task_annotate(self): @@ -806,7 +811,6 @@ from pypy.translator.goal import unixcheckpoint unixcheckpoint.restartable_point(auto='run') - def mkexename(name): if sys.platform == 'win32': name = os.path.normpath(name + '.exe') Added: pypy/dist/pypy/translator/goal/test2/test_timing.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/goal/test2/test_timing.py Mon Oct 22 21:10:40 2007 @@ -0,0 +1,17 @@ + +import py +from pypy.translator.goal.timing import Timer + +def test_timing(): + t = Timer(list(reversed([1,2,3,4,5,6])).pop) + t.start_event('x') + t.end_event('x') + t.start_event('y') + t.end_event('y') + py.test.raises(AssertionError, "t.end_event('y')") + t.start_event('z') + py.test.raises(AssertionError, "t.end_event('y')") + t.end_event('z') + assert t.events == [('x', 1), ('y', 1), ('z', 1)] + assert t.ttime() == 5 + Added: pypy/dist/pypy/translator/goal/timing.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/goal/timing.py Mon Oct 22 21:10:40 2007 @@ -0,0 +1,50 @@ + +""" Module for keeping detailed informations about +times of certain driver parts +""" + +import time +import py +from pypy.tool.ansi_print import ansi_log +log = py.log.Producer("Timer") +py.log.setconsumer("Timer", ansi_log) + +class Timer(object): + def __init__(self, timer=time.time): + self.events = [] + self.next_even = None + self.timer = timer + self.t0 = None + + def start_event(self, event): + now = self.timer() + if self.t0 is None: + self.t0 = now + self.next_event = event + self.start_time = now + + def end_event(self, event): + assert self.next_event == event + now = self.timer() + self.events.append((event, now - self.start_time)) + self.next_event = None + self.tk = now + + def ttime(self): + return self.tk - self.t0 + + def pprint(self): + """ Pretty print + """ + spacing = " "*(30 - len("Total:")) + total = "Total:%s - %.1f s" % (spacing, self.ttime()) + log.bold("Timings:") + for event, time in self.events: + spacing = " "*(30 - len(event)) + first = "%s%s - " % (event, spacing) + second = "%.1f s" % (time,) + additional_spaces = " " * (len(total) - len(first) - len(second)) + log.bold("%s%s%s" % (first, additional_spaces, second)) + log.bold("=" * len(total)) + log.bold(total) + Modified: pypy/dist/pypy/translator/goal/translate.py ============================================================================== --- pypy/dist/pypy/translator/goal/translate.py (original) +++ pypy/dist/pypy/translator/goal/translate.py Mon Oct 22 21:10:40 2007 @@ -269,8 +269,10 @@ drv.exe_name = targetspec_dic['__name__'] + '-%(backend)s' goals = translateconfig.goals - drv.proceed(goals) - + try: + drv.proceed(goals) + finally: + drv.timer.pprint() except SystemExit: raise except: From tismer at codespeak.net Mon Oct 22 21:13:53 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Mon, 22 Oct 2007 21:13:53 +0200 (CEST) Subject: [pypy-svn] r47714 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071022191353.C9478813E@code0.codespeak.net> Author: tismer Date: Mon Oct 22 21:13:53 2007 New Revision: 47714 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt (props changed) pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (props changed) Log: eolstyle From fijal at codespeak.net Mon Oct 22 21:49:39 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Oct 2007 21:49:39 +0200 (CEST) Subject: [pypy-svn] r47715 - pypy/dist/pypy/tool Message-ID: <20071022194939.9ABFF8139@code0.codespeak.net> Author: fijal Date: Mon Oct 22 21:49:39 2007 New Revision: 47715 Added: pypy/dist/pypy/tool/lsprofcalltree.py (contents, props changed) Log: Add lsprofcalltree to pypy tree. It took me some time to find it. Added: pypy/dist/pypy/tool/lsprofcalltree.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/tool/lsprofcalltree.py Mon Oct 22 21:49:39 2007 @@ -0,0 +1,119 @@ +# lsprofcalltree.py: lsprof output which is readable by kcachegrind +# David Allouche +# Jp Calderone & Itamar Shtull-Trauring +# Johan Dahlin + +import optparse +import os +import sys + +try: + import cProfile +except ImportError: + raise SystemExit("This script requires cProfile from Python 2.5") + +def label(code): + if isinstance(code, str): + return ('~', 0, code) # built-in functions ('~' sorts at the end) + else: + return '%s %s:%d' % (code.co_name, + code.co_filename, + code.co_firstlineno) + +class KCacheGrind(object): + def __init__(self, profiler): + self.data = profiler.getstats() + self.out_file = None + + def output(self, out_file): + self.out_file = out_file + print >> out_file, 'events: Ticks' + self._print_summary() + for entry in self.data: + self._entry(entry) + + def _print_summary(self): + max_cost = 0 + for entry in self.data: + totaltime = int(entry.totaltime * 1000) + max_cost = max(max_cost, totaltime) + print >> self.out_file, 'summary: %d' % (max_cost,) + + def _entry(self, entry): + out_file = self.out_file + + code = entry.code + #print >> out_file, 'ob=%s' % (code.co_filename,) + if isinstance(code, str): + print >> out_file, 'fi=~' + else: + print >> out_file, 'fi=%s' % (code.co_filename,) + print >> out_file, 'fn=%s' % (label(code),) + + inlinetime = int(entry.inlinetime * 1000) + if isinstance(code, str): + print >> out_file, '0 ', inlinetime + else: + print >> out_file, '%d %d' % (code.co_firstlineno, inlinetime) + + # recursive calls are counted in entry.calls + if entry.calls: + calls = entry.calls + else: + calls = [] + + if isinstance(code, str): + lineno = 0 + else: + lineno = code.co_firstlineno + + for subentry in calls: + self._subentry(lineno, subentry) + print >> out_file + + def _subentry(self, lineno, subentry): + out_file = self.out_file + code = subentry.code + #print >> out_file, 'cob=%s' % (code.co_filename,) + print >> out_file, 'cfn=%s' % (label(code),) + if isinstance(code, str): + print >> out_file, 'cfi=~' + print >> out_file, 'calls=%d 0' % (subentry.callcount,) + else: + print >> out_file, 'cfi=%s' % (code.co_filename,) + print >> out_file, 'calls=%d %d' % ( + subentry.callcount, code.co_firstlineno) + + totaltime = int(subentry.totaltime * 1000) + print >> out_file, '%d %d' % (lineno, totaltime) + +def main(args): + usage = "%s [-o output_file_path] scriptfile [arg] ..." + parser = optparse.OptionParser(usage=usage % sys.argv[0]) + parser.allow_interspersed_args = False + parser.add_option('-o', '--outfile', dest="outfile", + help="Save stats to ", default=None) + + if not sys.argv[1:]: + parser.print_usage() + sys.exit(2) + + options, args = parser.parse_args() + + if not options.outfile: + options.outfile = '%s.log' % os.path.basename(args[0]) + + sys.argv[:] = args + + prof = cProfile.Profile() + try: + try: + prof = prof.run('execfile(%r)' % (sys.argv[0],)) + except SystemExit: + pass + finally: + kg = KCacheGrind(prof) + kg.output(file(options.outfile, 'w')) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) From tismer at codespeak.net Mon Oct 22 23:21:46 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Mon, 22 Oct 2007 23:21:46 +0200 (CEST) Subject: [pypy-svn] r47716 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071022212146.884C48141@code0.codespeak.net> Author: tismer Date: Mon Oct 22 23:21:45 2007 New Revision: 47716 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: added my info and a small task Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Mon Oct 22 23:21:45 2007 @@ -48,6 +48,8 @@ - think about/remove orphaned parts - finish ll2ctypes, especially rffi_platform + + - remove orebuilt costate vars (might happen before) Interpreter +++++++++++ Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Mon Oct 22 23:21:45 2007 @@ -13,7 +13,7 @@ Anders Chrigstrom lives there Jacob Hallen lives there Laura Creighton lives there -Christian Tismer Laura and Jacob's +Christian Tismer 18-26 Laura and Jacob's Maciej Fijalkowski 19-26 Laura and Jacob's Armin Rigo lives thereish ==================== ============== ===================== @@ -38,7 +38,7 @@ Lene Wagner ? ? Amaury Forgeot d'Arc ? ? Valentino Volonghi ? ? -Boris Feigin ? ? +Boris Feigin ? ? Bert Freudenberg ? ? Beatrice Duering ? ? Johan Hahn ? ? From fijal at codespeak.net Mon Oct 22 23:54:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Oct 2007 23:54:46 +0200 (CEST) Subject: [pypy-svn] r47717 - pypy/dist/pypy/doc Message-ID: <20071022215446.EC5578109@code0.codespeak.net> Author: fijal Date: Mon Oct 22 23:54:46 2007 New Revision: 47717 Modified: pypy/dist/pypy/doc/translation.txt Log: Don't confuse people (too much) and be honest Modified: pypy/dist/pypy/doc/translation.txt ============================================================================== --- pypy/dist/pypy/doc/translation.txt (original) +++ pypy/dist/pypy/doc/translation.txt Mon Oct 22 23:54:46 2007 @@ -803,6 +803,8 @@ way of motivating the design of the ootypesystem. More recently, it has been worked on by Niklaus Haldimann. +GenSquek is considered dead now. + GenJS +++++ @@ -845,6 +847,7 @@ GenCL has seen work from many people, but the driving force has mostly been Seo Sanghyeon. +GenCL is considered dead now. .. _`Python again`: .. _`Python back-end`: @@ -866,178 +869,9 @@ External Function Calls ======================= -*Warning* This approach while is still used at the moment by PyPy to -expose native operations (like ``os.read``, ``os.write``) is going to be -phased out. It has shown to be too cumbersome in practice. An approach -based on `rctypes`_ should replace it. - -.. _rctypes: rctypes.html - -There are some functions that we don't want to implement in Python for various -reasons (e.g. if they need to make calls into the OS). These can be -implemented by writing backend code by hand that either implements the -functionality itself or calls appropriate libraries. - -Of course the annotator does not know what types these functions return so we -need a way to attach fixed annotations to such functions. Additionally for -every such function we need a low level helper function that behaves like the -hand-written backend code so that we can still test a call to such an external -function. - - -Annotating external function calls ----------------------------------- - -All the information about external functions that are important for the -annotation process are written into `pypy/rpython/extfunctable.py`_. There is a -function called ``declare`` that allows to tell the annotator the return type -of these functions and to attach a low level dummy implementation:: - - declare(funct, annotation_factory, lowlevel_dummy) - -Here ``funct`` is the function that is supposed to be implemented in the -backend, ``annotation_factory`` is a function that returns an appropriate -annotation of the return value (an instance of a primitive type is also ok, so -you can do ``declare(func, int...)``), lowlevel_dummy is a string of the form -``module/ll_function`` that specifies where the low level dummy function is -defined. Here ``module`` means a Python file in `pypy/rpython/module/`_ and -``ll_function`` is a low level function defined in that file. - -If the annotator discovers a call to ``func`` it does not try to follow that -call further (even though that might be possible if the function is written in -Python) but annotates it with the given type immediately. The `RPython Typer`_ -replaces calls to ``func`` with calls to the function -``module.ll_function``. The backend is supposed to replace calls to functions -to ``module.ll_function`` by whatever code it finds appropriate. - -Implementing low level replacement functions in Python ---------------------------------------------------------- - -The dummy low level replacement functions are there to implement the external -function on the low level. In contrast to the original function they should -take arguments that are of `low-level type`_. Most of the times they are -implemented by calling appropriate low level to high level conversion -functions and then calling the original function again. - -.. _`low-level type`: rtyper.html#low-level-type - -If the function is supposed to really be implemented by the backend then the -low level function should have an attribute ``.suggested_primitive = True`` -attached. If this is not the case the low level function itself will be -translated and used. - - -Implementing the external functions in the C backend ----------------------------------------------------- - -When the C-backend produces C code and finds a function call to one of the -dummy functions it replaces the call to it by a call to a function written in -C. This mapping is done in the file `pypy/translator/c/extfunc.py`_. In there -is a dictionary called ``EXTERNALS`` which contains mappings from dummy -functions to strings:: - - EXTERNALS = { - module.ll_function: "LL_c_name_of_function" - ... - } - -Here ``LL_c_name_of_function`` is the name of the C function that implements -the functionality of the ``module.ll_function``. These C implementation are -found in the directory `pypy/translator/c/src/`_. - -It sometimes necessary to access a certain function or type that is written -in Python from the C code you write by hand (for example for constructing the -return value in the correct low level type). This can be a problem because the -C backend mangles names to prevent clashes. To get a certain low level type -under a fixed name the function ``predeclare_common_types`` needs to be -changed. This function is a generator that yields tuples of the form -``('c_name', LLTYPE)``. This makes the genc assign the name ``c_name`` to the -type ``LLTYPE``. Similarly all the function defined in -``predeclare_utility_functions`` are automatically made accessible under their -name in C. - -Testing -------- - -It's not always trivial to properly test that all the information that is -needed for a given external function is correct. Usually for a given function -you test at least two different things: Once the dummy low level function by -calling it directly. This is usually done in `pypy/rpython/module/test`_ by -just calling the low level function with appropriately constructed arguments -directly. In addition there should be test that actually tries to translate -a function calling the external function to backend code, compile and run -it. This makes sure that the annotator gets the annotation right and that the -backend code is correct. For the C backend this is done in -`pypy/translator/c/test/test_extfunc.py`_. - -Example -------- - -To make this clearer the following shows all the relevant pieces that are -needed for implementing os.open. It is implemented in the following way at -interp-level in the `mixed posix module`_:: - - def open(space, fname, flag, mode=0777): - try: - fd = os.open(fname, flag, mode) - except OSError, e: - raise wrap_oserror(space, e) - return space.wrap(fd) - open.unwrap_spec = [ObjSpace, str, int, int] - - -If the annotator tries to annotate this function it will use the declaration -it finds in the file `pypy/rpython/extfunctable.py`_:: - - declare(os.open, int , 'll_os/open') - -This means that the function returns an int and that the dummy low level -function can be found in `pypy/rpython/module/ll_os.py`_:: - - def ll_os_open(fname, flag, mode): - return os.open(from_rstr(fname), flag, mode) - ll_os_open.suggested_primitive = True - -The ``suggested_primitive`` attribute of the ``ll_os_open`` is set to True, -because it is reasonable that this function is written directly in the backend. -If the C backend encounters a call to ``ll_os_open`` somewhere in the code it -checks the ``EXTERNALS`` table in `pypy/translator/c/extfunc.py`_. The -relevant part for ``os.open`` is:: - - from pypy.rpython.module import ll_os - EXTERNALS = { - ... - ll_os.ll_os_open: 'LL_os_open', - ... - } - -The `LL_os_open` function is implemented in the file -`pypy/translator/c/src/ll_os.h`_:: - - int LL_os_open(RPyString *filename, int flag, int mode) - { - int fd = open(RPyString_AsString(filename), flag, mode); - if (fd < 0) - RPYTHON_RAISE_OSERROR(errno); - return fd; - } - -One test for all this can be found in -`pypy/translator/c/test/test_extfunc.py`_:: - - def test_os_open(): - tmpfile = str(udir.join('test_os_open.txt')) - def does_stuff(): - fd = os.open(tmpfile, os.O_WRONLY | os.O_CREAT, 0777) - return fd - f1 = compile(does_stuff, []) - fd = f1() - os.close(fd) - assert os.path.exists(tmpfile) - - -.. _`mixed posix module`: ../module/posix/ +External function call approach is described in `rffi`_ documentation. +.. _`rffi`: rffi.html How It Fits Together ==================== From tismer at codespeak.net Tue Oct 23 00:35:13 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Tue, 23 Oct 2007 00:35:13 +0200 (CEST) Subject: [pypy-svn] r47719 - pypy/dist/pypy/doc Message-ID: <20071022223513.5A6B680F3@code0.codespeak.net> Author: tismer Date: Tue Oct 23 00:35:12 2007 New Revision: 47719 Modified: pypy/dist/pypy/doc/translation.txt Log: typo Modified: pypy/dist/pypy/doc/translation.txt ============================================================================== --- pypy/dist/pypy/doc/translation.txt (original) +++ pypy/dist/pypy/doc/translation.txt Tue Oct 23 00:35:12 2007 @@ -803,7 +803,7 @@ way of motivating the design of the ootypesystem. More recently, it has been worked on by Niklaus Haldimann. -GenSquek is considered dead now. +GenSqueak is considered dead now. GenJS +++++ From fijal at codespeak.net Tue Oct 23 00:37:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Oct 2007 00:37:46 +0200 (CEST) Subject: [pypy-svn] r47720 - pypy/dist/pypy/translator Message-ID: <20071022223746.4B7CE80F3@code0.codespeak.net> Author: fijal Date: Tue Oct 23 00:37:45 2007 New Revision: 47720 Modified: pypy/dist/pypy/translator/driver.py Log: Add a profiler hook for translation steps. No need for option as this will probably be removed in some point in future Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Tue Oct 23 00:37:45 2007 @@ -51,6 +51,7 @@ def backend_to_typesystem(backend): return _BACKEND_TO_TYPESYSTEM.get(backend, 'ootype') +PROFILE = set([]) class Instrument(Exception): pass @@ -254,6 +255,15 @@ def info(self, msg): log.info(msg) + def _profile(self, goal, func): + from cProfile import Profile + from pypy.tool.lsprofcalltree import KCacheGrind + d = {'func':func} + prof = Profile() + prof.runctx("res = func()", globals(), d) + KCacheGrind(prof).output(open(goal + ".out", "w")) + return d['res'] + def _do(self, goal, func, *args, **kwds): title = func.task_title if goal in self.done: @@ -265,7 +275,10 @@ try: instrument = False try: - res = func() + if goal in PROFILE: + res = self._profile(goal, func) + else: + res = func() except Instrument: instrument = True if not func.task_idempotent: From fijal at codespeak.net Tue Oct 23 01:11:53 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Oct 2007 01:11:53 +0200 (CEST) Subject: [pypy-svn] r47721 - pypy/dist/pypy/tool/algo Message-ID: <20071022231153.420E580F9@code0.codespeak.net> Author: fijal Date: Tue Oct 23 01:11:51 2007 New Revision: 47721 Modified: pypy/dist/pypy/tool/algo/graphlib.py Log: This is apparently faster :-/ Modified: pypy/dist/pypy/tool/algo/graphlib.py ============================================================================== --- pypy/dist/pypy/tool/algo/graphlib.py (original) +++ pypy/dist/pypy/tool/algo/graphlib.py Tue Oct 23 01:11:51 2007 @@ -95,7 +95,7 @@ if stackpos[v] is not None: # back-edge result.append(edgestack[stackpos[v]:]) visit(root) - return result + return result def break_cycles(vertices, edges): """Enumerates a reasonably minimal set of edges that must be removed to @@ -111,7 +111,10 @@ for cycle in all_cycles(random_vertex, component, edges): #print '\tcycle:', [e.source+e.target for e in cycle] for edge in cycle: - edge_weights[edge] = edge_weights.get(edge, 0) + 1 + try: + edge_weights[edge] += 1 + except KeyError: + edge_weights[edge] = 1 if edge_weights: max_weight = max(edge_weights.values()) for edge, weight in edge_weights.iteritems(): From fijal at codespeak.net Tue Oct 23 01:56:15 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Oct 2007 01:56:15 +0200 (CEST) Subject: [pypy-svn] r47722 - pypy/dist/pypy/tool/algo Message-ID: <20071022235615.AC1B780F5@code0.codespeak.net> Author: fijal Date: Tue Oct 23 01:56:15 2007 New Revision: 47722 Modified: pypy/dist/pypy/tool/algo/graphlib.py Log: Revert last checkin as I see inconsistent results around Modified: pypy/dist/pypy/tool/algo/graphlib.py ============================================================================== --- pypy/dist/pypy/tool/algo/graphlib.py (original) +++ pypy/dist/pypy/tool/algo/graphlib.py Tue Oct 23 01:56:15 2007 @@ -111,10 +111,7 @@ for cycle in all_cycles(random_vertex, component, edges): #print '\tcycle:', [e.source+e.target for e in cycle] for edge in cycle: - try: - edge_weights[edge] += 1 - except KeyError: - edge_weights[edge] = 1 + edge_weights[edge] = edge_weights.get(edge, 0) + 1 if edge_weights: max_weight = max(edge_weights.values()) for edge, weight in edge_weights.iteritems(): From fijal at codespeak.net Tue Oct 23 14:11:39 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Oct 2007 14:11:39 +0200 (CEST) Subject: [pypy-svn] r47723 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071023121139.CC23780E2@code0.codespeak.net> Author: fijal Date: Tue Oct 23 14:11:36 2007 New Revision: 47723 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Log: Some tasks. Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Tue Oct 23 14:11:36 2007 @@ -46,9 +46,9 @@ - decide how to implement constants in rffi - think about/remove orphaned parts + + - implement callback mechanism for rffi/lltype/ll2ctypes - - finish ll2ctypes, especially rffi_platform - - remove orebuilt costate vars (might happen before) Interpreter @@ -61,6 +61,8 @@ - review the things implemented at applevel whether they are performance-critical + - app-level ctypes + JIT +++ From fijal at codespeak.net Tue Oct 23 14:17:02 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Oct 2007 14:17:02 +0200 (CEST) Subject: [pypy-svn] r47724 - in pypy/dist/pypy/module/zipimport: . test Message-ID: <20071023121702.D50D880E4@code0.codespeak.net> Author: fijal Date: Tue Oct 23 14:17:01 2007 New Revision: 47724 Added: pypy/dist/pypy/module/zipimport/ pypy/dist/pypy/module/zipimport/__init__.py (contents, props changed) pypy/dist/pypy/module/zipimport/app_zipimport.py (contents, props changed) pypy/dist/pypy/module/zipimport/interp_zipimport.py (contents, props changed) pypy/dist/pypy/module/zipimport/test/ pypy/dist/pypy/module/zipimport/test/test_zipimport.py (contents, props changed) Log: Zipimport module, first test passing. Added: pypy/dist/pypy/module/zipimport/__init__.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/zipimport/__init__.py Tue Oct 23 14:17:01 2007 @@ -0,0 +1,15 @@ + +""" Zipimport module +""" + +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + applevelname = 'zipimport' + + interpleveldefs = {'zipimporter':'interp_zipimport.W_ZipImporter'} + + appleveldefs = { + 'ZipImporterError' : 'app_zipimport.ZipImporterError', + } + Added: pypy/dist/pypy/module/zipimport/app_zipimport.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/zipimport/app_zipimport.py Tue Oct 23 14:17:01 2007 @@ -0,0 +1,3 @@ + +class ZipImporterError(Exception): + pass Added: pypy/dist/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/zipimport/interp_zipimport.py Tue Oct 23 14:17:01 2007 @@ -0,0 +1,99 @@ + +from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable, \ + Arguments +from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.gateway import app2interp +from pypy.module.zipimport.rzipfile import is_zipfile +from pypy.interpreter.eval import Code +from pypy.interpreter.module import Module +import os +import stat + +class W_ZipImporter(Wrappable): + def __init__(self, space, name, w_dir, w_zipfile): + self.space = space + self.name = name + self.w_dir = w_dir + self.w_zipfile = w_zipfile + self.filelist_w = space.unpackiterable( + space.getattr(w_dir, space.wrap('filelist'))) + # XXX unicode? + self.namelist = [space.str_w(i) for i in space.unpackiterable( + space.call(space.getattr(w_dir, space.wrap('namelist')), + space.newlist([])))] + + def import_py_file(self, space, modname, filename, w_buf): + w = space.wrap + # XXX A bit of boilerplatish code stolen from importing.py, + # some refactoring to use common code base? + # all this error reporting and whatnot + w_mod = w(Module(space, w(modname))) + space.sys.setmodule(w_mod) + space.setattr(w_mod, w('__file__'), w(filename)) + space.setattr(w_mod, w('__doc__'), space.w_None) + w_mode = w("exec") + w_code = space.builtin.call('compile', w_buf, w(modname), w_mode) + pycode = space.interp_w(Code, w_code) + w_dict = space.getattr(w_mod, w('__dict__')) + space.call_method(w_dict, 'setdefault', + w('__builtins__'), + w(space.builtin)) + pycode.exec_code(space, w_dict, w_dict) + return w_mod + + def get_module(self, space, name): + w = space.wrap + try: + return space.call(space.getattr(self.w_dir, w('getinfo')), + space.newlist([w(name)])) + except OperationError, e: + if not e.match(space, space.w_KeyError): + # should never happen + raise e + + def find_module(self, space, import_name, w_path=None): + import_name = import_name.replace('.', os.path.sep) + for ext in ['.py', 'pyc', '.pyo']: + if self.get_module(space, import_name + ext): + return space.wrap(self) + find_module.unwrap_spec = ['self', ObjSpace, str, W_Root] + + def load_module(self, space, name): + w = space.wrap + filename = name.replace('.', os.path.sep) + '.py' + w_buf = space.call(space.getattr(self.w_dir, w('read')), + space.newlist([w(filename)])) + return self.import_py_file(space, name, filename, w_buf) + load_module.unwrap_spec = ['self', ObjSpace, str] + +def descr_new_zipimporter(space, w_type, name): + try: + s = os.stat(name) + except OSError: + return + if stat.S_ISDIR(s.st_mode): + return + w_import = space.builtin.get('__import__') + w_zipfile = space.call(w_import, space.newlist([ + space.wrap('zipfile'), + space.newdict(), + space.newdict(), + space.newlist([])])) + w_ZipFile = space.getattr(w_zipfile, space.wrap('ZipFile')) + try: + w_dir = space.call(w_ZipFile, space.newlist([space.wrap(name)])) + except OperationError: # we catch everything as this function + # should not raise + return None + return space.wrap(W_ZipImporter(space, name, w_dir, w_zipfile)) + +descr_new_zipimporter.unwrap_spec = [ObjSpace, W_Root, str] + +W_ZipImporter.typedef = TypeDef( + 'zipimporter', + __new__ = interp2app(descr_new_zipimporter), + find_module = interp2app(W_ZipImporter.find_module), + load_module = interp2app(W_ZipImporter.load_module), +) Added: pypy/dist/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/zipimport/test/test_zipimport.py Tue Oct 23 14:17:01 2007 @@ -0,0 +1,64 @@ + +import imp +from pypy.conftest import gettestobjspace +import marshal +import py +import time +import struct + +from pypy.tool.udir import udir +from zipfile import ZIP_STORED, ZIP_DEFLATED, ZipInfo + +class AppTestZipimport: + """ A bit structurized tests stolen and adapted from + cpy's regression tests + """ + def make_pyc(cls, co, mtime): + data = marshal.dumps(co) + if type(mtime) is type(0.0): + # Mac mtimes need a bit of special casing + if mtime < 0x7fffffff: + mtime = int(mtime) + else: + mtime = int(-0x100000000L + long(mtime)) + pyc = imp.get_magic() + struct.pack(" Author: fijal Date: Tue Oct 23 14:25:44 2007 New Revision: 47725 Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py Log: Oops Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/interp_zipimport.py Tue Oct 23 14:25:44 2007 @@ -5,7 +5,6 @@ from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import app2interp -from pypy.module.zipimport.rzipfile import is_zipfile from pypy.interpreter.eval import Code from pypy.interpreter.module import Module import os From arigo at codespeak.net Tue Oct 23 19:29:01 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 23 Oct 2007 19:29:01 +0200 (CEST) Subject: [pypy-svn] r47792 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071023172901.E5D1680FD@code0.codespeak.net> Author: arigo Date: Tue Oct 23 19:29:00 2007 New Revision: 47792 Added: pypy/dist/pypy/lang/smalltalk/ - copied from r47791, user/arigo/Misc/PySqueak/pypy/lang/smalltalk/ Log: Today's work on Smalltalk was done in a private repository because codespeak was down. Move it (with history) into its proper place. From tismer at codespeak.net Tue Oct 23 22:33:09 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Tue, 23 Oct 2007 22:33:09 +0200 (CEST) Subject: [pypy-svn] r47794 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071023203309.D8DA280FE@code0.codespeak.net> Author: tismer Date: Tue Oct 23 22:33:08 2007 New Revision: 47794 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: changed my info a little. Because Raymond is coming, I left departure open, some day after the 26th Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Tue Oct 23 22:33:08 2007 @@ -13,7 +13,7 @@ Anders Chrigstrom lives there Jacob Hallen lives there Laura Creighton lives there -Christian Tismer 18-26 Laura and Jacob's +Christian Tismer 19- after 26 Laura and Jacob's Maciej Fijalkowski 19-26 Laura and Jacob's Armin Rigo lives thereish ==================== ============== ===================== From cfbolz at codespeak.net Tue Oct 23 23:18:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 23 Oct 2007 23:18:22 +0200 (CEST) Subject: [pypy-svn] r47795 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071023211822.A93CC80F4@code0.codespeak.net> Author: cfbolz Date: Tue Oct 23 23:18:22 2007 New Revision: 47795 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py Log: (akuhn) strange, even though tagged pointers are not yet taken into account, no errors?! added intermediate generic object Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Tue Oct 23 23:18:22 2007 @@ -1,7 +1,12 @@ import py import struct -class Reader(object): +# ____________________________________________________________ +# +# Reads an image file and created all model objects + +class Stream(object): + """ Simple input stream """ def __init__(self, inputfile): try: self.data = inputfile.read() @@ -41,29 +46,42 @@ class CorruptImageError(Exception): pass + +# ____________________________________________________________ class ImageReader(object): - def __init__(self, reader): - self.reader = reader + def __init__(self, stream): + self.stream = stream + + def initialize(self): + self.read_header() + self.read_body() + self.init_specialobjectdumps() + self.init_compactclassdumps() + self.init_genericobjects() + + def init_genericobjects(self): + for dump in self.pointer2dump.itervalues(): + dump.as_g_object(self) - def readheader(self): - version = self.reader.next() + def read_header(self): + version = self.stream.next() if version != 0x1966: raise NotImplementedError - self.headersize = self.reader.next() - self.endofmemory = self.reader.next() - self.oldbaseaddress = self.reader.next() - self.specialobjectspointer = self.reader.next() - lasthash = self.reader.next() - savedwindowssize = self.reader.next() - fullscreenflag = self.reader.next() - extravmmemory = self.reader.next() - self.reader.skipbytes(self.headersize - (9 * 4)) + self.headersize = self.stream.next() + self.endofmemory = self.stream.next() + self.oldbaseaddress = self.stream.next() + self.specialobjectspointer = self.stream.next() + lasthash = self.stream.next() + savedwindowssize = self.stream.next() + fullscreenflag = self.stream.next() + extravmmemory = self.stream.next() + self.stream.skipbytes(self.headersize - (9 * 4)) - def readbody(self): + def read_body(self): dumps = [] self.pointer2dump = {} - while self.reader.pos <= self.endofmemory: - dump = self.readobject() + while self.stream.pos <= self.endofmemory: + dump = self.read_object() dumps.append(dump) self.pointer2dump[dump.pos - self.headersize + self.oldbaseaddress] = dump return dumps @@ -80,59 +98,85 @@ assert dump.format == 2 self.ccdumps = [self.pointer2dump[pointer] for pointer in dump.data] - def mark_classdescription(self): - #XXX quite some assumptions here - for dump in self.pointer2dump.itervalues(): - if dump.compact: - classdump = self.ccdumps[dump.classid] - else: - classdump = self.pointer2dump[dump.classid] - classdump.classdescription = True - def init_actualobjects(self): for dump in self.pointer2dump.itervalues(): dump.get_actual() # initialization - def readobject(self): - kind, = splitbits(self.reader.peek(), [2]) - if kind == 0: # 00r2 - dump = self.read3wordobjectheader() - elif kind == 1: # 01r2 - dump = self.read2wordobjectheader() - elif kind == 3: # 11r2 - dump = self.read1wordobjectheader() - else: + def read_object(self): + kind = self.stream.peek() & 3 # 2 bits + if kind == 0: # 00 bits + dump = self.read_3wordobjectheader() + elif kind == 1: # 01 bits + dump = self.read_2wordobjectheader() + elif kind == 3: # 11 bits + dump = self.read_1wordobjectheader() + else: # 10 bits raise CorruptImageError("Unused block not allowed in image") size = dump.size - dump.data = [self.reader.next() + dump.data = [self.stream.next() for _ in range(size - 1)] #size-1, excluding header return dump - def read1wordobjectheader(self): + def read_1wordobjectheader(self): kind, size, format, classid, idhash = ( - splitbits(self.reader.next(), [2,6,4,5,12])) + splitbits(self.stream.next(), [2,6,4,5,12])) assert kind == 3 - return ObjectDump(size, format, classid, idhash, self.reader.pos - 4, + return ObjectDump(size, format, classid, idhash, self.stream.pos - 4, compact = True) - def read2wordobjectheader(self): - assert splitbits(self.reader.peek(), [2])[0] == 1 #kind - classid = self.reader.next() - 1 # remove headertype to get pointer - kind, size, format, _, idhash = splitbits(self.reader.next(), [2,6,4,5,12]) + def read_2wordobjectheader(self): + assert splitbits(self.stream.peek(), [2])[0] == 1 #kind + classid = self.stream.next() - 1 # remove headertype to get pointer + kind, size, format, _, idhash = splitbits(self.stream.next(), [2,6,4,5,12]) assert kind == 1 - return ObjectDump(size, format, classid, idhash, self.reader.pos - 4) + return ObjectDump(size, format, classid, idhash, self.stream.pos - 4) - def read3wordobjectheader(self): - kind, size = splitbits(self.reader.next(), [2,30]) + def read_3wordobjectheader(self): + kind, size = splitbits(self.stream.next(), [2,30]) assert kind == 0 - assert splitbits(self.reader.peek(), [2])[0] == 0 #kind - classid = self.reader.next() - 0 # remove headertype to get pointer - kind, _, format, _, idhash = splitbits(self.reader.next(), [2,6,4,5,12]) + assert splitbits(self.stream.peek(), [2])[0] == 0 #kind + classid = self.stream.next() - 0 # remove headertype to get pointer + kind, _, format, _, idhash = splitbits(self.stream.next(), [2,6,4,5,12]) assert kind == 0 - return ObjectDump(size, format, classid, idhash, self.reader.pos - 4) + return ObjectDump(size, format, classid, idhash, self.stream.pos - 4) COMPACT_CLASSES_ARRAY = 28 +class GenericObject(object): + def __init__(self): + self.owner = None + + def isinitialized(self): + return self.owner is not None + + def initialize(self, dump, reader): + self.owner = reader + self.size = dump.size + self.hash12 = dump.idhash + self.format = dump.format + self.init_class(dump) + self.init_data(dump) + + def init_class(self, dump): + if dump.compact: + self.g_class = self.owner.ccdumps[dump.classid].as_g_object(self.owner) + else: + self.g_class = self.owner.pointer2dump[dump.classid].as_g_object(self.owner) + + def isbytes(self): + return 8 <= self.format <= 11 + + def iswords(self): + return self.format == 6 + + def ispointers(self): + return self.format < 8 #TODO, what about compiled methods? + + def init_data(self, dump): + if not self.ispointers(): return + self.data = [self.owner.pointer2dump[p].as_g_object(self.owner) + for p in dump.data] + class ObjectDump(object): def __init__(self, size, format, classid, idhash, pos, compact = False): self.pos = pos @@ -144,10 +188,16 @@ self.classdescription = False self.actual = None self.compact = compact + self.g_object = GenericObject() def __eq__(self, other): "(for testing)" - return self.__class__ is other.__class__ and self.__dict__ == other.__dict__ + return (self.__class__ is other.__class__ and + self.pos == other.pos and + self.format == other.format and + self.classid == other.classid and + self.idhash == other.idhash and + self.compact == other.compact) def __ne__(self, other): "(for testing)" @@ -158,6 +208,11 @@ self.actual = self.create_actual() return self.actual + def as_g_object(self, reader): + if self.g_object.isinitialized(): + self.g_object.initialize(self, reader) + return self.g_object + def create_actual(self): from pypy.lang.smalltalk import model if self.classdescription: Modified: pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py Tue Oct 23 23:18:22 2007 @@ -9,7 +9,7 @@ def test_reader(): f = StringIO.StringIO('\x00\x00\x19\x66') - reader = squeakimage.Reader(f) + reader = squeakimage.Stream(f) first = reader.peek() assert first == 6502 first = reader.next() @@ -18,7 +18,7 @@ def test_swap(): f = StringIO.StringIO('\x66\x19\x00\x00') - reader = squeakimage.Reader(f) + reader = squeakimage.Stream(f) reader.swap = True first = reader.next() assert first == 6502 @@ -26,7 +26,7 @@ def test_many(): f = StringIO.StringIO('\x00\x00\x19\x66' * 5) - reader = squeakimage.Reader(f) + reader = squeakimage.Stream(f) for each in range(5): first = reader.peek() assert first == 6502 @@ -36,7 +36,7 @@ def test_skipbytes(): f = StringIO.StringIO('\xFF\xFF\xFF\x00\x00\x19\x66') - reader = squeakimage.Reader(f) + reader = squeakimage.Stream(f) reader.skipbytes(3) value = reader.next() assert value == 6502 @@ -49,9 +49,9 @@ assert [1,3,7,15] == squeakimage.splitbits(0xFfFfFfFf, [1,2,3,4]) def test_readheader(): - reader = squeakimage.Reader(filepath.open()) + reader = squeakimage.Stream(filepath.open()) ireader = squeakimage.ImageReader(reader) - ireader.readheader() + ireader.read_header() assert ireader.endofmemory == 0x93174 assert ireader.oldbaseaddress == 0x6649000 assert ireader.specialobjectspointer == 0x6668380 @@ -59,16 +59,16 @@ assert next != 0 #expects object header, which can not be 0x0 def test_readheader_and_body(): - reader = squeakimage.Reader(filepath.open()) + reader = squeakimage.Stream(filepath.open()) ireader = squeakimage.ImageReader(reader) - ireader.readheader() - objects = ireader.readbody() + ireader.read_header() + objects = ireader.read_body() assert len(objects) > 0 assert 15000 < len(objects) < 16000 def imagereader_mock(string): f = StringIO.StringIO(string) - reader = squeakimage.Reader(f) + reader = squeakimage.Stream(f) return squeakimage.ImageReader(reader) def joinbits(values, lengths): @@ -94,47 +94,52 @@ def test_freeblock(): r = imagereader_mock("\x00\x00\x00\x02") - py.test.raises(squeakimage.CorruptImageError, lambda: r.readobject()) + py.test.raises(squeakimage.CorruptImageError, lambda: r.read_object()) def test_1wordobjectheader(): s = ints2str(joinbits([3, 1, 2, 3, 4], [2,6,4,5,12])) r = imagereader_mock(s) - assert squeakimage.ObjectDump(1, 2, 3, 4, 0, True) == r.read1wordobjectheader() + assert squeakimage.ObjectDump(1, 2, 3, 4, 0, True) == r.read_1wordobjectheader() def test_1wordobjectheader2(): s = ints2str(joinbits([3, 1, 2, 3, 4], [2,6,4,5,12])) r = imagereader_mock(s * 3) - assert squeakimage.ObjectDump(1, 2, 3, 4, 0, True) == r.read1wordobjectheader() - assert squeakimage.ObjectDump(1, 2, 3, 4, 4, True) == r.read1wordobjectheader() - assert squeakimage.ObjectDump(1, 2, 3, 4, 8, True) == r.read1wordobjectheader() + assert squeakimage.ObjectDump(1, 2, 3, 4, 0, True) == r.read_1wordobjectheader() + assert squeakimage.ObjectDump(1, 2, 3, 4, 4, True) == r.read_1wordobjectheader() + assert squeakimage.ObjectDump(1, 2, 3, 4, 8, True) == r.read_1wordobjectheader() def test_2wordobjectheader(): s = ints2str(4200 + 1, joinbits([1, 1, 2, 3, 4], [2,6,4,5,12])) r = imagereader_mock(s) - assert squeakimage.ObjectDump(1, 2, 4200, 4, 4) == r.read2wordobjectheader() + assert squeakimage.ObjectDump(1, 2, 4200, 4, 4) == r.read_2wordobjectheader() def test_3wordobjectheader(): s = ints2str(1701 << 2, 4200 + 0, joinbits([0, 1, 2, 3, 4], [2,6,4,5,12])) r = imagereader_mock(s) - assert squeakimage.ObjectDump(1701, 2, 4200, 4, 8) == r.read3wordobjectheader() + assert squeakimage.ObjectDump(1701, 2, 4200, 4, 8) == r.read_3wordobjectheader() def test_read3wordheaderobject(): size = 42 s = ints2str(size << 2, 4200 + 0, joinbits([0, 1, 2, 3, 4], [2,6,4,5,12])) r = imagereader_mock(s + '\x00\x00\x19\x66' * (size - 1)) - dump = r.readobject() + dump = r.read_object() dump0 = squeakimage.ObjectDump(size, 2, 4200, 4, 8) dump0.data = [6502] * (size - 1) assert dump0 == dump +def test_smoketest0(): + reader = squeakimage.Stream(filepath.open()) + ireader = squeakimage.ImageReader(reader) + ireader.initialize() + + def test_smoketest(): - reader = squeakimage.Reader(filepath.open()) + reader = squeakimage.Stream(filepath.open()) ireader = squeakimage.ImageReader(reader) - ireader.readheader() - ireader.readbody() + ireader.read_header() + ireader.read_body() ireader.init_specialobjectdumps() ireader.init_compactclassdumps() - ireader.mark_classdescription() #ireader.init_actualobjects() \ No newline at end of file From arigo at codespeak.net Wed Oct 24 10:41:07 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 24 Oct 2007 10:41:07 +0200 (CEST) Subject: [pypy-svn] r47797 - pypy/dist/pypy/translator/c Message-ID: <20071024084107.D1E9D80F2@code0.codespeak.net> Author: arigo Date: Wed Oct 24 10:41:06 2007 New Revision: 47797 Modified: pypy/dist/pypy/translator/c/funcgen.py Log: Support for debug_print(boolean). Modified: pypy/dist/pypy/translator/c/funcgen.py ============================================================================== --- pypy/dist/pypy/translator/c/funcgen.py (original) +++ pypy/dist/pypy/translator/c/funcgen.py Wed Oct 24 10:41:06 2007 @@ -688,6 +688,10 @@ format.append(arg.value.replace('%', '%%')) continue format.append('%c') + elif T == Bool: + format.append('%s') + argv.append('(%s) ? "True" : "False"' % self.expr(arg)) + continue else: raise Exception("don't know how to debug_print %r" % (T,)) argv.append(self.expr(arg)) From arigo at codespeak.net Wed Oct 24 10:41:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 24 Oct 2007 10:41:37 +0200 (CEST) Subject: [pypy-svn] r47798 - pypy/dist/pypy/translator/backendopt Message-ID: <20071024084137.1CFDF80F2@code0.codespeak.net> Author: arigo Date: Wed Oct 24 10:41:36 2007 New Revision: 47798 Modified: pypy/dist/pypy/translator/backendopt/constfold.py Log: Check tryfold before trying to constant-fold. This avoids constant-folding debug_print... Modified: pypy/dist/pypy/translator/backendopt/constfold.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/constfold.py (original) +++ pypy/dist/pypy/translator/backendopt/constfold.py Wed Oct 24 10:41:36 2007 @@ -31,7 +31,7 @@ except AttributeError: sideeffects = True else: - if len(args) == len(vargs): + if op.tryfold and len(args) == len(vargs): RESTYPE = spaceop.result.concretetype try: result = op(RESTYPE, *args) From akuhn at codespeak.net Wed Oct 24 10:42:39 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Wed, 24 Oct 2007 10:42:39 +0200 (CEST) Subject: [pypy-svn] r47799 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024084239.844FD80F2@code0.codespeak.net> Author: akuhn Date: Wed Oct 24 10:42:39 2007 New Revision: 47799 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py Log: try to commit Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Wed Oct 24 10:42:39 2007 @@ -70,6 +70,29 @@ return len(self.words) class W_CompiledMethod(W_Object): + """My instances are methods suitable for interpretation by the virtual machine. This is the only class in the system whose instances intermix both indexable pointer fields and indexable integer fields. + + + The current format of a CompiledMethod is as follows: + + header (4 bytes) + literals (4 bytes each) + bytecodes (variable) + trailer (variable) + + The header is a 30-bit integer with the following format: + + (index 0) 9 bits: main part of primitive number (#primitive) + (index 9) 8 bits: number of literals (#numLiterals) + (index 17) 1 bit: whether a large frame size is needed (#frameSize) + (index 18) 6 bits: number of temporary variables (#numTemps) + (index 24) 4 bits: number of arguments to the method (#numArgs) + (index 28) 1 bit: high-bit of primitive number (#primitive) + (index 29) 1 bit: flag bit, ignored by the VM (#flag) + + + The trailer has two variant formats. In the first variant, the last byte is at least 252 and the last four bytes represent a source pointer into one of the sources files (see #sourcePointer). In the second variant, the last byte is less than 252, and the last several bytes are a compressed version of the names of the method's temporary variables. The number of bytes used for this purpose is the value of the last byte in the method. + """ def __init__(self, w_class, size, bytes="", argsize=0, tempsize=0, primitive=0): W_Object.__init__(self, w_class) Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Wed Oct 24 10:42:39 2007 @@ -14,6 +14,7 @@ inputfile.close() self.swap = False self.pos = 0 + self.count = 0 def peek(self): if self.pos >= len(self.data): @@ -28,12 +29,17 @@ def next(self): integer = self.peek() self.pos += 4 + self.count += 4 return integer + def reset_count(self): + self.count = 0 + def skipbytes(self, jump): assert jump > 0 assert (self.pos + jump) <= len(self.data) - self.pos += jump + self.pos += jump + self.count += jump def splitbits(integer, lengths): assert sum(lengths) <= 32 @@ -56,18 +62,17 @@ def initialize(self): self.read_header() self.read_body() - self.init_specialobjectdumps() - self.init_compactclassdumps() - self.init_genericobjects() - - def init_genericobjects(self): - for dump in self.pointer2dump.itervalues(): - dump.as_g_object(self) + self.init_compactclassesarray() + self.init_g_objects() + + def init_g_objects(self): + for chunk in self.chunks.itervalues(): + chunk.as_g_object(self) def read_header(self): version = self.stream.next() if version != 0x1966: raise NotImplementedError - self.headersize = self.stream.next() + headersize = self.stream.next() self.endofmemory = self.stream.next() self.oldbaseaddress = self.stream.next() self.specialobjectspointer = self.stream.next() @@ -75,61 +80,56 @@ savedwindowssize = self.stream.next() fullscreenflag = self.stream.next() extravmmemory = self.stream.next() - self.stream.skipbytes(self.headersize - (9 * 4)) + self.stream.skipbytes(headersize - (9 * 4)) def read_body(self): - dumps = [] - self.pointer2dump = {} - while self.stream.pos <= self.endofmemory: - dump = self.read_object() - dumps.append(dump) - self.pointer2dump[dump.pos - self.headersize + self.oldbaseaddress] = dump - return dumps - - def init_specialobjectdumps(self): - dump = self.pointer2dump[self.specialobjectspointer] - assert dump.size > 24 #and more - assert dump.format == 2 - self.sodumps = [self.pointer2dump[pointer] for pointer in dump.data] - - def init_compactclassdumps(self): - dump = self.sodumps[COMPACT_CLASSES_ARRAY] - assert len(dump.data) == 31 - assert dump.format == 2 - self.ccdumps = [self.pointer2dump[pointer] for pointer in dump.data] + self.chunks = {} + self.stream.reset_count() + while self.stream.count < self.endofmemory: + chunk, pos = self.read_object() + self.chunks[pos + self.oldbaseaddress] = chunk + return self.chunks.values() + + def init_compactclassesarray(self): + special = self.chunks[self.specialobjectspointer] + assert special.size > 24 #at least + assert special.format == 2 + chunk = self.chunks[special.data[COMPACT_CLASSES_ARRAY]] + assert len(chunk.data) == 31 + assert chunk.format == 2 + self.compactclasses = [self.chunks[pointer] for pointer in chunk.data] def init_actualobjects(self): - for dump in self.pointer2dump.itervalues(): - dump.get_actual() # initialization + for chunk in self.chunks.itervalues(): + chunk.get_actual() # initialization def read_object(self): kind = self.stream.peek() & 3 # 2 bits if kind == 0: # 00 bits - dump = self.read_3wordobjectheader() + chunk, pos = self.read_3wordobjectheader() elif kind == 1: # 01 bits - dump = self.read_2wordobjectheader() + chunk, pos = self.read_2wordobjectheader() elif kind == 3: # 11 bits - dump = self.read_1wordobjectheader() + chunk, pos = self.read_1wordobjectheader() else: # 10 bits raise CorruptImageError("Unused block not allowed in image") - size = dump.size - dump.data = [self.stream.next() + size = chunk.size + chunk.data = [self.stream.next() for _ in range(size - 1)] #size-1, excluding header - return dump + return chunk, pos def read_1wordobjectheader(self): kind, size, format, classid, idhash = ( splitbits(self.stream.next(), [2,6,4,5,12])) assert kind == 3 - return ObjectDump(size, format, classid, idhash, self.stream.pos - 4, - compact = True) + return ImageChunk(size, format, classid, idhash), self.stream.count - 4 def read_2wordobjectheader(self): assert splitbits(self.stream.peek(), [2])[0] == 1 #kind classid = self.stream.next() - 1 # remove headertype to get pointer kind, size, format, _, idhash = splitbits(self.stream.next(), [2,6,4,5,12]) assert kind == 1 - return ObjectDump(size, format, classid, idhash, self.stream.pos - 4) + return ImageChunk(size, format, classid, idhash), self.stream.count - 4 def read_3wordobjectheader(self): kind, size = splitbits(self.stream.next(), [2,30]) @@ -138,30 +138,56 @@ classid = self.stream.next() - 0 # remove headertype to get pointer kind, _, format, _, idhash = splitbits(self.stream.next(), [2,6,4,5,12]) assert kind == 0 - return ObjectDump(size, format, classid, idhash, self.stream.pos - 4) + return ImageChunk(size, format, classid, idhash), self.stream.count - 4 COMPACT_CLASSES_ARRAY = 28 +# ____________________________________________________________ + class GenericObject(object): + """ Intermediate representation of squeak objects. To establish all + pointers as object references, ImageReader creates instances of + GenericObject from the image chunks, and uses them as starting + point for the actual create of pypy.lang.smalltalk.model classes. + """ def __init__(self): self.owner = None def isinitialized(self): return self.owner is not None - def initialize(self, dump, reader): + def initialize_int(self, value, reader): self.owner = reader - self.size = dump.size - self.hash12 = dump.idhash - self.format = dump.format - self.init_class(dump) - self.init_data(dump) - - def init_class(self, dump): - if dump.compact: - self.g_class = self.owner.ccdumps[dump.classid].as_g_object(self.owner) + self.value = value + self.size = -1 + + def initialize(self, chunk, reader): + self.owner = reader + self.size = chunk.size + self.hash12 = chunk.hash12 + self.format = chunk.format + self.init_class(chunk) + self.init_data(chunk) + self.w_object = None + + def init_class(self, chunk): + if chunk.iscompact(): + self.g_class = self.owner.compactclasses[chunk.classid].g_object + else: + self.g_class = self.owner.chunks[chunk.classid].g_object + + def init_data(self, chunk): + if not self.ispointers(): return + self.pointers = [] + for pointer in chunk.data: + g_object = self.decode_pointer(pointer) + self.pointers.append(g_object) + + def decode_pointer(self, pointer): + if (pointer & 1) == 1: + return GenericObject().initialize_int(pointer >> 1, self.owner) else: - self.g_class = self.owner.pointer2dump[dump.classid].as_g_object(self.owner) + return self.owner.chunks[pointer].g_object def isbytes(self): return 8 <= self.format <= 11 @@ -170,54 +196,13 @@ return self.format == 6 def ispointers(self): - return self.format < 8 #TODO, what about compiled methods? - - def init_data(self, dump): - if not self.ispointers(): return - self.data = [self.owner.pointer2dump[p].as_g_object(self.owner) - for p in dump.data] + return self.format < 5 #TODO, what about compiled methods? -class ObjectDump(object): - def __init__(self, size, format, classid, idhash, pos, compact = False): - self.pos = pos - self.size = size - self.format = format - self.classid = classid - self.idhash = idhash - self.data = None - self.classdescription = False - self.actual = None - self.compact = compact - self.g_object = GenericObject() - - def __eq__(self, other): - "(for testing)" - return (self.__class__ is other.__class__ and - self.pos == other.pos and - self.format == other.format and - self.classid == other.classid and - self.idhash == other.idhash and - self.compact == other.compact) - - def __ne__(self, other): - "(for testing)" - return not self == other - - def get_actual(self): - if self.actual is None: - self.actual = self.create_actual() - return self.actual - - def as_g_object(self, reader): - if self.g_object.isinitialized(): - self.g_object.initialize(self, reader) - return self.g_object - - def create_actual(self): + def init_w_object(self): from pypy.lang.smalltalk import model - if self.classdescription: - print self.format - return None + #if self.classdescription: + # print self.format + # return None if self.format == 0: # no instvars, non-indexed assert self.size == 0 return model.W_PointersObject(size = 0) @@ -244,5 +229,35 @@ else: assert 0, "not reachable" + + +class ImageChunk(object): + def __init__(self, size, format, classid, hash12): + self.size = size + self.format = format + self.classid = classid + self.hash12 = hash12 + self.data = None + self.g_object = GenericObject() + + def __eq__(self, other): + "(for testing)" + return (self.__class__ is other.__class__ and + self.format == other.format and + self.classid == other.classid and + self.hash12 == other.hash12 and + self.data == other.data) + + def __ne__(self, other): + "(for testing)" + return not self == other + + def as_g_object(self, reader): + if not self.g_object.isinitialized(): + self.g_object.initialize(self, reader) + return self.g_object + + def iscompact(self): + return 0 < self.classid < 32 Modified: pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py Wed Oct 24 10:42:39 2007 @@ -1,92 +1,90 @@ import py -import StringIO from pypy.lang.smalltalk import squeakimage -filepath = py.magic.autopath().dirpath().dirpath().join('mini.image') +# ----- helpers ---------------------------------------------- -def test_miniimageexists(): - assert filepath.check(dir=False) +def ints2str(*ints): + import struct + return struct.pack(">" + "i" * len(ints), *ints) -def test_reader(): - f = StringIO.StringIO('\x00\x00\x19\x66') - reader = squeakimage.Stream(f) - first = reader.peek() - assert first == 6502 - first = reader.next() - assert first == 6502 - py.test.raises(IndexError, lambda: reader.next()) - -def test_swap(): - f = StringIO.StringIO('\x66\x19\x00\x00') - reader = squeakimage.Stream(f) - reader.swap = True - first = reader.next() +def joinbits(values, lengths): + result = 0 + for each, length in reversed(zip(values, lengths)): + result = result << length + result += each + return result + +def imagereader_mock(string): + import StringIO + f = StringIO.StringIO(string) + stream = squeakimage.Stream(f) + return squeakimage.ImageReader(stream) + + +# ----- tests ------------------------------------------------ + +def test_stream(): + stream = imagereader_mock('\x00\x00\x19\x66').stream + n = stream.peek() + assert n == 6502 + n = stream.next() + assert n == 6502 + py.test.raises(IndexError, lambda: stream.next()) + +def test_stream_swap(): + stream = imagereader_mock('\x66\x19\x00\x00').stream + stream.swap = True + first = stream.next() assert first == 6502 - py.test.raises(IndexError, lambda: reader.next()) + py.test.raises(IndexError, lambda: stream.next()) -def test_many(): - f = StringIO.StringIO('\x00\x00\x19\x66' * 5) - reader = squeakimage.Stream(f) +def test_stream_many(): + stream = imagereader_mock('\x00\x00\x19\x66' * 5).stream for each in range(5): - first = reader.peek() + first = stream.peek() assert first == 6502 - value = reader.next() + value = stream.next() assert value == 6502 - py.test.raises(IndexError, lambda: reader.next()) + py.test.raises(IndexError, lambda: stream.next()) -def test_skipbytes(): - f = StringIO.StringIO('\xFF\xFF\xFF\x00\x00\x19\x66') - reader = squeakimage.Stream(f) - reader.skipbytes(3) - value = reader.next() +def test_stream_skipbytes(): + stream = imagereader_mock('\xFF\xFF\xFF\x00\x00\x19\x66').stream + stream.skipbytes(3) + value = stream.next() assert value == 6502 - py.test.raises(IndexError, lambda: reader.next()) + py.test.raises(IndexError, lambda: stream.next()) -def test_splitbits(): +def test_stream_count(): + stream = imagereader_mock('\xFF' * 20).stream + stream.next() + stream.next() + stream.reset_count() + assert stream.count == 0 + stream.next() + assert stream.count == 4 + stream.next() + assert stream.count == 8 + +def test_simple_splitbits(): assert ([1] * 4) == squeakimage.splitbits(0x01010101, [8,8,8,8]) assert ([255] * 4) == squeakimage.splitbits(0xFfFfFfFf, [8,8,8,8]) + +def test_fancy_splitbits(): assert [4,3,2,1] == squeakimage.splitbits(0x01020304, [8,8,8,8]) assert [1,3,7,15] == squeakimage.splitbits(0xFfFfFfFf, [1,2,3,4]) -def test_readheader(): - reader = squeakimage.Stream(filepath.open()) - ireader = squeakimage.ImageReader(reader) - ireader.read_header() - assert ireader.endofmemory == 0x93174 - assert ireader.oldbaseaddress == 0x6649000 - assert ireader.specialobjectspointer == 0x6668380 - next = reader.next() - assert next != 0 #expects object header, which can not be 0x0 - -def test_readheader_and_body(): - reader = squeakimage.Stream(filepath.open()) - ireader = squeakimage.ImageReader(reader) - ireader.read_header() - objects = ireader.read_body() - assert len(objects) > 0 - assert 15000 < len(objects) < 16000 - -def imagereader_mock(string): - f = StringIO.StringIO(string) - reader = squeakimage.Stream(f) - return squeakimage.ImageReader(reader) - -def joinbits(values, lengths): - result = 0 - for each, length in reversed(zip(values, lengths)): - result = result << length - result += each - return result +def test_format_splitbits(): + x = 0xAA + assert [x & 3] == squeakimage.splitbits(x, [2]) -def test_joinbits(): +def test_simple_joinbits(): assert 0x01010101 == joinbits(([1] * 4), [8,8,8,8]) assert 0xFfFfFfFf == joinbits([255] * 4, [8,8,8,8]) + +def test_fancy_joinbits(): assert 0x01020304 == joinbits([4,3,2,1], [8,8,8,8]) assert 0x3Ff == joinbits([1,3,7,15], [1,2,3,4]) -def ints2str(*ints): - import struct - return struct.pack(">" + "i" * len(ints), *ints) def test_ints2str(): assert "\x00\x00\x00\x02" == ints2str(2) @@ -99,47 +97,75 @@ def test_1wordobjectheader(): s = ints2str(joinbits([3, 1, 2, 3, 4], [2,6,4,5,12])) r = imagereader_mock(s) - assert squeakimage.ObjectDump(1, 2, 3, 4, 0, True) == r.read_1wordobjectheader() + assert (squeakimage.ImageChunk(1, 2, 3, 4), 0) == r.read_1wordobjectheader() def test_1wordobjectheader2(): s = ints2str(joinbits([3, 1, 2, 3, 4], [2,6,4,5,12])) r = imagereader_mock(s * 3) - assert squeakimage.ObjectDump(1, 2, 3, 4, 0, True) == r.read_1wordobjectheader() - assert squeakimage.ObjectDump(1, 2, 3, 4, 4, True) == r.read_1wordobjectheader() - assert squeakimage.ObjectDump(1, 2, 3, 4, 8, True) == r.read_1wordobjectheader() + assert (squeakimage.ImageChunk(1, 2, 3, 4), 0) == r.read_1wordobjectheader() + assert (squeakimage.ImageChunk(1, 2, 3, 4), 4) == r.read_1wordobjectheader() + assert (squeakimage.ImageChunk(1, 2, 3, 4), 8) == r.read_1wordobjectheader() def test_2wordobjectheader(): s = ints2str(4200 + 1, joinbits([1, 1, 2, 3, 4], [2,6,4,5,12])) r = imagereader_mock(s) - assert squeakimage.ObjectDump(1, 2, 4200, 4, 4) == r.read_2wordobjectheader() + assert (squeakimage.ImageChunk(1, 2, 4200, 4), 4) == r.read_2wordobjectheader() def test_3wordobjectheader(): s = ints2str(1701 << 2, 4200 + 0, joinbits([0, 1, 2, 3, 4], [2,6,4,5,12])) r = imagereader_mock(s) - assert squeakimage.ObjectDump(1701, 2, 4200, 4, 8) == r.read_3wordobjectheader() + assert (squeakimage.ImageChunk(1701, 2, 4200, 4), 8) == r.read_3wordobjectheader() def test_read3wordheaderobject(): size = 42 s = ints2str(size << 2, 4200 + 0, joinbits([0, 1, 2, 3, 4], [2,6,4,5,12])) r = imagereader_mock(s + '\x00\x00\x19\x66' * (size - 1)) - dump = r.read_object() - dump0 = squeakimage.ObjectDump(size, 2, 4200, 4, 8) - dump0.data = [6502] * (size - 1) - assert dump0 == dump - -def test_smoketest0(): - reader = squeakimage.Stream(filepath.open()) - ireader = squeakimage.ImageReader(reader) - ireader.initialize() - - -def test_smoketest(): - reader = squeakimage.Stream(filepath.open()) - ireader = squeakimage.ImageReader(reader) - ireader.read_header() - ireader.read_body() - ireader.init_specialobjectdumps() - ireader.init_compactclassdumps() - #ireader.init_actualobjects() - + chunk, pos = r.read_object() + chunk0 = squeakimage.ImageChunk(size, 2, 4200, 4) + chunk0.data = [6502] * (size - 1) + assert pos == 8 + assert chunk0 == chunk + +# ----- mini.image productline ------------------------------- +# relying on order of methods + +mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') + +def test_miniimageexists(): + assert mini_image.check(dir=False) + +example = squeakimage.ImageReader(squeakimage.Stream(mini_image.open())) + +def test_read_header(): + example.read_header() + assert example.endofmemory == 0x93174 + assert example.oldbaseaddress == 0x6649000 + assert example.specialobjectspointer == 0x6668380 + +def test_read_all_header(): + next = example.stream.peek() + assert next != 0 #expects object header, which must not be 0x00000000 + +def test_read_body(): + objects = example.read_body() + assert len(objects) > 0 + assert 15000 < len(objects) < 16000 + +def test_read_all_body(): + py.test.raises(IndexError, lambda: example.stream.next()) + +def test_all_pointers_valid(): + for each in example.chunks.itervalues(): + if each.format < 5: + for pointer in each.data: + if (pointer & 1) != 1: + assert pointer in example.chunks + +def test_init_compactclasses(): + example.init_compactclassesarray() + assert len(example.compactclasses) == 31 + +def test_init_g_objects(): + example.init_g_objects() + \ No newline at end of file From arigo at codespeak.net Wed Oct 24 10:59:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 24 Oct 2007 10:59:05 +0200 (CEST) Subject: [pypy-svn] r47800 - pypy/extradoc/sprintinfo/bern2007 Message-ID: <20071024085905.7F88080EB@code0.codespeak.net> Author: arigo Date: Wed Oct 24 10:59:05 2007 New Revision: 47800 Added: pypy/extradoc/sprintinfo/bern2007/planning.txt (contents, props changed) Log: Move this file here out of the private repository. Added: pypy/extradoc/sprintinfo/bern2007/planning.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/bern2007/planning.txt Wed Oct 24 10:59:05 2007 @@ -0,0 +1,34 @@ +People present +-------------- + +Carl Friedrich +Armin +Adrian K +Tudor +Lukas +Oscar +Adrian L +Toon +Niko + +Task ideas +---------- + +- Print the blue book (Adrian K) + +- Solve the svn problem (Armin) + +- Complete the simple model (a bit all) + +Afterwards: +----------- + +- Think about loading the image (Adrian K, Carl Friedrich) + +- interpreter loop and bytecode dispatch (Armin, Adrian L, + Lukas, Toon) + +- think about implementing numbered primitives (arithmetic) + (Niko, Tudor, Oscar) + + From akuhn at codespeak.net Wed Oct 24 11:13:21 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Wed, 24 Oct 2007 11:13:21 +0200 (CEST) Subject: [pypy-svn] r47801 - pypy/extradoc/sprintinfo/bern2007 Message-ID: <20071024091321.D3B868107@code0.codespeak.net> Author: akuhn Date: Wed Oct 24 11:13:21 2007 New Revision: 47801 Modified: pypy/extradoc/sprintinfo/bern2007/planning.txt Log: (all), planning for today Modified: pypy/extradoc/sprintinfo/bern2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/bern2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/bern2007/planning.txt Wed Oct 24 11:13:21 2007 @@ -4,31 +4,43 @@ Carl Friedrich Armin Adrian K -Tudor -Lukas -Oscar -Adrian L Toon -Niko +Niko (virtually) Task ideas ---------- -- Print the blue book (Adrian K) +- Print the blue book (Adrian K) DONE -- Solve the svn problem (Armin) +- Solve the svn problem (Armin) DONE -- Complete the simple model (a bit all) +- Complete the simple model (a bit all) DONE Afterwards: ----------- -- Think about loading the image (Adrian K, Carl Friedrich) +- Implement loading the image (Adrian K, Carl Friedrich) IN PROGRESS - interpreter loop and bytecode dispatch (Armin, Adrian L, - Lukas, Toon) + Lukas, Toon) DONE -- think about implementing numbered primitives (arithmetic) - (Niko, Tudor, Oscar) +- implementing numbered primitives (arithmetic) + (Niko, Tudor, Oscar) IN PROGRESS + - begin implementing the base image, including class objects + - templates for the math operations more or less complete, except for + promoting from small to large integer + - plenty of primitives to go :) (guess that's about it) + +- implement all the bytecodes (Armin, Toon, Lukas, Adrian L) IN PROGRESS + +- define a common interface for real image and mock image (Niko, Carl + Friedrich, Adrian K) + +- try to generate the fib example and fight with RPython :) + (Armin, together to teach the smalltalkers) + +- write the report for yesterday (Carl Friedrich, Adrian K) + +okay, done. From tverwaes at codespeak.net Wed Oct 24 11:21:50 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 11:21:50 +0200 (CEST) Subject: [pypy-svn] r47802 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024092150.076E98113@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 11:21:49 2007 New Revision: 47802 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (toon) added singleExtendedSendBytecode Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Wed Oct 24 11:21:49 2007 @@ -196,7 +196,10 @@ self.pop() def singleExtendedSendBytecode(self, interp): - raise MissingBytecode + descriptor = self.getByte() + selector = self.method.literals[descriptor & 31] + argcount = descriptor >> 5 + self._sendSelector(selector, argcount, interp) def doubleExtendedDoAnythingBytecode(self, interp): raise MissingBytecode Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Wed Oct 24 11:21:49 2007 @@ -403,3 +403,28 @@ assert interp.activeContext.stack == [interp.TRUE, interp.FALSE, interp.TRUE, interp.FALSE, interp.FALSE, interp.TRUE] + +def test_singleExtendedSendBytecode(): + w_class = model.W_Class(None, None) + w_object = w_class.new() + for bytecode, result in [ (returnReceiver, w_object), + (returnTrue, interpreter.Interpreter.TRUE), + (returnFalse, interpreter.Interpreter.FALSE), + (returnNil, interpreter.Interpreter.NIL), + (returnTopFromMethod, interpreter.Interpreter.ONE) ]: + w_class.methoddict["foo"] = model.W_CompiledMethod(None, 0, pushConstantOneBytecode + bytecode) + interp = new_interpreter(singleExtendedSendBytecode + chr(0)) + interp.activeContext.method.literals = ["foo"] + interp.activeContext.push(w_object) + callerContext = interp.activeContext + interp.step() + assert interp.activeContext.sender == callerContext + assert interp.activeContext.stack == [] + assert interp.activeContext.receiver == w_object + assert interp.activeContext.method == w_class.methoddict["foo"] + assert callerContext.stack == [] + interp.step() + interp.step() + assert interp.activeContext == callerContext + assert interp.activeContext.stack == [result] + From fijal at codespeak.net Wed Oct 24 11:22:40 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 24 Oct 2007 11:22:40 +0200 (CEST) Subject: [pypy-svn] r47803 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071024092240.16CB88107@code0.codespeak.net> Author: fijal Date: Wed Oct 24 11:22:39 2007 New Revision: 47803 Modified: pypy/dist/pypy/translator/llvm/test/test_lladdresses.py Log: Hopeless approach to fix llvm tests - tests are skipped anyway. Modified: pypy/dist/pypy/translator/llvm/test/test_lladdresses.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lladdresses.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lladdresses.py Wed Oct 24 11:22:39 2007 @@ -1,5 +1,5 @@ import py -from pypy.rpython.memory.lladdress import * +from pypy.rpython.lltypesystem.llmemory import * from pypy.annotation.model import SomeAddress, SomeChar from pypy.rlib.objectmodel import free_non_gc_object From tverwaes at codespeak.net Wed Oct 24 11:38:05 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 11:38:05 +0200 (CEST) Subject: [pypy-svn] r47804 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024093805.89F468102@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 11:38:04 2007 New Revision: 47804 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (toon) added singleExtendedSuperBytecode Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Wed Oct 24 11:38:04 2007 @@ -115,11 +115,21 @@ def sendLiteralSelectorBytecode(self, interp): selector = self.method.literals[self.currentBytecode & 15] argcount = ((self.currentBytecode >> 4) & 3) - 1 - self._sendSelector(selector, argcount, interp) + self._sendSelfSelector(selector, argcount, interp) - def _sendSelector(self, selector, argcount, interp): + def _sendSelfSelector(self, selector, argcount, interp): receiver = self.peek(argcount) - method = receiver.w_class.lookup(selector) + self._sendSelector(selector, argcount, interp, + receiver.w_class) + + def _sendSuperSelector(self, selector, argcount, interp): + receiver = self.peek(argcount) + self._sendSelector(selector, argcount, interp, + receiver.w_class.w_superclass) + + def _sendSelector(self, selector, argcount, interp, receiverclass): + receiver = self.peek(argcount) + method = receiverclass.lookup(selector) assert method if method.primitive: func = primitives.prim_table[method.primitive] @@ -195,17 +205,20 @@ self.extendedStoreBytecode(interp) self.pop() - def singleExtendedSendBytecode(self, interp): + def getExtendedSelectorArgcount(self): descriptor = self.getByte() - selector = self.method.literals[descriptor & 31] - argcount = descriptor >> 5 - self._sendSelector(selector, argcount, interp) + return (self.method.literals[descriptor & 31]), (descriptor >> 5) + + def singleExtendedSendBytecode(self, interp): + selector, argcount = self.getExtendedSelectorArgcount() + self._sendSelfSelector(selector, argcount, interp) def doubleExtendedDoAnythingBytecode(self, interp): raise MissingBytecode def singleExtendedSuperBytecode(self, interp): - raise MissingBytecode + selector, argcount = self.getExtendedSelectorArgcount() + self._sendSuperSelector(selector, argcount, interp) def secondExtendedSendBytecode(self, interp): raise MissingBytecode @@ -250,7 +263,7 @@ try: self.push(primitives.prim_table[primitive](self)) except primitives.PrimitiveFailedError: - self._sendSelector(selector, argcount, interp) + self._sendSelfSelector(selector, argcount, interp) def bytecodePrimAdd(self, interp): self.callPrimitiveAndPush(primitives.ADD, "+", 1, interp) Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Wed Oct 24 11:38:04 2007 @@ -427,4 +427,29 @@ interp.step() assert interp.activeContext == callerContext assert interp.activeContext.stack == [result] + +def test_singleExtendedSuperBytecode(): + w_super = model.W_Class(None, None) + w_class = model.W_Class(None, w_super) + w_object = w_class.new() + for bytecode, result in [ (returnReceiver, w_object), + (returnTrue, interpreter.Interpreter.TRUE), + (returnFalse, interpreter.Interpreter.FALSE), + (returnNil, interpreter.Interpreter.NIL), + (returnTopFromMethod, interpreter.Interpreter.ONE) ]: + w_super.methoddict["foo"] = model.W_CompiledMethod(None, 0, pushConstantOneBytecode + bytecode) + interp = new_interpreter(singleExtendedSuperBytecode + chr(0)) + interp.activeContext.method.literals = ["foo"] + interp.activeContext.push(w_object) + callerContext = interp.activeContext + interp.step() + assert interp.activeContext.sender == callerContext + assert interp.activeContext.stack == [] + assert interp.activeContext.receiver == w_object + assert interp.activeContext.method == w_super.methoddict["foo"] + assert callerContext.stack == [] + interp.step() + interp.step() + assert interp.activeContext == callerContext + assert interp.activeContext.stack == [result] From arigo at codespeak.net Wed Oct 24 11:38:36 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 24 Oct 2007 11:38:36 +0200 (CEST) Subject: [pypy-svn] r47805 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071024093836.D73858102@code0.codespeak.net> Author: arigo Date: Wed Oct 24 11:38:36 2007 New Revision: 47805 Modified: pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/semispace.py Log: Implement a "red zone" mechanism in the semispace GC to double the space size before it is completely full. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Wed Oct 24 11:38:36 2007 @@ -5,6 +5,7 @@ _alloc_flavor_ = "raw" moving_gc = False needs_write_barrier = False + needs_zero_gc_pointers = True def set_query_functions(self, is_varsize, getfinalizer, offsets_to_gc_pointers, Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Wed Oct 24 11:38:36 2007 @@ -4,6 +4,7 @@ from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object, debug_assert +from pypy.rpython.lltypesystem.lloperation import llop # The following flag is never set on young objects, i.e. the ones living # in the nursery. It is initially set on all prebuilt and old objects, @@ -148,6 +149,7 @@ debug_assert(self.nursery_size <= self.top_of_space - self.free, "obtain_free_space failed to do its job") if self.nursery: + #llop.debug_print(lltype.Void, "minor collect") # a nursery-only collection scan = self.free self.collect_oldrefs_to_nursery() Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Wed Oct 24 11:38:36 2007 @@ -20,6 +20,7 @@ class SemiSpaceGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True + needs_zero_gc_pointers = False HDR = lltype.Struct('header', ('forw', llmemory.Address), ('tid', lltype.Signed)) @@ -45,6 +46,7 @@ self.run_finalizers = self.AddressLinkedList() self.objects_with_weakrefs = self.AddressLinkedList() self.finalizer_lock_count = 0 + self.red_zone = 0 def disable_finalizers(self): self.finalizer_lock_count += 1 @@ -109,7 +111,11 @@ def try_obtain_free_space(self, needed): # XXX for bonus points do big objects differently needed = raw_malloc_usage(needed) - self.semispace_collect() + if (self.red_zone >= 2 and self.space_size < self.max_space_size and + self.double_space_size()): + pass # collect was done during double_space_size() + else: + self.semispace_collect() missing = needed - (self.top_of_space - self.free) if missing <= 0: return True # success @@ -133,6 +139,7 @@ return True def double_space_size(self): + self.red_zone = 0 old_fromspace = self.fromspace newsize = self.space_size * 2 newspace = llarena.arena_malloc(newsize, True) @@ -192,7 +199,24 @@ self.notify_objects_just_moved() if not size_changing: llarena.arena_reset(fromspace, self.space_size, True) + self.record_red_zone() self.execute_finalizers() + #llop.debug_print(lltype.Void, 'collected', self.space_size, size_changing, self.top_of_space - self.free) + + def record_red_zone(self): + # red zone: if the space is more than 80% full, the next collection + # should double its size. If it is more than 66% full twice in a row, + # then it should double its size too. (XXX adjust) + # The goal is to avoid many repeated collection that don't free a lot + # of memory each, if the size of the live object set is just below the + # size of the space. + free_after_collection = self.top_of_space - self.free + if free_after_collection > self.space_size // 3: + self.red_zone = 0 + else: + self.red_zone += 1 + if free_after_collection < self.space_size // 5: + self.red_zone += 1 def scan_copied(self, scan): while scan < self.free: From arigo at codespeak.net Wed Oct 24 11:39:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 24 Oct 2007 11:39:27 +0200 (CEST) Subject: [pypy-svn] r47806 - in pypy/dist/pypy/rpython/memory: . gctransform Message-ID: <20071024093927.E3BF88102@code0.codespeak.net> Author: arigo Date: Wed Oct 24 11:39:27 2007 New Revision: 47806 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctypelayout.py pypy/dist/pypy/rpython/memory/gcwrapper.py Log: * zero_gc_pointers wasn't emulated by the gcwrapper. * the gctransformer must not insert zeroing operations again for GCs that give you zero-initialized memory anyway. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Wed Oct 24 11:39:27 2007 @@ -295,6 +295,7 @@ s_gc = self.translator.annotator.bookkeeper.valueoftype(GCClass) r_gc = self.translator.rtyper.getrepr(s_gc) self.c_const_gc = rmodel.inputconst(r_gc, self.gcdata.gc) + self.needs_zero_gc_pointers = GCClass.needs_zero_gc_pointers HDR = self._gc_HDR = self.gcdata.gc.gcheaderbuilder.HDR self._gc_fields = fields = [] @@ -533,9 +534,10 @@ self.pop_roots(hop, livevars) def gct_zero_gc_pointers_inside(self, hop): - v_ob = hop.spaceop.args[0] - TYPE = v_ob.concretetype.TO - gen_zero_gc_pointers(TYPE, v_ob, hop.llops) + if self.needs_zero_gc_pointers: + v_ob = hop.spaceop.args[0] + TYPE = v_ob.concretetype.TO + gen_zero_gc_pointers(TYPE, v_ob, hop.llops) def gct_weakref_create(self, hop): op = hop.spaceop Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Wed Oct 24 11:39:27 2007 @@ -220,6 +220,27 @@ adr + llmemory.itemoffsetof(t, i)): yield a +def zero_gc_pointers(p): + TYPE = lltype.typeOf(p).TO + zero_gc_pointers_inside(p, TYPE) + +def zero_gc_pointers_inside(p, TYPE): + if isinstance(TYPE, lltype.Struct): + for name, FIELD in TYPE._flds.items(): + if isinstance(FIELD, lltype.Ptr) and FIELD.TO._gckind == 'gc': + setattr(p, name, lltype.nullptr(FIELD.TO)) + elif isinstance(FIELD, lltype.ContainerType): + zero_gc_pointers_inside(getattr(p, name), FIELD) + elif isinstance(TYPE, lltype.Array): + ITEM = TYPE.OF + if isinstance(ITEM, lltype.Ptr) and ITEM.TO._gckind == 'gc': + null = lltype.nullptr(ITEM.TO) + for i in range(p._obj.getlength()): + p[i] = null + elif isinstance(ITEM, lltype.ContainerType): + for i in range(p._obj.getlength()): + zero_gc_pointers_inside(p[i], ITEM) + ########## weakrefs ########## # framework: weakref objects are small structures containing only an address Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Wed Oct 24 11:39:27 2007 @@ -51,7 +51,10 @@ if flavor == 'gc': typeid = self.get_type_id(TYPE) addr = self.gc.malloc(typeid, n, zero=zero) - return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) + result = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) + if self.gc.needs_zero_gc_pointers: + gctypelayout.zero_gc_pointers(result) + return result else: return lltype.malloc(TYPE, n, flavor=flavor, zero=zero) From tverwaes at codespeak.net Wed Oct 24 11:45:19 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 11:45:19 +0200 (CEST) Subject: [pypy-svn] r47807 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071024094519.191358107@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 11:45:18 2007 New Revision: 47807 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (toon) added singleExtendedSuperBytecode and refactored send-tests Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Wed Oct 24 11:45:18 2007 @@ -190,16 +190,18 @@ interp.step() assert interp.activeContext.stack == [interp.ZERO, interp.ZERO] -def test_sendLiteralSelectorBytecode(): - w_class = model.W_Class(None, None) - w_object = w_class.new() +# w_class - the class from which the method is going to be called +# (and on which it is going to be installed) +# w_object - the actual object we will be sending the method to +# bytecodes - the bytecode to be executed +def sendBytecodesTest(w_class, w_object, bytecodes): for bytecode, result in [ (returnReceiver, w_object), (returnTrue, interpreter.Interpreter.TRUE), (returnFalse, interpreter.Interpreter.FALSE), (returnNil, interpreter.Interpreter.NIL), (returnTopFromMethod, interpreter.Interpreter.ONE) ]: w_class.methoddict["foo"] = model.W_CompiledMethod(None, 0, pushConstantOneBytecode + bytecode) - interp = new_interpreter(sendLiteralSelectorBytecode(0)) + interp = new_interpreter(bytecodes) interp.activeContext.method.literals = ["foo"] interp.activeContext.push(w_object) callerContext = interp.activeContext @@ -213,6 +215,11 @@ interp.step() assert interp.activeContext == callerContext assert interp.activeContext.stack == [result] + +def test_sendLiteralSelectorBytecode(): + w_class = model.W_Class(None, None) + w_object = w_class.new() + sendBytecodesTest(w_class, w_object, sendLiteralSelectorBytecode(0)) def test_fibWithArgument(): bytecode = ''.join(map(chr, [ 16, 119, 178, 154, 118, 164, 11, 112, 16, 118, 177, 224, 112, 16, 119, 177, 224, 176, 124 ])) @@ -407,49 +414,11 @@ def test_singleExtendedSendBytecode(): w_class = model.W_Class(None, None) w_object = w_class.new() - for bytecode, result in [ (returnReceiver, w_object), - (returnTrue, interpreter.Interpreter.TRUE), - (returnFalse, interpreter.Interpreter.FALSE), - (returnNil, interpreter.Interpreter.NIL), - (returnTopFromMethod, interpreter.Interpreter.ONE) ]: - w_class.methoddict["foo"] = model.W_CompiledMethod(None, 0, pushConstantOneBytecode + bytecode) - interp = new_interpreter(singleExtendedSendBytecode + chr(0)) - interp.activeContext.method.literals = ["foo"] - interp.activeContext.push(w_object) - callerContext = interp.activeContext - interp.step() - assert interp.activeContext.sender == callerContext - assert interp.activeContext.stack == [] - assert interp.activeContext.receiver == w_object - assert interp.activeContext.method == w_class.methoddict["foo"] - assert callerContext.stack == [] - interp.step() - interp.step() - assert interp.activeContext == callerContext - assert interp.activeContext.stack == [result] + sendBytecodesTest(w_class, w_object, singleExtendedSendBytecode + chr(0)) def test_singleExtendedSuperBytecode(): w_super = model.W_Class(None, None) w_class = model.W_Class(None, w_super) w_object = w_class.new() - for bytecode, result in [ (returnReceiver, w_object), - (returnTrue, interpreter.Interpreter.TRUE), - (returnFalse, interpreter.Interpreter.FALSE), - (returnNil, interpreter.Interpreter.NIL), - (returnTopFromMethod, interpreter.Interpreter.ONE) ]: - w_super.methoddict["foo"] = model.W_CompiledMethod(None, 0, pushConstantOneBytecode + bytecode) - interp = new_interpreter(singleExtendedSuperBytecode + chr(0)) - interp.activeContext.method.literals = ["foo"] - interp.activeContext.push(w_object) - callerContext = interp.activeContext - interp.step() - assert interp.activeContext.sender == callerContext - assert interp.activeContext.stack == [] - assert interp.activeContext.receiver == w_object - assert interp.activeContext.method == w_super.methoddict["foo"] - assert callerContext.stack == [] - interp.step() - interp.step() - assert interp.activeContext == callerContext - assert interp.activeContext.stack == [result] + sendBytecodesTest(w_super, w_object, singleExtendedSuperBytecode + chr(0)) From niko at codespeak.net Wed Oct 24 11:51:48 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Wed, 24 Oct 2007 11:51:48 +0200 (CEST) Subject: [pypy-svn] r47808 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024095148.3C5568107@code0.codespeak.net> Author: niko Date: Wed Oct 24 11:51:47 2007 New Revision: 47808 Modified: pypy/dist/pypy/lang/smalltalk/fakeimage.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: 1. rewrite test_primitives to guarantee that the stack is untouched in the event of error, and fully popped in the event of success 2. add a decorator in primitives.py that allows a primitive to declare how many stack items it uses, and handles peeking/popping automatically 3. implement STRING_AT_PUT Modified: pypy/dist/pypy/lang/smalltalk/fakeimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/fakeimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/fakeimage.py Wed Oct 24 11:51:47 2007 @@ -1,20 +1,15 @@ import pypy.lang.smalltalk.classtable as ct from pypy.lang.smalltalk import model +# ___________________________________________________________________________ +# Utility Methods + def small_int(i): return model.W_SmallInteger(ct.w_SmallInteger, i) def wrap_float(i): return model.W_Float(ct.w_Float,i) -def make_char_table(): - global CharacterTable - def make_char(i): - w_cinst = ct.w_Character.new() - w_cinst.setnamedvar(0, small_int(i)) - CharacterTable = [make_char(i) for i in range(256)] -make_char_table() - def make_string(str): w_inst = ct.w_ByteString.new(len(str)) for i in range(len(str)): @@ -24,12 +19,30 @@ def make_char(c): return CharacterTable[ord(c)] +def ord_w_char(w_c): + assert w_c.w_class is ct.w_Character + w_ord = w_c.getnamedvar(0) + assert w_ord.w_class is ct.w_SmallInteger + assert isinstance(w_ord, model.W_SmallInteger) + return w_ord.value + def wrap_bool(bool): if bool: return w_true else: return w_false +# ___________________________________________________________________________ +# Global Data + +def make_char_table(): + global CharacterTable + def make_char(i): + w_cinst = ct.w_Character.new() + w_cinst.setnamedvar(0, small_int(i)) + CharacterTable = [make_char(i) for i in range(256)] +make_char_table() + w_true = ct.w_True.new() w_false = ct.w_False.new() w_nil = ct.w_UndefinedObject.new() Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Wed Oct 24 11:51:47 2007 @@ -30,25 +30,36 @@ prim_table = [None] * 127 def primitive(code): - def f(func): + def decorator(func): prim_table[code] = func return func - return f + return decorator + +def stack(n): + def decorator(wrapped): + def result(frame): + items = [frame.peek(i) for i in range(n)] + res = wrapped(items) + frame.pop_n(n) # only if no exception occurs! + return res + return result + return decorator # ___________________________________________________________________________ # Small Integer Primitives ADD = 1 SUBTRACT = 2 +MAKE_POINT = 18 math_ops = { ADD: operator.add, SUBTRACT: operator.sub } for (code,op) in math_ops.items(): - def func(frame, op=op): # n.b. capture op value - w_v1 = frame.peek(1) - w_v2 = frame.peek(0) + @stack(2) + def func(stack, op=op): # n.b. capture op value + [w_v2, w_v1] = stack v1 = unwrap_int(w_v1) v2 = unwrap_int(w_v2) res = op(v1, v2) @@ -58,11 +69,9 @@ if res < -1073741824: raise PrimitiveFailedError() w_res = fimg.small_int(res) - frame.pop_n(2) return w_res prim_table[code] = func -MAKE_POINT = 18 @primitive(MAKE_POINT) def primitivemakepoint(frame): raise PrimitiveNotYetWrittenError(MAKE_POINT) @@ -86,13 +95,12 @@ FLOAT_SUBTRACT: operator.sub } for (code,op) in math_ops.items(): - def func(frame, op=op): # n.b. capture op value - w_v1 = frame.peek(0) - w_v2 = frame.peek(1) + @stack(2) + def func(res, op=op): # n.b. capture op value + [w_v2, w_v1] = res v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) w_res = fimg.wrap_float(op(v1, v2)) - frame.pop_n(2) return w_res prim_table[code] = func @@ -105,59 +113,56 @@ STRING_AT = 63 STRING_AT_PUT = 64 -def common_at(frame): - w_idx = frame.peek(0) - w_obj = frame.peek(1) +def common_at(stack): + [w_idx, w_obj] = stack idx = unwrap_int(w_idx) if not is_valid_index(idx, w_obj): raise PrimitiveFailedError() - frame.pop_n(2) return idx, w_obj -def common_at_put(frame): - w_val = frame.peek(0) - w_idx = frame.peek(1) - w_obj = frame.peek(2) +def common_at_put(stack): + [w_val, w_idx, w_obj] = stack idx = unwrap_int(w_idx) if not is_valid_index(idx, w_obj): raise PrimitiveFailedError() - frame.pop_n(2) - return idx, w_obj + return w_val, idx, w_obj @primitive(AT) -def func(frame): - idx, w_obj = common_at(frame) + at stack(2) +def func(stack): + idx, w_obj = common_at(stack) return w_obj.getindexedvar(idx) @primitive(AT_PUT) -def func(frame): - w_val = frame.peek(0) - w_idx = frame.peek(1) - w_obj = frame.peek(2) - idx = unwrap_int(w_idx) - if not is_valid_index(idx, w_obj): - raise PrimitiveFailedError() + at stack(3) +def func(stack): + w_val, idx, w_obj = common_at_put(stack) w_obj.setindexedvar(idx, w_val) - frame.pop_n(3) return w_val @primitive(SIZE) -def func(frame): - w_obj = frame.peek(0) + at stack(1) +def func(stack): + [w_obj] = stack if not w_obj.w_class.isindexable(): raise PrimitiveFailedError() - frame.pop_n(1) return w_obj.size() @primitive(STRING_AT) -def func(frame): - idx, w_obj = common_at(frame) + at stack(2) +def func(stack): + idx, w_obj = common_at(stack) byte = w_obj.getbyte(idx) return fimg.CharacterTable[byte] @primitive(STRING_AT_PUT) -def func(frame): - raise PrimitiveNotYetWrittenError() + at stack(3) +def func(stack): + w_val, idx, w_obj = common_at_put(stack) + if w_val.w_class is not ct.w_Character: + raise PrimitiveFailedError() + w_obj.setbyte(idx, fimg.ord_w_char(w_val)) + return w_val # ___________________________________________________________________________ # Boolean Primitives Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Wed Oct 24 11:51:47 2007 @@ -19,50 +19,71 @@ def mock(stack): mapped_stack = [wrap(x) for x in stack] return MockFrame(mapped_stack) - + +def prim(code, stack): + stack_w = mock(stack) + res = prim_table[code](stack_w) + assert not len(stack_w.stack) # only pass as many arguments as it uses + return res + +def prim_fails(code, stack): + stack_w = mock(stack) + orig_stack = list(stack_w.stack) + try: + prim_table[code](stack_w) + py.test.fail("Expected PrimitiveFailedError") + except PrimitiveFailedError: + assert stack_w.stack == orig_stack + def test_small_int_plus(): - assert prim_table[p.ADD](mock([1,2])).value == 3 - assert prim_table[p.ADD](mock([3,4])).value == 7 + assert prim(p.ADD, [1,2]).value == 3 + assert prim(p.ADD, [3,4]).value == 7 def test_small_int_minus(): - assert prim_table[p.SUBTRACT](mock([5,9])).value == -4 + assert prim(p.SUBTRACT, [5,9]).value == -4 def test_small_int_overflow(): - def f(): - prim_table[p.ADD](mock([1073741823,2])) - py.test.raises(PrimitiveFailedError, f) + prim_fails(p.ADD, [1073741823,2]) def test_float(): - assert prim_table[p.FLOAT_ADD](mock([1.0,2.0])).value == 3.0 - assert prim_table[p.FLOAT_ADD](mock([3,4.5])).value == 7.5 + assert prim(p.FLOAT_ADD, [1.0,2.0]).value == 3.0 + assert prim(p.FLOAT_ADD, [3,4.5]).value == 7.5 def test_at(): w_obj = model.W_Class(None, None, 0, format=model.VAR_POINTERS).new(1) w_obj.setindexedvar(0, "foo") - assert prim_table[p.AT](mock([w_obj, 0])) == "foo" + assert prim(p.AT, [w_obj, 0]) == "foo" + +def test_invalid_at(): + w_obj = model.W_Class(None, None, 0, format=model.POINTERS).new() + prim_fails(p.AT, [w_obj, 0]) def test_at_put(): w_obj = model.W_Class(None, None, 0, format=model.VAR_POINTERS).new(1) - assert prim_table[p.AT_PUT](mock([w_obj, 0, 22])).value == 22 - assert prim_table[p.AT](mock([w_obj, 0])).value == 22 + assert prim(p.AT_PUT, [w_obj, 0, 22]).value == 22 + assert prim(p.AT, [w_obj, 0]).value == 22 +def test_invalid_at_put(): + w_obj = model.W_Class(None, None, 0, format=model.POINTERS).new() + prim_fails(p.AT_PUT, [w_obj, 0, 22]) + def test_string_at(): w_str = fimg.make_string("foobar") - assert prim_table[p.STRING_AT](mock([w_str, 3])) == \ + assert prim(p.STRING_AT, [w_str, 3]) == \ fimg.make_char("b") def test_boolean(): - assert prim_table[p.LESSTHAN](mock([1,2])) == fimg.w_true - assert prim_table[p.GREATERTHAN](mock([3,4])) == fimg.w_false - assert prim_table[p.LESSOREQUAL](mock([1,2])) == fimg.w_true - assert prim_table[p.GREATEROREQUAL](mock([3,4])) == fimg.w_false - assert prim_table[p.EQUAL](mock([2,2])) == fimg.w_true - assert prim_table[p.NOTEQUAL](mock([2,2])) == fimg.w_false + assert prim(p.LESSTHAN, [1,2]) == fimg.w_true + assert prim(p.GREATERTHAN, [3,4]) == fimg.w_false + assert prim(p.LESSOREQUAL, [1,2]) == fimg.w_true + assert prim(p.GREATEROREQUAL, [3,4]) == fimg.w_false + assert prim(p.EQUAL, [2,2]) == fimg.w_true + assert prim(p.NOTEQUAL, [2,2]) == fimg.w_false def test_float_boolean(): - assert prim_table[p.FLOAT_LESSTHAN](mock([1.0,2.0])) == fimg.w_true - assert prim_table[p.FLOAT_GREATERTHAN](mock([3.0,4.0])) == fimg.w_false - assert prim_table[p.FLOAT_LESSOREQUAL](mock([1.3,2.6])) == fimg.w_true - assert prim_table[p.FLOAT_GREATEROREQUAL](mock([3.5,4.9])) == fimg.w_false - assert prim_table[p.FLOAT_EQUAL](mock([2.2,2.2])) == fimg.w_true - assert prim_table[p.FLOAT_NOTEQUAL](mock([2.2,2.2])) == fimg.w_false + assert prim(p.FLOAT_LESSTHAN, [1.0,2.0]) == fimg.w_true + assert prim(p.FLOAT_GREATERTHAN, [3.0,4.0]) == fimg.w_false + assert prim(p.FLOAT_LESSOREQUAL, [1.3,2.6]) == fimg.w_true + assert prim(p.FLOAT_GREATEROREQUAL, [3.5,4.9]) == fimg.w_false + assert prim(p.FLOAT_EQUAL, [2.2,2.2]) == fimg.w_true + assert prim(p.FLOAT_NOTEQUAL, [2.2,2.2]) == fimg.w_false From niko at codespeak.net Wed Oct 24 12:03:17 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Wed, 24 Oct 2007 12:03:17 +0200 (CEST) Subject: [pypy-svn] r47809 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024100317.C0C988107@code0.codespeak.net> Author: niko Date: Wed Oct 24 12:03:17 2007 New Revision: 47809 Modified: pypy/dist/pypy/lang/smalltalk/fakeimage.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: add tests for STRING_AT_PUT and fix a rather obvious bug in the character table generation Modified: pypy/dist/pypy/lang/smalltalk/fakeimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/fakeimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/fakeimage.py Wed Oct 24 12:03:17 2007 @@ -37,10 +37,11 @@ def make_char_table(): global CharacterTable - def make_char(i): + def bld_char(i): w_cinst = ct.w_Character.new() w_cinst.setnamedvar(0, small_int(i)) - CharacterTable = [make_char(i) for i in range(256)] + return w_cinst + CharacterTable = [bld_char(i) for i in range(256)] make_char_table() w_true = ct.w_True.new() Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Wed Oct 24 12:03:17 2007 @@ -14,6 +14,8 @@ if isinstance(x, int): return fimg.small_int(x) if isinstance(x, float): return fimg.wrap_float(x) if isinstance(x, model.W_Object): return x + if isinstance(x, str) and len(x) == 1: return fimg.make_char(x) + if isinstance(x, str): return fimg.make_string(x) raise NotImplementedError def mock(stack): @@ -68,9 +70,13 @@ prim_fails(p.AT_PUT, [w_obj, 0, 22]) def test_string_at(): - w_str = fimg.make_string("foobar") - assert prim(p.STRING_AT, [w_str, 3]) == \ - fimg.make_char("b") + assert prim(p.STRING_AT, ["foobar", 3]) == wrap("b") + +def test_string_at_put(): + assert prim(p.STRING_AT_PUT, ["foobar", 3, "c"]) == wrap("c") + exp = "foocar" + for i in range(6): + assert prim(p.STRING_AT, [exp, i]) == wrap(exp[i]) def test_boolean(): assert prim(p.LESSTHAN, [1,2]) == fimg.w_true From niko at codespeak.net Wed Oct 24 12:04:47 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Wed, 24 Oct 2007 12:04:47 +0200 (CEST) Subject: [pypy-svn] r47810 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071024100447.6E966808F@code0.codespeak.net> Author: niko Date: Wed Oct 24 12:04:47 2007 New Revision: 47810 Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: um, make test_string_at_put actually *test* STRING_AT_PUT, rather than comparing expected values against themselves. Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Wed Oct 24 12:04:47 2007 @@ -73,10 +73,11 @@ assert prim(p.STRING_AT, ["foobar", 3]) == wrap("b") def test_string_at_put(): - assert prim(p.STRING_AT_PUT, ["foobar", 3, "c"]) == wrap("c") + test_str = wrap("foobar") + assert prim(p.STRING_AT_PUT, [test_str, 3, "c"]) == wrap("c") exp = "foocar" - for i in range(6): - assert prim(p.STRING_AT, [exp, i]) == wrap(exp[i]) + for i in range(len(exp)): + assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i]) def test_boolean(): assert prim(p.LESSTHAN, [1,2]) == fimg.w_true From fijal at codespeak.net Wed Oct 24 12:18:25 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 24 Oct 2007 12:18:25 +0200 (CEST) Subject: [pypy-svn] r47811 - in pypy/dist/pypy/module/__builtin__: . test Message-ID: <20071024101825.3C985808F@code0.codespeak.net> Author: fijal Date: Wed Oct 24 12:18:24 2007 New Revision: 47811 Modified: pypy/dist/pypy/module/__builtin__/importing.py pypy/dist/pypy/module/__builtin__/test/test_import.py Log: Slight refactoring to allow re-use of internal functions by zipimport Modified: pypy/dist/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/importing.py (original) +++ pypy/dist/pypy/module/__builtin__/importing.py Wed Oct 24 12:18:24 2007 @@ -69,6 +69,14 @@ else: return NOFILE +def _prepare_module(space, w_mod, filename, pkgdir): + w = space.wrap + space.sys.setmodule(w_mod) + space.setattr(w_mod, w('__file__'), space.wrap(filename)) + space.setattr(w_mod, w('__doc__'), space.w_None) + if pkgdir is not None: + space.setattr(w_mod, w('__path__'), space.newlist([w(pkgdir)])) + def try_import_mod(space, w_modulename, filepart, w_parent, w_name, pkgdir=None): # decide what type we want (pyc/py) @@ -89,18 +97,16 @@ filename = filepart + ".pyc" stream = streamio.open_file_as_stream(filename, "rb") - space.sys.setmodule(w_mod) - space.setattr(w_mod, w('__file__'), space.wrap(filename)) - space.setattr(w_mod, w('__doc__'), space.w_None) - if pkgdir is not None: - space.setattr(w_mod, w('__path__'), space.newlist([w(pkgdir)])) - + _prepare_module(space, w_mod, filename, pkgdir) try: try: if modtype == PYFILE: - load_source_module(space, w_modulename, w_mod, filename, stream) + load_source_module(space, w_modulename, w_mod, filename, stream.readall()) else: - load_compiled_module(space, w_modulename, w_mod, filename, stream) + magic = _r_long(stream) + timestamp = _r_long(stream) + load_compiled_module(space, w_modulename, w_mod, filename, + magic, timestamp, stream.readall()) finally: stream.close() @@ -423,10 +429,9 @@ return result -def parse_source_module(space, pathname, stream): +def parse_source_module(space, pathname, source): """ Parse a source file and return the corresponding code object """ w = space.wrap - source = stream.readall() w_source = w(source) w_mode = w("exec") w_pathname = w(pathname) @@ -434,13 +439,14 @@ pycode = space.interp_w(Code, w_code) return pycode -def load_source_module(space, w_modulename, w_mod, pathname, stream): +def load_source_module(space, w_modulename, w_mod, pathname, source, + write_pyc=True): """ Load a source module from a given file and return its module object. """ w = space.wrap - pycode = parse_source_module(space, pathname, stream) + pycode = parse_source_module(space, pathname, source) w_dict = space.getattr(w_mod, w('__dict__')) space.call_method(w_dict, 'setdefault', @@ -448,7 +454,7 @@ w(space.builtin)) pycode.exec_code(space, w_dict, w_dict) - if space.config.objspace.usepycfiles: + if space.config.objspace.usepycfiles and write_pyc: mtime = os.stat(pathname)[stat.ST_MTIME] cpathname = pathname + 'c' write_compiled_module(space, pycode, cpathname, mtime) @@ -508,11 +514,10 @@ stream.close() return 1 -def read_compiled_module(space, cpathname, stream): +def read_compiled_module(space, cpathname, strbuf): """ Read a code object from a file and check it for validity """ w_marshal = space.getbuiltinmodule('marshal') - strbuf = stream.readall() w_code = space.call_method(w_marshal, 'loads', space.wrap(strbuf)) pycode = space.interpclass_w(w_code) if pycode is None or not isinstance(pycode, Code): @@ -520,19 +525,18 @@ "Non-code object in %s" % cpathname)) return pycode -def load_compiled_module(space, w_modulename, w_mod, cpathname, stream): +def load_compiled_module(space, w_modulename, w_mod, cpathname, magic, + timestamp, source): """ Load a module from a compiled file, execute it, and return its module object. """ w = space.wrap - magic = _r_long(stream) if magic != get_pyc_magic(space): raise OperationError(space.w_ImportError, w( "Bad magic number in %s" % cpathname)) - _r_long(stream) # skip time stamp #print "loading pyc file:", cpathname - code_w = read_compiled_module(space, cpathname, stream) + code_w = read_compiled_module(space, cpathname, source) #if (Py_VerboseFlag) # PySys_WriteStderr("import %s # precompiled from %s\n", # name, cpathname); Modified: pypy/dist/pypy/module/__builtin__/test/test_import.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_import.py Wed Oct 24 12:18:24 2007 @@ -71,7 +71,7 @@ w_modname, w(importing.Module(space, w_modname)), filename, - stream) + stream.readall()) finally: stream.close() @@ -328,7 +328,7 @@ try: stream.seek(8, 0) w_code = importing.read_compiled_module( - space, cpathname, stream) + space, cpathname, stream.readall()) pycode = space.interpclass_w(w_code) finally: stream.close() @@ -349,11 +349,15 @@ stream = streamio.open_file_as_stream(cpathname, "r") try: w_mod = space.wrap(Module(space, w_modulename)) + magic = importing._r_long(stream) + timestamp = importing._r_long(stream) w_ret = importing.load_compiled_module(space, w_modulename, w_mod, cpathname, - stream) + magic, + timestamp, + stream.readall()) finally: stream.close() assert w_mod is w_ret @@ -368,7 +372,7 @@ try: w_ret = importing.parse_source_module(space, pathname, - stream) + stream.readall()) finally: stream.close() pycode = space.interpclass_w(w_ret) @@ -410,7 +414,7 @@ w_modulename, w_mod, pathname, - stream) + stream.readall()) finally: stream.close() assert w_mod is w_ret @@ -427,7 +431,7 @@ try: w_ret = importing.parse_source_module(space, pathname, - stream) + stream.readall()) finally: stream.close() pycode = space.interpclass_w(w_ret) @@ -452,7 +456,8 @@ stream = streamio.open_file_as_stream(cpathname, "r") try: stream.seek(8, 0) - w_code = importing.read_compiled_module(space, cpathname, stream) + w_code = importing.read_compiled_module(space, cpathname, + stream.readall()) pycode = space.interpclass_w(w_code) finally: stream.close() @@ -485,13 +490,17 @@ stream = streamio.open_file_as_stream(cpathname, "r") try: w_mod = space2.wrap(Module(space2, w_modulename)) + magic = importing._r_long(stream) + timestamp = importing._r_long(stream) space2.raises_w(space2.w_ImportError, importing.load_compiled_module, space2, w_modulename, w_mod, cpathname, - stream) + magic, + timestamp, + stream.readall()) finally: stream.close() From fijal at codespeak.net Wed Oct 24 12:30:02 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 24 Oct 2007 12:30:02 +0200 (CEST) Subject: [pypy-svn] r47812 - pypy/dist/pypy/module/zipimport Message-ID: <20071024103002.926F980D3@code0.codespeak.net> Author: fijal Date: Wed Oct 24 12:30:00 2007 New Revision: 47812 Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py Log: The same dot, but refactor inner parts to re-use the __builtin__.importing Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/interp_zipimport.py Wed Oct 24 12:30:00 2007 @@ -7,6 +7,7 @@ from pypy.interpreter.gateway import app2interp from pypy.interpreter.eval import Code from pypy.interpreter.module import Module +from pypy.module.__builtin__ import importing import os import stat @@ -24,22 +25,12 @@ space.newlist([])))] def import_py_file(self, space, modname, filename, w_buf): + buf = space.str_w(w_buf) w = space.wrap - # XXX A bit of boilerplatish code stolen from importing.py, - # some refactoring to use common code base? - # all this error reporting and whatnot w_mod = w(Module(space, w(modname))) - space.sys.setmodule(w_mod) - space.setattr(w_mod, w('__file__'), w(filename)) - space.setattr(w_mod, w('__doc__'), space.w_None) - w_mode = w("exec") - w_code = space.builtin.call('compile', w_buf, w(modname), w_mode) - pycode = space.interp_w(Code, w_code) - w_dict = space.getattr(w_mod, w('__dict__')) - space.call_method(w_dict, 'setdefault', - w('__builtins__'), - w(space.builtin)) - pycode.exec_code(space, w_dict, w_dict) + importing._prepare_module(space, w_mod, filename, None) + return importing.load_source_module(space, w(modname), w_mod, + filename, buf, write_pyc=False) return w_mod def get_module(self, space, name): @@ -67,13 +58,29 @@ return self.import_py_file(space, name, filename, w_buf) load_module.unwrap_spec = ['self', ObjSpace, str] + def get_data(self, space): + pass + get_data.unwrap_spec = ['self', ObjSpace] + + def get_code(self, space): + pass + get_code.unwrap_spec = ['self', ObjSpace] + + def get_source(self, space): + pass + get_source.unwrap_spec = ['self', ObjSpace] + + def is_package(self, space): + pass + is_package.unwrap_spec = ['self', ObjSpace] + def descr_new_zipimporter(space, w_type, name): try: s = os.stat(name) except OSError: - return + return space.w_None if stat.S_ISDIR(s.st_mode): - return + return space.w_None w_import = space.builtin.get('__import__') w_zipfile = space.call(w_import, space.newlist([ space.wrap('zipfile'), @@ -85,7 +92,7 @@ w_dir = space.call(w_ZipFile, space.newlist([space.wrap(name)])) except OperationError: # we catch everything as this function # should not raise - return None + return space.w_None return space.wrap(W_ZipImporter(space, name, w_dir, w_zipfile)) descr_new_zipimporter.unwrap_spec = [ObjSpace, W_Root, str] @@ -94,5 +101,9 @@ 'zipimporter', __new__ = interp2app(descr_new_zipimporter), find_module = interp2app(W_ZipImporter.find_module), + get_data = interp2app(W_ZipImporter.get_data), + get_code = interp2app(W_ZipImporter.get_code), + get_source = interp2app(W_ZipImporter.get_source), + is_package = interp2app(W_ZipImporter.is_package), load_module = interp2app(W_ZipImporter.load_module), ) From niko at codespeak.net Wed Oct 24 12:52:37 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Wed, 24 Oct 2007 12:52:37 +0200 (CEST) Subject: [pypy-svn] r47813 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024105237.7CDF58110@code0.codespeak.net> Author: niko Date: Wed Oct 24 12:52:37 2007 New Revision: 47813 Modified: pypy/dist/pypy/lang/smalltalk/fakeimage.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: implement OBJECT_AT and OBJECT_AT_PUT. add constants for known slot indices into fakeimage.py. add simple tests for OBJECT_AT, OBJECT_AT_PUT Modified: pypy/dist/pypy/lang/smalltalk/fakeimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/fakeimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/fakeimage.py Wed Oct 24 12:52:37 2007 @@ -21,7 +21,7 @@ def ord_w_char(w_c): assert w_c.w_class is ct.w_Character - w_ord = w_c.getnamedvar(0) + w_ord = w_c.getnamedvar(CHARACTER_VALUE_INDEX) assert w_ord.w_class is ct.w_SmallInteger assert isinstance(w_ord, model.W_SmallInteger) return w_ord.value @@ -33,13 +33,23 @@ return w_false # ___________________________________________________________________________ +# Slot Names + +CHARACTER_VALUE_INDEX = 0 # Page 630 of the blue book + +STREAM_ARRAY_INDEX = 0 # Page 631 of the blue book +STREAM_INDEX_INDEX = 1 +STREAM_READ_LIMIT_INDEX = 2 +STREAM_WRITE_LIMIT_INDEX = 3 + +# ___________________________________________________________________________ # Global Data def make_char_table(): global CharacterTable def bld_char(i): w_cinst = ct.w_Character.new() - w_cinst.setnamedvar(0, small_int(i)) + w_cinst.setnamedvar(CHARACTER_VALUE_INDEX, small_int(i)) return w_cinst CharacterTable = [bld_char(i) for i in range(256)] make_char_table() Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Wed Oct 24 12:52:37 2007 @@ -112,6 +112,8 @@ SIZE = 62 STRING_AT = 63 STRING_AT_PUT = 64 +OBJECT_AT = 68 +OBJECT_AT_PUT = 69 def common_at(stack): [w_idx, w_obj] = stack @@ -164,6 +166,25 @@ w_obj.setbyte(idx, fimg.ord_w_char(w_val)) return w_val + at primitive(OBJECT_AT) + at stack(2) +def func(stack): + [w_idx, w_rcvr] = stack + idx = unwrap_int(w_idx) + if idx < 0 or idx >= w_rcvr.w_class.instvarsize: + raise PrimitiveFailedError() + return w_rcvr.getnamedvar(idx) + + at primitive(OBJECT_AT_PUT) + at stack(3) +def func(stack): + [w_val, w_idx, w_rcvr] = stack + idx = unwrap_int(w_idx) + if idx < 0 or idx >= w_rcvr.w_class.instvarsize: + raise PrimitiveFailedError() + w_rcvr.setnamedvar(idx, w_val) + return w_val + # ___________________________________________________________________________ # Boolean Primitives @@ -190,27 +211,23 @@ NOTEQUAL: operator.ne } for (code,op) in bool_ops.items(): - def func(frame, op=op): # n.b. capture op value - w_v1 = frame.peek(1) - w_v2 = frame.peek(0) + @primitive(code) + @stack(2) + def func(stack, op=op): # n.b. capture op value + [w_v2, w_v1] = stack v1 = unwrap_int(w_v1) v2 = unwrap_int(w_v2) res = op(v1, v2) - w_res = fimg.wrap_bool(res) - frame.pop_n(2) return w_res - prim_table[code] = func for (code,op) in bool_ops.items(): - def func(frame, op=op): # n.b. capture op value - w_v1 = frame.peek(1) - w_v2 = frame.peek(0) + @primitive(code+_FLOAT_OFFSET) + @stack(2) + def func(stack, op=op): # n.b. capture op value + [w_v2, w_v1] = stack v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) res = op(v1, v2) - w_res = fimg.wrap_bool(res) - frame.pop_n(2) return w_res - prim_table[code+_FLOAT_OFFSET] = func Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Wed Oct 24 12:52:37 2007 @@ -79,6 +79,29 @@ for i in range(len(exp)): assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i]) +def test_object_at(): + w_v = prim(p.OBJECT_AT, ["q", fimg.CHARACTER_VALUE_INDEX]) + assert w_v.value == ord("q") + +def test_invalid_object_at(): + prim_fails(p.OBJECT_AT, ["q", fimg.CHARACTER_VALUE_INDEX+1]) + +def test_object_at_put(): + w_obj = model.W_Class(None, None, 1, format=model.POINTERS).new() + assert prim(p.OBJECT_AT_PUT, [w_obj, 0, "q"]) is wrap("q") + assert prim(p.OBJECT_AT, [w_obj, 0]) is wrap("q") + +def test_invalid_object_at_put(): + w_obj = model.W_Class(None, None, 1, format=model.POINTERS).new() + prim_fails(p.OBJECT_AT, [w_obj, 1, 1]) + +def test_string_at_put(): + test_str = wrap("foobar") + assert prim(p.STRING_AT_PUT, [test_str, 3, "c"]) == wrap("c") + exp = "foocar" + for i in range(len(exp)): + assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i]) + def test_boolean(): assert prim(p.LESSTHAN, [1,2]) == fimg.w_true assert prim(p.GREATERTHAN, [3,4]) == fimg.w_false From fijal at codespeak.net Wed Oct 24 13:03:15 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 24 Oct 2007 13:03:15 +0200 (CEST) Subject: [pypy-svn] r47814 - pypy/dist/pypy/module/__builtin__ Message-ID: <20071024110315.B990980EE@code0.codespeak.net> Author: fijal Date: Wed Oct 24 13:03:15 2007 New Revision: 47814 Modified: pypy/dist/pypy/module/__builtin__/importing.py Log: Small refactoring Modified: pypy/dist/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/importing.py (original) +++ pypy/dist/pypy/module/__builtin__/importing.py Wed Oct 24 13:03:15 2007 @@ -461,11 +461,7 @@ return w_mod -# helper, to avoid exposing internals of marshal and the -# difficulties of using it though applevel. -_r_correction = intmask(1L<<32) # == 0 on 32-bit machines -def _r_long(stream): - s = stream.read(4) # XXX XXX could return smaller string +def _get_long(s): if len(s) < 4: return -1 # good enough for our purposes a = ord(s[0]) @@ -475,7 +471,14 @@ x = a | (b<<8) | (c<<16) | (d<<24) if _r_correction and d & 0x80 and x > 0: x -= _r_correction - return int(x) + return int(x) + +# helper, to avoid exposing internals of marshal and the +# difficulties of using it though applevel. +_r_correction = intmask(1L<<32) # == 0 on 32-bit machines +def _r_long(stream): + s = stream.read(4) # XXX XXX could return smaller string + return _get_long(s) def _w_long(stream, x): a = x & 0xff From cfbolz at codespeak.net Wed Oct 24 13:15:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 24 Oct 2007 13:15:38 +0200 (CEST) Subject: [pypy-svn] r47815 - in pypy/dist/pypy/rpython/memory/gctransform: . test Message-ID: <20071024111538.4B6E9810E@code0.codespeak.net> Author: cfbolz Date: Wed Oct 24 13:15:37 2007 New Revision: 47815 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py pypy/dist/pypy/rpython/memory/gctransform/test/test_transform.py Log: a small heuristic to find initializing stores (basically stores that happen directly after mallocs without any operation in between that can cause a malloc). Those stores don't need a write barrier for the generational GC. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Wed Oct 24 13:15:37 2007 @@ -28,6 +28,64 @@ flags = op.args[1].value return flags['flavor'] == 'gc' and not flags.get('nocollect', False) +def find_initializing_stores(collect_analyzer, graph): + from pypy.objspace.flow.model import mkentrymap + entrymap = mkentrymap(graph) + # a bit of a hackish analysis: if a block contains a malloc and check that + # the result is not zero, then the block following the True link will + # usually initialize the newly allocated object + result = {} + def find_in_block(block, mallocvars): + for i, op in enumerate(block.operations): + if op.opname in ("cast_pointer", "same_as"): + if op.args[0] in mallocvars: + mallocvars[op.result] = True + elif op.opname in ("setfield", "setarrayitem", "setinteriorfield"): + if (op.args[0] in mallocvars and + op.args[-1].concretetype.TO._gckind == "gc"): + result[op] = True + else: + if collect_analyzer.analyze(op): + return + for exit in block.exits: + if len(entrymap[exit.target]) != 1: + continue + newmallocvars = {} + for i, var in enumerate(exit.args): + if var in mallocvars: + newmallocvars[exit.target.inputargs[i]] = True + if newmallocvars: + find_in_block(exit.target, newmallocvars) + mallocnum = 0 + blockset = set(graph.iterblocks()) + while blockset: + block = blockset.pop() + if len(block.operations) < 2: + continue + mallocop = block.operations[-2] + checkop = block.operations[-1] + if not (mallocop.opname == "malloc" and + checkop.opname == "ptr_nonzero" and + mallocop.result is checkop.args[0] and + block.exitswitch is checkop.result): + continue + exits = [exit for exit in block.exits if exit.llexitcase] + if len(exits) != 1: + continue + exit = exits[0] + if len(entrymap[exit.target]) != 1: + continue + try: + index = exit.args.index(mallocop.result) + except ValueError: + continue + target = exit.target + mallocvars = {target.inputargs[index]: True} + mallocnum += 1 + find_in_block(target, mallocvars) + print graph.name, mallocnum, len(result) + return result + ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void) class FrameworkGCTransformer(GCTransformer): @@ -439,6 +497,14 @@ for typeid, TYPE in all: f.write("%s %s\n" % (typeid, TYPE)) + def transform_graph(self, graph): + if self.write_barrier_ptr: + self.initializing_stores = find_initializing_stores( + self.collect_analyzer, graph) + super(FrameworkGCTransformer, self).transform_graph(graph) + if self.write_barrier_ptr: + self.initializing_stores = None + def gct_direct_call(self, hop): if self.collect_analyzer.analyze(hop.spaceop): livevars = self.push_roots(hop) @@ -603,7 +669,8 @@ # ok if (self.write_barrier_ptr is not None and not isinstance(v_newvalue, Constant) - and v_struct.concretetype.TO._gckind == "gc"): + and v_struct.concretetype.TO._gckind == "gc" + and hop.spaceop not in self.initializing_stores): self.write_barrier_calls += 1 v_oldvalue = hop.genop('g' + opname[1:], hop.inputargs()[:-1], Modified: pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py Wed Oct 24 13:15:37 2007 @@ -1,14 +1,17 @@ from pypy.objspace.flow.model import Constant, SpaceOperation from pypy.annotation.model import SomeInteger from pypy.rpython.memory.gc.marksweep import MarkSweepGC -from pypy.rpython.memory.gctransform.test.test_transform import rtype +from pypy.rpython.memory.gctransform.test.test_transform import rtype, \ + rtype_and_transform from pypy.rpython.memory.gctransform.transform import GcHighLevelOp -from pypy.rpython.memory.gctransform.framework import FrameworkGCTransformer, CollectAnalyzer +from pypy.rpython.memory.gctransform.framework import FrameworkGCTransformer, \ + CollectAnalyzer, find_initializing_stores from pypy.rpython.lltypesystem import lltype from pypy.rpython.rtyper import LowLevelOpList from pypy.translator.c.gc import FrameworkGcPolicy from pypy.translator.translator import TranslationContext, graphof from pypy.translator.unsimplify import varoftype +from pypy.translator.exceptiontransform import ExceptionTransformer from pypy import conftest import py @@ -119,3 +122,45 @@ [varoftype(ARRAYPTR2), varoftype(lltype.Signed), Constant('b', lltype.Void), varoftype(PTR_TYPE2)], varoftype(lltype.Void))) + +def test_find_initializing_stores(): + + class A(object): + pass + class B(object): + pass + def f(): + a = A() + b = B() + b.a = a + t = rtype(f, []) + etrafo = ExceptionTransformer(t) + graphs = etrafo.transform_completely() + collect_analyzer = CollectAnalyzer(t) + init_stores = find_initializing_stores(collect_analyzer, t.graphs[0]) + assert len(init_stores) == 1 + +def test_find_initializing_stores_across_blocks(): + + class A(object): + pass + class B(object): + pass + def f(x): + a1 = A() + a2 = A() + a = A() + b = B() + b.a = a + if x: + b.b = a1 + b.c = a2 + else: + b.c = a1 + b.b = a2 + t = rtype(f, [int]) + etrafo = ExceptionTransformer(t) + graphs = etrafo.transform_completely() + collect_analyzer = CollectAnalyzer(t) + init_stores = find_initializing_stores(collect_analyzer, t.graphs[0]) + assert len(init_stores) == 5 Modified: pypy/dist/pypy/rpython/memory/gctransform/test/test_transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/test/test_transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/test/test_transform.py Wed Oct 24 13:15:37 2007 @@ -149,6 +149,8 @@ t.buildannotator().build_types(func, inputtypes) if specialize: t.buildrtyper().specialize() + if conftest.option.view: + t.view() return t def rtype_and_transform(func, inputtypes, transformcls, specialize=True, check=True): From niko at codespeak.net Wed Oct 24 13:31:13 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Wed, 24 Oct 2007 13:31:13 +0200 (CEST) Subject: [pypy-svn] r47816 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024113113.88F3C80DA@code0.codespeak.net> Author: niko Date: Wed Oct 24 13:31:12 2007 New Revision: 47816 Modified: pypy/dist/pypy/lang/smalltalk/classtable.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: implemnt NEW and NEW_WITH_ARG Modified: pypy/dist/pypy/lang/smalltalk/classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/classtable.py Wed Oct 24 13:31:12 2007 @@ -26,9 +26,9 @@ meta_nm = cls_nm + "Class" meta_super_nm = super_cls_nm + "Class" metacls = define_core_cls( - meta_nm, W_MetaClass(None, classtable[meta_super_nm], meta_nm)) + meta_nm, W_MetaClass(None, classtable[meta_super_nm], name=meta_nm)) define_core_cls( - cls_nm, W_Class(metacls, classtable[super_cls_nm], cls_nm)) + cls_nm, W_Class(metacls, classtable[super_cls_nm], name=cls_nm)) w_ProtoObjectClass.w_superclass = w_Class for nm, w_cls_obj in classtable.items(): if w_cls_obj.ismetaclass(): Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Wed Oct 24 13:31:12 2007 @@ -114,6 +114,8 @@ STRING_AT_PUT = 64 OBJECT_AT = 68 OBJECT_AT_PUT = 69 +NEW = 70 +NEW_WITH_ARG = 71 def common_at(stack): [w_idx, w_obj] = stack @@ -185,6 +187,23 @@ w_rcvr.setnamedvar(idx, w_val) return w_val + at primitive(NEW) + at stack(1) +def func(stack): + [w_cls] = stack + if not isinstance(w_cls, model.W_Class) or w_cls.isindexable(): + raise PrimitiveFailedError() + return w_cls.new() + + at primitive(NEW_WITH_ARG) + at stack(2) +def func(stack): + [w_size, w_cls] = stack + if not isinstance(w_cls, model.W_Class) or not w_cls.isindexable(): + raise PrimitiveFailedError() + size = unwrap_int(w_size) + return w_cls.new(size) + # ___________________________________________________________________________ # Boolean Primitives Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Wed Oct 24 13:31:12 2007 @@ -102,6 +102,21 @@ for i in range(len(exp)): assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i]) +def test_new(): + w_res = prim(p.NEW, [ct.w_Object]) + assert w_res.w_class == ct.w_Object + +def test_invalid_new(): + prim_fails(p.NEW, [ct.w_ByteString]) + +def test_new_with_arg(): + w_res = prim(p.NEW_WITH_ARG, [ct.w_ByteString, 20]) + assert w_res.w_class == ct.w_ByteString + assert w_res.size() == 20 + +def test_invalid_new_with_arg(): + prim_fails(p.NEW_WITH_ARG, [ct.w_Object, 20]) + def test_boolean(): assert prim(p.LESSTHAN, [1,2]) == fimg.w_true assert prim(p.GREATERTHAN, [3,4]) == fimg.w_false From cfbolz at codespeak.net Wed Oct 24 13:43:49 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 24 Oct 2007 13:43:49 +0200 (CEST) Subject: [pypy-svn] r47817 - in pypy/dist/pypy/rpython/memory/gctransform: . test Message-ID: <20071024114349.237D780DA@code0.codespeak.net> Author: cfbolz Date: Wed Oct 24 13:43:48 2007 New Revision: 47817 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py Log: humpf, fix bugs so that integer stores work again Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Wed Oct 24 13:43:48 2007 @@ -41,8 +41,10 @@ if op.args[0] in mallocvars: mallocvars[op.result] = True elif op.opname in ("setfield", "setarrayitem", "setinteriorfield"): + TYPE = op.args[-1].concretetype if (op.args[0] in mallocvars and - op.args[-1].concretetype.TO._gckind == "gc"): + isinstance(TYPE, lltype.Ptr) and + TYPE.TO._gckind == "gc"): result[op] = True else: if collect_analyzer.analyze(op): Modified: pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py Wed Oct 24 13:43:48 2007 @@ -65,6 +65,7 @@ class WriteBarrierTransformer(FrameworkGCTransformer): + initializing_stores = {} GC_PARAMS = {} class GCClass(MarkSweepGC): needs_write_barrier = True @@ -133,6 +134,7 @@ a = A() b = B() b.a = a + b.b = 1 t = rtype(f, []) etrafo = ExceptionTransformer(t) graphs = etrafo.transform_completely() From tverwaes at codespeak.net Wed Oct 24 14:02:52 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 14:02:52 +0200 (CEST) Subject: [pypy-svn] r47818 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024120252.6AA9980DD@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 14:02:51 2007 New Revision: 47818 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_model.py Log: (toon, armin) fixed supersend and singleextended and secondextended Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Wed Oct 24 14:02:51 2007 @@ -120,15 +120,14 @@ def _sendSelfSelector(self, selector, argcount, interp): receiver = self.peek(argcount) self._sendSelector(selector, argcount, interp, - receiver.w_class) + receiver, receiver.w_class) def _sendSuperSelector(self, selector, argcount, interp): - receiver = self.peek(argcount) - self._sendSelector(selector, argcount, interp, - receiver.w_class.w_superclass) + self._sendSelector(selector, argcount, interp, self.receiver, + self.method.w_compiledin.w_superclass) - def _sendSelector(self, selector, argcount, interp, receiverclass): - receiver = self.peek(argcount) + def _sendSelector(self, selector, argcount, interp, + receiver, receiverclass): method = receiverclass.lookup(selector) assert method if method.primitive: @@ -221,7 +220,10 @@ self._sendSuperSelector(selector, argcount, interp) def secondExtendedSendBytecode(self, interp): - raise MissingBytecode + descriptor = self.getByte() + selector = self.method.literals[descriptor & 63] + argcount = descriptor >> 6 + self._sendSelfSelector(selector, argcount, interp) def popStackBytecode(self, interp): self.pop() Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Wed Oct 24 14:02:51 2007 @@ -93,10 +93,11 @@ The trailer has two variant formats. In the first variant, the last byte is at least 252 and the last four bytes represent a source pointer into one of the sources files (see #sourcePointer). In the second variant, the last byte is less than 252, and the last several bytes are a compressed version of the names of the method's temporary variables. The number of bytes used for this purpose is the value of the last byte in the method. """ - def __init__(self, w_class, size, - bytes="", argsize=0, tempsize=0, primitive=0): + def __init__(self, w_class, size, bytes="", argsize=0, + tempsize=0, primitive=0, w_compiledin=None): W_Object.__init__(self, w_class) self.literals = [None] * size + self.w_compiledin = w_compiledin self.bytes = bytes self.argsize = argsize self.tempsize = tempsize @@ -155,6 +156,10 @@ else: return None + def installmethod(self, selector, method): + self.methoddict[selector] = method + method.w_compiledin = self + class W_MetaClass(W_Class): def ismetaclass(self): return True Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Wed Oct 24 14:02:51 2007 @@ -200,7 +200,8 @@ (returnFalse, interpreter.Interpreter.FALSE), (returnNil, interpreter.Interpreter.NIL), (returnTopFromMethod, interpreter.Interpreter.ONE) ]: - w_class.methoddict["foo"] = model.W_CompiledMethod(None, 0, pushConstantOneBytecode + bytecode) + w_class.installmethod("foo", + model.W_CompiledMethod(None, 0, pushConstantOneBytecode + bytecode)) interp = new_interpreter(bytecodes) interp.activeContext.method.literals = ["foo"] interp.activeContext.push(w_object) @@ -224,7 +225,7 @@ def test_fibWithArgument(): bytecode = ''.join(map(chr, [ 16, 119, 178, 154, 118, 164, 11, 112, 16, 118, 177, 224, 112, 16, 119, 177, 224, 176, 124 ])) w_class = model.W_Class(None, None) - w_class.methoddict["fib:"] = model.W_CompiledMethod(None, 1, bytecode, 1) + w_class.installmethod("fib:", model.W_CompiledMethod(None, 1, bytecode, 1)) w_class.methoddict["fib:"].literals[0] = "fib:" w_object = w_class.new() interp = new_interpreter(sendLiteralSelectorBytecode(16) + returnTopFromMethod) @@ -238,7 +239,7 @@ w_smallintclass = model.W_Class(None, None) prim_meth = model.W_CompiledMethod(None, 0, "", argsize=1, primitive=primitives.SUBTRACT) - w_smallintclass.methoddict["sub"] = prim_meth + w_smallintclass.installmethod("sub", prim_meth) w_50 = model.W_SmallInteger(w_smallintclass, 50) w_50.w_class = w_smallintclass w_8 = model.W_SmallInteger(w_smallintclass, 8) @@ -386,7 +387,7 @@ def test_callPrimitiveAndPush_fallback(): interp = new_interpreter(bytecodePrimAdd) w_class = model.W_Class(None, None) - w_class.methoddict["+"] = model.W_CompiledMethod(None, 1, "", 1) + w_class.installmethod("+", model.W_CompiledMethod(None, 1, "", 1)) w_object = w_class.new() interp.activeContext.push(w_object) interp.activeContext.push(interp.ONE) @@ -417,8 +418,35 @@ sendBytecodesTest(w_class, w_object, singleExtendedSendBytecode + chr(0)) def test_singleExtendedSuperBytecode(): - w_super = model.W_Class(None, None) + w_supersuper = model.W_Class(None, None) + w_super = model.W_Class(None, w_supersuper) w_class = model.W_Class(None, w_super) w_object = w_class.new() - sendBytecodesTest(w_super, w_object, singleExtendedSuperBytecode + chr(0)) - + # first call method installed in w_class + bytecodes = singleExtendedSendBytecode + chr(0) + # which does a call to its super + w_class.installmethod("foo", + model.W_CompiledMethod(None, 0, singleExtendedSuperBytecode + chr(0))) + # and that one again to its super + w_super.installmethod("foo", + model.W_CompiledMethod(None, 0, singleExtendedSuperBytecode + chr(0))) + w_supersuper.installmethod("foo", + model.W_CompiledMethod(None, 0, "")) + w_class.methoddict["foo"].literals = ["foo"] + w_super.methoddict["foo"].literals = ["foo"] + interp = new_interpreter(bytecodes) + interp.activeContext.method.literals = ["foo"] + interp.activeContext.push(w_object) + for w_specificclass in [w_class, w_super, w_supersuper]: + callerContext = interp.activeContext + interp.step() + assert interp.activeContext.sender == callerContext + assert interp.activeContext.stack == [] + assert interp.activeContext.receiver == w_object + assert interp.activeContext.method == w_specificclass.methoddict["foo"] + assert callerContext.stack == [] + +def test_secondExtendedSendBytecode(): + w_class = model.W_Class(None, None) + w_object = w_class.new() + sendBytecodesTest(w_class, w_object, secondExtendedSendBytecode + chr(0)) Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Wed Oct 24 14:02:51 2007 @@ -66,4 +66,11 @@ assert w_class.lookup("zork") == None assert w_subclass.lookup("foo") == 3 assert w_subclass.lookup("bar") == 2 - assert w_subclass.lookup("zork") == None \ No newline at end of file + assert w_subclass.lookup("zork") == None + +def test_w_compiledin(): + w_super = model.W_Class(None,None) + w_class = model.W_Class(None,w_super) + w_super.installmethod("foo", + model.W_CompiledMethod(None, 0)) + assert w_class.lookup("foo").w_compiledin == w_super From akuhn at codespeak.net Wed Oct 24 14:55:49 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Wed, 24 Oct 2007 14:55:49 +0200 (CEST) Subject: [pypy-svn] r47824 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024125549.15B28811D@code0.codespeak.net> Author: akuhn Date: Wed Oct 24 14:55:48 2007 New Revision: 47824 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (cfbolz, akuhn) changed internal representation of W_PointersObject to use one list only, as on image loading there is no clue about instVarSize (the file just specifies the total size) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Wed Oct 24 14:55:48 2007 @@ -23,23 +23,25 @@ """ The normal object """ def __init__(self, w_class, size=0): W_Object.__init__(self, w_class) - self.named_vars = [None] * w_class.instvarsize - self.indexed_vars = [None] * size + self.vars = [None] * (w_class.instvarsize + size) def getnamedvar(self, index): - return self.named_vars[index] + if not index < self.w_class.instvarsize: raise IndexError + return self.vars[index] def setnamedvar(self, index, w_value): - self.named_vars[index] = w_value + if not index < self.w_class.instvarsize: raise IndexError + self.vars[index] = w_value def size(self): - return len(self.indexed_vars) + return len(self.vars) - self.w_class.instvarsize def getindexedvar(self, index): - return self.indexed_vars[index] + return self.vars[index + self.w_class.instvarsize] def setindexedvar(self, index, w_value): - self.indexed_vars[index] = w_value + self.vars[index + self.w_class.instvarsize] = w_value + class W_BytesObject(W_Object): def __init__(self, w_class, size): @@ -72,7 +74,6 @@ class W_CompiledMethod(W_Object): """My instances are methods suitable for interpretation by the virtual machine. This is the only class in the system whose instances intermix both indexable pointer fields and indexable integer fields. - The current format of a CompiledMethod is as follows: header (4 bytes) @@ -90,7 +91,6 @@ (index 28) 1 bit: high-bit of primitive number (#primitive) (index 29) 1 bit: flag bit, ignored by the VM (#flag) - The trailer has two variant formats. In the first variant, the last byte is at least 252 and the last four bytes represent a source pointer into one of the sources files (see #sourcePointer). In the second variant, the last byte is less than 252, and the last several bytes are a compressed version of the names of the method's temporary variables. The number of bytes used for this purpose is the value of the last byte in the method. """ def __init__(self, w_class, size, bytes="", argsize=0, Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Wed Oct 24 14:55:48 2007 @@ -126,9 +126,9 @@ assert interp.activeContext.stack == [] for test_index in range(8): if test_index == index: - assert w_object.named_vars[test_index] == interp.TRUE + assert w_object.getnamedvar(test_index) == interp.TRUE else: - assert w_object.named_vars[test_index] == None + assert w_object.getnamedvar(test_index) == None def test_storeAndPopTemporaryVariableBytecode(): for index in range(8): @@ -367,9 +367,9 @@ assert interp.activeContext.stack == [] for test_index in range(8): if test_index == index: - assert w_object.named_vars[test_index] == interp.TRUE + assert w_object.getnamedvar(test_index) == interp.TRUE else: - assert w_object.named_vars[test_index] == None + assert w_object.getnamedvar(test_index) == None def test_extendedStoreAndPopBytecode1(): for index in range(8): From akuhn at codespeak.net Wed Oct 24 15:02:58 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Wed, 24 Oct 2007 15:02:58 +0200 (CEST) Subject: [pypy-svn] r47825 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024130258.41F3D812A@code0.codespeak.net> Author: akuhn Date: Wed Oct 24 15:02:56 2007 New Revision: 47825 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py Log: (akuhn, cfbolz) filling in the date for pointerobjects and compiledmethods Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Wed Oct 24 15:02:56 2007 @@ -1,5 +1,14 @@ import py import struct +from pypy.lang.smalltalk import model +from pypy.lang.smalltalk import fakeimage +from pypy.rlib import objectmodel + +def int2str(integer): + return (chr((integer & 0xff000000) >> 24) + + chr((integer & 0x00ff0000) >> 16) + + chr((integer & 0x0000ff00) >> 8) + + chr((integer & 0x000000ff))) # ____________________________________________________________ # @@ -68,6 +77,14 @@ def init_g_objects(self): for chunk in self.chunks.itervalues(): chunk.as_g_object(self) + + def init_w_objects(self): + for chunk in self.chunks.itervalues(): + chunk.g_object.init_w_object() + + def fillin_w_objects(self): + for chunk in self.chunks.itervalues(): + chunk.g_object.fillin_w_object() def read_header(self): version = self.stream.next() @@ -160,6 +177,7 @@ self.owner = reader self.value = value self.size = -1 + self.w_object = fakeimage.small_int(value) def initialize(self, chunk, reader): self.owner = reader @@ -167,8 +185,10 @@ self.hash12 = chunk.hash12 self.format = chunk.format self.init_class(chunk) - self.init_data(chunk) + self.init_data(chunk) # for pointers + self.chunk = chunk # for bytes, words and compiledmethod self.w_object = None + self.init_w_object() def init_class(self, chunk): if chunk.iscompact(): @@ -199,36 +219,73 @@ return self.format < 5 #TODO, what about compiled methods? def init_w_object(self): - from pypy.lang.smalltalk import model - #if self.classdescription: - # print self.format - # return None - if self.format == 0: # no instvars, non-indexed - assert self.size == 0 - return model.W_PointersObject(size = 0) - elif self.format == 1: # instvars, non-indexed - return model.W_PointersObject(size = self.size) - elif self.format == 2: # no instvars, indexed - return model.W_PointersObject(size = self.size) - elif self.format == 3: # instvars, indexed - return model.W_PointersObject(size = self.size) - elif self.format == 4: # XXX W_WeakPointersObject - return model.W_PointersObject(size = self.size) - elif self.format == 5: - raise CorruptImageError("Unknown format 5") - elif self.format == 6: - return model.W_WordsObject(size = self.size) - elif self.format == 7: - raise CorruptImageError("Unknown format 7, no 64-bit support yet :-)") - elif 8 <= self.format <= 11: - byte_size = self.size * 4 + self.format & 3 - return model.W_BytesObject(size = byte_size) - elif 12 <= self.format <= 15: - return model.W_CompiledMethod(size = 0) #XXX to be figured out how to get - #both the size of literals and the size of bytes in bytecode!!! - else: - assert 0, "not reachable" + if self.w_object is None: + if self.format < 5: + self.w_object = objectmodel.instantiate(model.W_PointersObject) + elif self.format == 5: + raise CorruptImageError("Unknown format 5") + elif self.format == 6: + self.w_object = objectmodel.instantiate(model.W_WordsObject) + elif self.format == 7: + raise CorruptImageError("Unknown format 7, no 64-bit support yet :-)") + elif 8 <= self.format <= 11: + self.w_object = objectmodel.instantiate(model.W_BytesObject) + elif 12 <= self.format <= 15: + self.w_object = objectmodel.instantiate(model.W_CompiledMethod) + else: + assert 0, "not reachable" + return self.w_object + + def fillin_w_object(self): + # below we are using an RPython idiom to 'cast' self.w_object + # and pass the casted reference to the fillin_* methods + casted = self.w_object + case = type(casted) + if case == model.W_PointersObject: + self.fillin_poingersobject(casted) + elif case == model.W_WordsObject: + self.fillin_wordsobject(casted) + elif case == model.W_BytesObject: + self.fillin_bytesobject(casted) + elif case == model.W_CompiledMethod: + self.fillin_compiledmethod(casted) + + def fillin_pointersobject(self, w_pointersobject): + w_pointersobject.vars = [g_object.w_object for g_object in self.pointers] + w_pointersobject.w_class = self.g_class.w_object + def fillin_wordsobject(self, w_wordsobject): + w_wordsobject.w_class = self.g_class.w_object + + def fillin_bytesobject(self, w_bytesobject): + w_bytesobject.w_class = self.g_class.w_object + + def fillin_compiledmethod(self, w_compiledmethod): + header = chunk.data[0] + #(index 0) 9 bits: main part of primitive number (#primitive) + #(index 9) 8 bits: number of literals (#numLiterals) + #(index 17) 1 bit: whether a large frame size is needed (#frameSize) + #(index 18) 6 bits: number of temporary variables (#numTemps) + #(index 24) 4 bits: number of arguments to the method (#numArgs) + #(index 28) 1 bit: high-bit of primitive number (#primitive) + #(index 29) 1 bit: flag bit, ignored by the VM (#flag) + highbit, numargs, tempsize, islarge, literalsize, primitive = ( + splitbits(header, [1,4,6,1,8,9])) + primitive = primitive + (highbit << 10) + assert (1 + literalsize) < len(chunk.data) + l = [] + for each in chunk.data[(1 + literalsize):]: + l.append(int2str(each)) + l[-1] = l[-1][:-(self.format & 3)] # omit odd bytes + bytes = "".join(l) + w_compiledmethod.__init__( + w_class = self.g_class.w_object, + size = self.literalsize, + bytes = bytes, + argsize = numargs, + tempsize = tempsize, + primitive = primitive) + class ImageChunk(object): Modified: pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py Wed Oct 24 15:02:56 2007 @@ -167,5 +167,11 @@ def test_init_g_objects(): example.init_g_objects() + +def test_init_w_objects(): + example.init_w_objects() + +def test_fillin_w_objects(): + example.fillin_w_objects() \ No newline at end of file From fijal at codespeak.net Wed Oct 24 15:04:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 24 Oct 2007 15:04:54 +0200 (CEST) Subject: [pypy-svn] r47826 - in pypy/dist/pypy/module/zipimport: . test Message-ID: <20071024130454.852F6812A@code0.codespeak.net> Author: fijal Date: Wed Oct 24 15:04:54 2007 New Revision: 47826 Added: pypy/dist/pypy/module/zipimport/test/__init__.py (contents, props changed) Modified: pypy/dist/pypy/module/zipimport/ (props changed) pypy/dist/pypy/module/zipimport/__init__.py pypy/dist/pypy/module/zipimport/app_zipimport.py pypy/dist/pypy/module/zipimport/interp_zipimport.py pypy/dist/pypy/module/zipimport/test/ (props changed) pypy/dist/pypy/module/zipimport/test/test_zipimport.py Log: More or less working zipimport. Packages are not working though. Modified: pypy/dist/pypy/module/zipimport/__init__.py ============================================================================== --- pypy/dist/pypy/module/zipimport/__init__.py (original) +++ pypy/dist/pypy/module/zipimport/__init__.py Wed Oct 24 15:04:54 2007 @@ -10,6 +10,6 @@ interpleveldefs = {'zipimporter':'interp_zipimport.W_ZipImporter'} appleveldefs = { - 'ZipImporterError' : 'app_zipimport.ZipImporterError', + 'ZipImportError' : 'app_zipimport.ZipImportError', } Modified: pypy/dist/pypy/module/zipimport/app_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/app_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/app_zipimport.py Wed Oct 24 15:04:54 2007 @@ -1,3 +1,3 @@ -class ZipImporterError(Exception): +class ZipImportError(Exception): pass Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/interp_zipimport.py Wed Oct 24 15:04:54 2007 @@ -8,9 +8,20 @@ from pypy.interpreter.eval import Code from pypy.interpreter.module import Module from pypy.module.__builtin__ import importing +from pypy.rlib.unroll import unrolling_iterable import os import stat +zip_importer_cache = {} + +ENUMERATE_EXTS = unrolling_iterable( + [(True, True, os.path.sep + '__init__.pyc'), + (True, True, os.path.sep + '__init__.pyo'), + (False, True, os.path.sep + '__init__.py'), + (True, False, '.pyc'), + (True, False, '.pyo'), + (False, False, '.py')] + class W_ZipImporter(Wrappable): def __init__(self, space, name, w_dir, w_zipfile): self.space = space @@ -24,14 +35,31 @@ space.call(space.getattr(w_dir, space.wrap('namelist')), space.newlist([])))] - def import_py_file(self, space, modname, filename, w_buf): + def import_py_file(self, space, modname, filename, w_buf, pkgpath): buf = space.str_w(w_buf) w = space.wrap w_mod = w(Module(space, w(modname))) - importing._prepare_module(space, w_mod, filename, None) - return importing.load_source_module(space, w(modname), w_mod, + real_name = self.name + os.path.sep + filename + importing._prepare_module(space, w_mod, real_name, pkgpath) + result = importing.load_source_module(space, w(modname), w_mod, filename, buf, write_pyc=False) - return w_mod + space.setattr(w_mod, w('__loader__'), space.wrap(self)) + return result + + def import_pyc_file(self, space, modname, filename, w_buf, pkgpath): + w = space.wrap + buf = space.str_w(w_buf) + magic = importing._get_long(buf[:4]) + timestamp = importing._get_long(buf[4:8]) + buf = buf[8:] # XXX ugly copy, should use sequential read instead + w_mod = w(Module(space, w(modname))) + real_name = self.name + os.path.sep + filename + importing._prepare_module(space, w_mod, real_name, pkgpath) + result = importing.load_compiled_module(space, w(modname), w_mod, + filename, magic, timestamp, + buf) + space.setattr(w_mod, w('__loader__'), space.wrap(self)) + return result def get_module(self, space, name): w = space.wrap @@ -42,20 +70,48 @@ if not e.match(space, space.w_KeyError): # should never happen raise e + return space.w_None def find_module(self, space, import_name, w_path=None): import_name = import_name.replace('.', os.path.sep) - for ext in ['.py', 'pyc', '.pyo']: - if self.get_module(space, import_name + ext): + for _, _, ext in ENUMERATE_EXTS: + if space.is_true(self.get_module(space, import_name + ext)): return space.wrap(self) find_module.unwrap_spec = ['self', ObjSpace, str, W_Root] def load_module(self, space, name): w = space.wrap - filename = name.replace('.', os.path.sep) + '.py' - w_buf = space.call(space.getattr(self.w_dir, w('read')), - space.newlist([w(filename)])) - return self.import_py_file(space, name, filename, w_buf) + w_modules = space.sys.get('modules') + try: + return space.getitem(w_modules, w(name)) + except OperationError, e: + pass + filename = name.replace('.', os.path.sep) + w_ZipImportError = space.getattr(space.getbuiltinmodule('zipimport'), + w('ZipImportError')) + last_exc = None + for compiled, is_package, ext in ENUMERATE_EXTS: + try: + w_buf = space.call(space.getattr(self.w_dir, w('read')), + space.newlist([w(filename + ext)])) + if is_package: + pkgpath = filename + os.path.sep + else: + pkgpath = None + if compiled: + return self.import_pyc_file(space, name, filename + ext, + w_buf, pkgpath) + else: + return self.import_py_file(space, name, filename + ext, + w_buf, pkgpath) + except OperationError, e: + last_exc = e + w_mods = space.sys.get('modules') + space.call_method(w_mods,'pop', w(name), space.w_None) + if last_exc: + raise OperationError(w_ZipImportError, last_exc.w_value) + # should never happen I think + return space.w_None load_module.unwrap_spec = ['self', ObjSpace, str] def get_data(self, space): @@ -76,6 +132,10 @@ def descr_new_zipimporter(space, w_type, name): try: + return zip_importer_cache[name] + except KeyError: + pass + try: s = os.stat(name) except OSError: return space.w_None @@ -93,7 +153,9 @@ except OperationError: # we catch everything as this function # should not raise return space.w_None - return space.wrap(W_ZipImporter(space, name, w_dir, w_zipfile)) + result = space.wrap(W_ZipImporter(space, name, w_dir, w_zipfile)) + zip_importer_cache[name] = result + return result descr_new_zipimporter.unwrap_spec = [ObjSpace, W_Root, str] Added: pypy/dist/pypy/module/zipimport/test/__init__.py ============================================================================== Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/test/test_zipimport.py Wed Oct 24 15:04:54 2007 @@ -1,10 +1,11 @@ -import imp from pypy.conftest import gettestobjspace import marshal import py import time import struct +from pypy.module.__builtin__.importing import get_pyc_magic, _w_long +from StringIO import StringIO from pypy.tool.udir import udir from zipfile import ZIP_STORED, ZIP_DEFLATED, ZipInfo @@ -13,7 +14,7 @@ """ A bit structurized tests stolen and adapted from cpy's regression tests """ - def make_pyc(cls, co, mtime): + def make_pyc(cls, space, co, mtime): data = marshal.dumps(co) if type(mtime) is type(0.0): # Mac mtimes need a bit of special casing @@ -21,7 +22,13 @@ mtime = int(mtime) else: mtime = int(-0x100000000L + long(mtime)) - pyc = imp.get_magic() + struct.pack(" Author: fijal Date: Wed Oct 24 15:05:44 2007 New Revision: 47827 Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py Log: Don't check in syntax errors. Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/interp_zipimport.py Wed Oct 24 15:05:44 2007 @@ -20,7 +20,7 @@ (False, True, os.path.sep + '__init__.py'), (True, False, '.pyc'), (True, False, '.pyo'), - (False, False, '.py')] + (False, False, '.py')]) class W_ZipImporter(Wrappable): def __init__(self, space, name, w_dir, w_zipfile): From akuhn at codespeak.net Wed Oct 24 15:15:47 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Wed, 24 Oct 2007 15:15:47 +0200 (CEST) Subject: [pypy-svn] r47828 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071024131547.D6EDC80A1@code0.codespeak.net> Author: akuhn Date: Wed Oct 24 15:15:46 2007 New Revision: 47828 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: (akuhn, cfbolz) filling in also word and byte indexed objects Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Wed Oct 24 15:15:46 2007 @@ -255,11 +255,19 @@ w_pointersobject.w_class = self.g_class.w_object def fillin_wordsobject(self, w_wordsobject): + w_wordsobject.words = self.chunk.data w_wordsobject.w_class = self.g_class.w_object def fillin_bytesobject(self, w_bytesobject): + bytes = [] + for each in chunk.data: + bytes.append((integer & 0xff000000) >> 24) + bytes.append((integer & 0xff000000) >> 16) + bytes.append((integer & 0xff000000) >> 8) + bytes.append((integer & 0xff000000)) w_bytesobject.w_class = self.g_class.w_object - + w_bytesobject.bytes = bytes[:-(self.format & 3)] # omit odd bytes + def fillin_compiledmethod(self, w_compiledmethod): header = chunk.data[0] #(index 0) 9 bits: main part of primitive number (#primitive) From tverwaes at codespeak.net Wed Oct 24 15:25:56 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 15:25:56 +0200 (CEST) Subject: [pypy-svn] r47829 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024132556.8F840810E@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 15:25:54 2007 New Revision: 47829 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (armin, toon) added doubleExtendedDoAnythingBytecode and refactored existing tests in order to test it Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Wed Oct 24 15:25:54 2007 @@ -213,7 +213,34 @@ self._sendSelfSelector(selector, argcount, interp) def doubleExtendedDoAnythingBytecode(self, interp): - raise MissingBytecode + second = self.getByte() + third = self.getByte() + opType = second >> 5 + if opType == 0: + # selfsend + self._sendSelfSelector(self.method.literals[third], + second & 31, interp) + elif opType == 1: + # supersend + self._sendSuperSelector(self.method.literals[third], + second & 31, interp) + elif opType == 2: + # pushReceiver + self.push(self.receiver.getnamedvar(third)) + elif opType == 3: + # pushLiteralConstant + self.push(self.method.literals[third]) + elif opType == 4: + # pushLiteralVariable + association = self.method.literals[third] + self.push(association.getnamedvar(1)) + elif opType == 5: + self.receiver.setnamedvar(third, self.top()) + elif opType == 6: + self.receiver.setnamedvar(third, self.pop()) + elif opType == 7: + association = self.method.literals[third] + association.setnamedvar(1,self.top()) def singleExtendedSuperBytecode(self, interp): selector, argcount = self.getExtendedSelectorArgcount() Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Wed Oct 24 15:25:54 2007 @@ -70,69 +70,73 @@ interp.step() assert interp.activeContext.top() == interp.activeContext.receiver -def test_pushReceiverVariableBytecode(): +def test_pushReceiverVariableBytecode(bytecode = (pushReceiverVariableBytecode(0) + + pushReceiverVariableBytecode(1) + + pushReceiverVariableBytecode(2))): w_democlass = model.W_Class(None, None, 3) w_demo = w_democlass.new() w_demo.setnamedvar(0, "egg") w_demo.setnamedvar(1, "bar") w_demo.setnamedvar(2, "baz") - interp = new_interpreter(pushReceiverVariableBytecode(0) + - pushReceiverVariableBytecode(1) + - pushReceiverVariableBytecode(2), - receiver = w_demo) + interp = new_interpreter(bytecode, receiver = w_demo) interp.step() interp.step() interp.step() assert interp.activeContext.stack == ["egg", "bar", "baz"] -def test_pushTemporaryVariableBytecode(): - interp = new_interpreter(pushTemporaryVariableBytecode(0) + - pushTemporaryVariableBytecode(1) + - pushTemporaryVariableBytecode(2)) +def test_pushTemporaryVariableBytecode(bytecode=(pushTemporaryVariableBytecode(0) + + pushTemporaryVariableBytecode(1) + + pushTemporaryVariableBytecode(2))): + interp = new_interpreter(bytecode) interp.activeContext.settemp(2, "temp") interp.step() interp.step() interp.step() assert interp.activeContext.stack == ["foo", "bar", "temp"] -def test_pushLiteralConstantBytecode(): - interp = new_interpreter(pushLiteralConstantBytecode(0) + - pushLiteralConstantBytecode(1) + - pushLiteralConstantBytecode(2)) +def test_pushLiteralConstantBytecode(bytecode=pushLiteralConstantBytecode(0) + + pushLiteralConstantBytecode(1) + + pushLiteralConstantBytecode(2)): + interp = new_interpreter(bytecode) interp.activeContext.method.literals = ["a", "b", "c"] interp.step() interp.step() interp.step() assert interp.activeContext.stack == ["a", "b", "c"] -def test_pushLiteralVariableBytecode(): +def test_pushLiteralVariableBytecode(bytecode=pushLiteralVariableBytecode(0)): w_associationclass = model.W_Class(None, None, 2) w_association = w_associationclass.new() w_association.setnamedvar(0, "mykey") w_association.setnamedvar(1, "myvalue") - interp = new_interpreter(pushLiteralVariableBytecode(0)) + interp = new_interpreter(bytecode) interp.activeContext.method.literals = [w_association] interp.step() assert interp.activeContext.stack == ["myvalue"] -def test_storeAndPopReceiverVariableBytecode(): +def test_storeAndPopReceiverVariableBytecode(bytecode=storeAndPopReceiverVariableBytecode, + popped=True): w_class = model.W_Class(None, None, 8) for index in range(8): w_object = w_class.new() - interp = new_interpreter(pushConstantTrueBytecode + storeAndPopReceiverVariableBytecode(index)) + interp = new_interpreter(pushConstantTrueBytecode + bytecode(index)) interp.activeContext.receiver = w_object interp.step() interp.step() - assert interp.activeContext.stack == [] + if popped: + assert interp.activeContext.stack == [] + else: + assert interp.activeContext.stack == [interp.TRUE] + for test_index in range(8): if test_index == index: assert w_object.getnamedvar(test_index) == interp.TRUE else: assert w_object.getnamedvar(test_index) == None - -def test_storeAndPopTemporaryVariableBytecode(): + +def test_storeAndPopTemporaryVariableBytecode(bytecode=storeAndPopTemporaryVariableBytecode): for index in range(8): - interp = new_interpreter(pushConstantTrueBytecode + storeAndPopTemporaryVariableBytecode(index)) + interp = new_interpreter(pushConstantTrueBytecode + bytecode(index)) interp.activeContext.temps = [None] * 8 interp.step() interp.step() @@ -310,79 +314,42 @@ interp.step() assert interp.activeContext.stack == [] -def test_extendedPushBytecode0(): - w_democlass = model.W_Class(None, None, 3) - w_demo = w_democlass.new() - w_demo.setnamedvar(0, "egg") - w_demo.setnamedvar(1, "bar") - w_demo.setnamedvar(2, "baz") - interp = new_interpreter(extendedPushBytecode + chr((0<<6) + 0) + - extendedPushBytecode + chr((0<<6) + 1) + - extendedPushBytecode + chr((0<<6) + 2), - receiver = w_demo) - interp.step() - interp.step() - interp.step() - assert interp.activeContext.stack == ["egg", "bar", "baz"] +def test_extendedPushBytecode(): + test_pushReceiverVariableBytecode(extendedPushBytecode + chr((0<<6) + 0) + + extendedPushBytecode + chr((0<<6) + 1) + + extendedPushBytecode + chr((0<<6) + 2)) + + test_pushTemporaryVariableBytecode(extendedPushBytecode + chr((1<<6) + 0) + + extendedPushBytecode + chr((1<<6) + 1) + + extendedPushBytecode + chr((1<<6) + 2)) + + test_pushLiteralConstantBytecode(extendedPushBytecode + chr((2<<6) + 0) + + extendedPushBytecode + chr((2<<6) + 1) + + extendedPushBytecode + chr((2<<6) + 2)) -def test_extendedPushBytecode1(): - interp = new_interpreter(extendedPushBytecode + chr((1<<6) + 0) + - extendedPushBytecode + chr((1<<6) + 1) + - extendedPushBytecode + chr((1<<6) + 2)) - interp.activeContext.settemp(2, "temp") - interp.step() - interp.step() - interp.step() - assert interp.activeContext.stack == ["foo", "bar", "temp"] - -def test_extendedPushBytecode2(): - interp = new_interpreter(extendedPushBytecode + chr((2<<6) + 0) + - extendedPushBytecode + chr((2<<6) + 1) + - extendedPushBytecode + chr((2<<6) + 2)) - interp.activeContext.method.literals = ["a", "b", "c"] - interp.step() - interp.step() - interp.step() - assert interp.activeContext.stack == ["a", "b", "c"] + test_pushLiteralVariableBytecode(extendedPushBytecode + chr((3<<6) + 0)) - -def test_extendedPushBytecode3(): +def storeAssociation(bytecode): w_associationclass = model.W_Class(None, None, 2) w_association = w_associationclass.new() w_association.setnamedvar(0, "mykey") w_association.setnamedvar(1, "myvalue") - interp = new_interpreter(extendedPushBytecode + chr((3<<6) + 0)) + interp = new_interpreter(pushConstantOneBytecode + bytecode) interp.activeContext.method.literals = [w_association] interp.step() - assert interp.activeContext.stack == ["myvalue"] + interp.step() + assert w_association.getnamedvar(1) == interp.ONE -def test_extendedStoreAndPopBytecode0(): - w_class = model.W_Class(None, None, 8) - for index in range(8): - w_object = w_class.new() - interp = new_interpreter(pushConstantTrueBytecode + extendedStoreAndPopBytecode + chr((0<<6) + index)) - interp.activeContext.receiver = w_object - interp.step() - interp.step() - assert interp.activeContext.stack == [] - for test_index in range(8): - if test_index == index: - assert w_object.getnamedvar(test_index) == interp.TRUE - else: - assert w_object.getnamedvar(test_index) == None +def test_extendedStoreAndPopBytecode(): + test_storeAndPopReceiverVariableBytecode(lambda index: extendedStoreAndPopBytecode + chr((0<<6) + index)) -def test_extendedStoreAndPopBytecode1(): - for index in range(8): - interp = new_interpreter(pushConstantTrueBytecode + extendedStoreAndPopBytecode + chr((1<<6) + index)) - interp.activeContext.temps = [None] * 8 - interp.step() - interp.step() - assert interp.activeContext.stack == [] - for test_index in range(8): - if test_index == index: - assert interp.activeContext.temps[test_index] == interp.TRUE - else: - assert interp.activeContext.temps[test_index] == None + test_storeAndPopTemporaryVariableBytecode(lambda index: extendedStoreAndPopBytecode + chr((1<<6) + index)) + + py.test.raises(interpreter.IllegalStoreError, + test_storeAndPopTemporaryVariableBytecode, + lambda index: extendedStoreAndPopBytecode + chr((2<<6) + index)) + + storeAssociation(extendedStoreAndPopBytecode + chr((3<<6) + 0)) def test_callPrimitiveAndPush_fallback(): interp = new_interpreter(bytecodePrimAdd) @@ -415,9 +382,9 @@ def test_singleExtendedSendBytecode(): w_class = model.W_Class(None, None) w_object = w_class.new() - sendBytecodesTest(w_class, w_object, singleExtendedSendBytecode + chr(0)) + sendBytecodesTest(w_class, w_object, singleExtendedSendBytecode + chr((0<<5)+0)) -def test_singleExtendedSuperBytecode(): +def test_singleExtendedSuperBytecode(bytecode=singleExtendedSuperBytecode + chr((0<<5) + 0)): w_supersuper = model.W_Class(None, None) w_super = model.W_Class(None, w_supersuper) w_class = model.W_Class(None, w_super) @@ -426,10 +393,10 @@ bytecodes = singleExtendedSendBytecode + chr(0) # which does a call to its super w_class.installmethod("foo", - model.W_CompiledMethod(None, 0, singleExtendedSuperBytecode + chr(0))) + model.W_CompiledMethod(None, 0, bytecode)) # and that one again to its super w_super.installmethod("foo", - model.W_CompiledMethod(None, 0, singleExtendedSuperBytecode + chr(0))) + model.W_CompiledMethod(None, 0, bytecode)) w_supersuper.installmethod("foo", model.W_CompiledMethod(None, 0, "")) w_class.methoddict["foo"].literals = ["foo"] @@ -450,3 +417,27 @@ w_class = model.W_Class(None, None) w_object = w_class.new() sendBytecodesTest(w_class, w_object, secondExtendedSendBytecode + chr(0)) + +def test_doubleExtendedDoAnythinBytecode(): + w_class = model.W_Class(None, None) + w_object = w_class.new() + + sendBytecodesTest(w_class, w_object, doubleExtendedDoAnythingBytecode + chr((0<<5) + 0) + chr(0)) + + test_singleExtendedSuperBytecode(doubleExtendedDoAnythingBytecode + (chr((1<<5) + 0) + chr(0))) + + test_pushReceiverVariableBytecode(doubleExtendedDoAnythingBytecode + chr(2<<5) + chr(0) + + doubleExtendedDoAnythingBytecode + chr(2<<5) + chr(1) + + doubleExtendedDoAnythingBytecode + chr(2<<5) + chr(2)) + + test_pushLiteralConstantBytecode(doubleExtendedDoAnythingBytecode + chr(3<<5) + chr(0) + + doubleExtendedDoAnythingBytecode + chr(3<<5) + chr(1) + + doubleExtendedDoAnythingBytecode + chr(3<<5) + chr(2)) + + test_pushLiteralVariableBytecode(doubleExtendedDoAnythingBytecode + chr(4<<5) + chr(0)) + + test_storeAndPopReceiverVariableBytecode(lambda index: doubleExtendedDoAnythingBytecode + chr(5<<5) + chr(index), False) + + test_storeAndPopReceiverVariableBytecode(lambda index: doubleExtendedDoAnythingBytecode + chr(6<<5) + chr(index)) + + storeAssociation(doubleExtendedDoAnythingBytecode + chr(7<<5) + chr(0)) From fijal at codespeak.net Wed Oct 24 15:28:18 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 24 Oct 2007 15:28:18 +0200 (CEST) Subject: [pypy-svn] r47830 - in pypy/dist/pypy/module/zipimport: . test Message-ID: <20071024132818.A319F80FD@code0.codespeak.net> Author: fijal Date: Wed Oct 24 15:28:15 2007 New Revision: 47830 Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py pypy/dist/pypy/module/zipimport/test/test_zipimport.py Log: Fix package problem. Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/interp_zipimport.py Wed Oct 24 15:28:15 2007 @@ -95,7 +95,7 @@ w_buf = space.call(space.getattr(self.w_dir, w('read')), space.newlist([w(filename + ext)])) if is_package: - pkgpath = filename + os.path.sep + pkgpath = self.name else: pkgpath = None if compiled: Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/test/test_zipimport.py Wed Oct 24 15:28:15 2007 @@ -152,7 +152,6 @@ assert mod(3) == 4 def test_package(self): - skip("Does not work, no idea why...") self.writefile(self, "xx/__init__.py", "") self.writefile(self, "xx/yy.py", "def f(x): return x") mod = __import__("xx", globals(), locals(), ['yy']) From niko at codespeak.net Wed Oct 24 15:46:01 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Wed, 24 Oct 2007 15:46:01 +0200 (CEST) Subject: [pypy-svn] r47831 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024134601.250518114@code0.codespeak.net> Author: niko Date: Wed Oct 24 15:46:00 2007 New Revision: 47831 Modified: pypy/dist/pypy/lang/smalltalk/fakeimage.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: 1. add w_hash to the model 2. add as_oop, inst_var_at primitive 3. rename make_XXX to wrap_XXX for consistency Modified: pypy/dist/pypy/lang/smalltalk/fakeimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/fakeimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/fakeimage.py Wed Oct 24 15:46:00 2007 @@ -4,19 +4,21 @@ # ___________________________________________________________________________ # Utility Methods -def small_int(i): - return model.W_SmallInteger(ct.w_SmallInteger, i) +def wrap_int(i): + if i <= 0x3FFFFFFF and i >= -0x40000000: + return model.W_SmallInteger(ct.w_SmallInteger, i) + raise NotImplementedError def wrap_float(i): return model.W_Float(ct.w_Float,i) -def make_string(str): +def wrap_string(str): w_inst = ct.w_ByteString.new(len(str)) for i in range(len(str)): w_inst.setbyte(i, ord(str[i])) return w_inst -def make_char(c): +def wrap_char(c): return CharacterTable[ord(c)] def ord_w_char(w_c): @@ -45,19 +47,19 @@ # ___________________________________________________________________________ # Global Data -def make_char_table(): +def wrap_char_table(): global CharacterTable def bld_char(i): w_cinst = ct.w_Character.new() - w_cinst.setnamedvar(CHARACTER_VALUE_INDEX, small_int(i)) + w_cinst.setnamedvar(CHARACTER_VALUE_INDEX, wrap_int(i)) return w_cinst CharacterTable = [bld_char(i) for i in range(256)] -make_char_table() +wrap_char_table() w_true = ct.w_True.new() w_false = ct.w_False.new() w_nil = ct.w_UndefinedObject.new() -w_mone = small_int(-1) -w_zero = small_int(0) -w_one = small_int(1) -w_two = small_int(2) +w_mone = wrap_int(-1) +w_zero = wrap_int(0) +w_one = wrap_int(1) +w_two = wrap_int(2) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Wed Oct 24 15:46:00 2007 @@ -2,10 +2,11 @@ class W_Object: def __init__(self, w_class): self.w_class = w_class - + self.w_hash = None # XXX initial value? + def size(self): return 0 - + def instvarsize(self): return self.w_class.instvarsize @@ -126,12 +127,38 @@ self.instvarsize = instvarsize self.format = format - def isindexable(self): + # _______________________________________________________________ + # Methods for querying the format word, taken from the blue book: + # + # included so that we can reproduce code from the reference impl + # more easily + + def ispointers(self): + " True if instances of this class have data stored as pointers " + return self.format in (POINTERS, VAR_POINTERS) + + def iswords(self): + " True if instances of this class have data stored as numerical words " + return self.format in (POINTERS, VAR_POINTERS, WORDS) + + def isbytes(self): + " True if instances of this class have data stored as numerical bytes " + return self.format == BYTES + + def isvariable(self): + " True if instances of this class have indexed inst variables " return self.format in (VAR_POINTERS, WORDS, BYTES) + def instsize(self): + " Number of named instance variables for each instance of this class " + return self.instvarsize + def ismetaclass(self): return False + # _______________________________________________________________ + # Methods for querying the format word, taken from the blue book: + def new(self, size=0): if self.format == POINTERS: assert size == 0 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Wed Oct 24 15:46:00 2007 @@ -21,6 +21,15 @@ def is_valid_index(idx, w_obj): return idx >= 0 and idx < w_obj.size() +def subscript(idx, w_obj): + if isinstance(w_obj, model.W_PointersObject): + return w_obj.getindexedvar(idx) + elif isinstance(w_obj, model.W_WordsObject): + return fimg.wrap_int(w_obj.getword(idx)) + elif isinstance(w_obj, model.W_BytesObject): + return fimg.wrap_int(w_obj.getbyte(idx)) + raise PrimitiveFailedError() + # ___________________________________________________________________________ # Primitive table: it is filled in at initialization time with the # primitive functions. Each primitive function takes a single argument, @@ -68,7 +77,7 @@ if res > 1073741823: raise PrimitiveFailedError() if res < -1073741824: raise PrimitiveFailedError() - w_res = fimg.small_int(res) + w_res = fimg.wrap_int(res) return w_res prim_table[code] = func @@ -116,6 +125,8 @@ OBJECT_AT_PUT = 69 NEW = 70 NEW_WITH_ARG = 71 +INST_VAR_AT = 73 +AS_OOP = 75 def common_at(stack): [w_idx, w_obj] = stack @@ -148,7 +159,7 @@ @stack(1) def func(stack): [w_obj] = stack - if not w_obj.w_class.isindexable(): + if not w_obj.w_class.isvariable(): raise PrimitiveFailedError() return w_obj.size() @@ -187,11 +198,12 @@ w_rcvr.setnamedvar(idx, w_val) return w_val + @primitive(NEW) @stack(1) def func(stack): [w_cls] = stack - if not isinstance(w_cls, model.W_Class) or w_cls.isindexable(): + if not isinstance(w_cls, model.W_Class) or w_cls.isvariable(): raise PrimitiveFailedError() return w_cls.new() @@ -199,11 +211,36 @@ @stack(2) def func(stack): [w_size, w_cls] = stack - if not isinstance(w_cls, model.W_Class) or not w_cls.isindexable(): + if not isinstance(w_cls, model.W_Class) or not w_cls.isvariable(): raise PrimitiveFailedError() size = unwrap_int(w_size) return w_cls.new(size) + at primitive(INST_VAR_AT) + at stack(2) +def func(stack): + # I *think* this is the correct behavior, but I'm not quite sure. + # Might be restricted to fixed length fields? + [w_idx, w_rcvr] = stack + idx = unwrap_int(w_idx) + w_cls = w_rcvr.w_class + if idx < 0: + raise PrimitiveFailedError() + if idx < w_cls.instvarsize: + return w_rcvr.getnamedvar(idx) + idx -= w_cls.instvarsize + if idx < w_rcvr.size(): + return subscript(idx, w_rcvr) + raise PrimitiveFailedError() + + at primitive(AS_OOP) + at stack(1) +def func(stack): + [w_rcvr] = stack + if isinstance(w_rcvr, model.W_SmallInteger): + raise PrimitiveFailedError() + return w_rcvr.w_hash + # ___________________________________________________________________________ # Boolean Primitives Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Wed Oct 24 15:46:00 2007 @@ -11,11 +11,11 @@ self.stack = stack def wrap(x): - if isinstance(x, int): return fimg.small_int(x) + if isinstance(x, int): return fimg.wrap_int(x) if isinstance(x, float): return fimg.wrap_float(x) if isinstance(x, model.W_Object): return x - if isinstance(x, str) and len(x) == 1: return fimg.make_char(x) - if isinstance(x, str): return fimg.make_string(x) + if isinstance(x, str) and len(x) == 1: return fimg.wrap_char(x) + if isinstance(x, str): return fimg.wrap_string(x) raise NotImplementedError def mock(stack): @@ -116,6 +116,22 @@ def test_invalid_new_with_arg(): prim_fails(p.NEW_WITH_ARG, [ct.w_Object, 20]) + +def test_inst_var_at(): + # I am not entirely sure if this is what this primitive is + # supposed to do, so the test may be bogus: + w_v = prim(p.INST_VAR_AT, ["q", fimg.CHARACTER_VALUE_INDEX]) + assert w_v.value == ord("q") + w_v = prim(p.INST_VAR_AT, ["abc", 1]) + assert w_v.value == ord("b") + +def test_as_oop(): + w_obj = model.W_Class(None, None, 0, format=model.POINTERS).new() + w_obj.w_hash = wrap(22) + assert prim(p.AS_OOP, [w_obj]).value == 22 + +def test_as_oop_not_applicable_to_int(): + prim_fails(p.AS_OOP, [22]) def test_boolean(): assert prim(p.LESSTHAN, [1,2]) == fimg.w_true From fijal at codespeak.net Wed Oct 24 16:07:31 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 24 Oct 2007 16:07:31 +0200 (CEST) Subject: [pypy-svn] r47832 - in pypy/dist/pypy/module/zipimport: . test Message-ID: <20071024140731.AB2C08108@code0.codespeak.net> Author: fijal Date: Wed Oct 24 16:07:30 2007 New Revision: 47832 Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py pypy/dist/pypy/module/zipimport/test/test_zipimport.py Log: Implement functions Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/interp_zipimport.py Wed Oct 24 16:07:30 2007 @@ -70,15 +70,17 @@ if not e.match(space, space.w_KeyError): # should never happen raise e - return space.w_None def find_module(self, space, import_name, w_path=None): import_name = import_name.replace('.', os.path.sep) for _, _, ext in ENUMERATE_EXTS: - if space.is_true(self.get_module(space, import_name + ext)): + if self.get_module(space, import_name + ext): return space.wrap(self) find_module.unwrap_spec = ['self', ObjSpace, str, W_Root] + def mangle(self, name): + return name.replace('.', os.path.sep) + def load_module(self, space, name): w = space.wrap w_modules = space.sys.get('modules') @@ -86,14 +88,13 @@ return space.getitem(w_modules, w(name)) except OperationError, e: pass - filename = name.replace('.', os.path.sep) + filename = self.mangle(name) w_ZipImportError = space.getattr(space.getbuiltinmodule('zipimport'), w('ZipImportError')) last_exc = None for compiled, is_package, ext in ENUMERATE_EXTS: try: - w_buf = space.call(space.getattr(self.w_dir, w('read')), - space.newlist([w(filename + ext)])) + w_buf = self._get_data(space, w(filename + ext)) if is_package: pkgpath = self.name else: @@ -114,21 +115,52 @@ return space.w_None load_module.unwrap_spec = ['self', ObjSpace, str] - def get_data(self, space): - pass - get_data.unwrap_spec = ['self', ObjSpace] - - def get_code(self, space): - pass - get_code.unwrap_spec = ['self', ObjSpace] + def _get_data(self, space, w_filename): + w = space.wrap + try: + return space.call(space.getattr(self.w_dir, w('read')), + space.newlist([w_filename])) + except OperationError, e: + raise OperationError(space.w_IOError, e.w_value) - def get_source(self, space): - pass - get_source.unwrap_spec = ['self', ObjSpace] + def get_data(self, space, filename): + filename = self.mangle(filename) + return self._get_data(space, space.wrap(filename)) + get_data.unwrap_spec = ['self', ObjSpace, str] - def is_package(self, space): - pass - is_package.unwrap_spec = ['self', ObjSpace] + def get_code(self, space, filename): + w = space.wrap + filename = self.mangle(filename) + for compiled, _, ext in ENUMERATE_EXTS: + if self.get_module(space, filename + ext): + if compiled: + return self._get_data(space, space.wrap(filename + ext)) + else: + w_source = self._get_data(space, space.wrap(filename + ext)) + w_code = space.builtin.call('compile', w_source, + w(filename + ext), w('exec')) + return w_code + raise OperationError(space.w_ImportError, space.wrap( + "Cannot find source or code for %s in %s" % (filename, self.name))) + get_code.unwrap_spec = ['self', ObjSpace, str] + + def get_source(self, space, filename): + filename = self.mangle(filename) + for compiled, _, ext in ENUMERATE_EXTS: + if not compiled: + if self.get_module(space, filename + ext): + return self._get_data(space, space.wrap(filename + ext)) + raise OperationError(space.w_ImportError, space.wrap( + "Cannot find source for %s in %s" % (filename, self.name))) + get_source.unwrap_spec = ['self', ObjSpace, str] + + def is_package(self, space, filename): + for _, is_package, ext in ENUMERATE_EXTS: + if self.get_module(space, filename + ext): + return space.wrap(is_package) + raise OperationError(space.w_ImportError, space.wrap( + "Cannot find module %s in %s" % (filename, self.name))) + is_package.unwrap_spec = ['self', ObjSpace, str] def descr_new_zipimporter(space, w_type, name): try: Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/test/test_zipimport.py Wed Oct 24 16:07:30 2007 @@ -31,7 +31,6 @@ pyc = s.getvalue() + struct.pack(" Author: niko Date: Wed Oct 24 16:18:35 2007 New Revision: 47833 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: define the "quick push constant" primitives Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Wed Oct 24 16:18:35 2007 @@ -18,9 +18,6 @@ elif isinstance(w_v, model.W_SmallInteger): return float(w_v.value) raise PrimitiveFailedError() -def is_valid_index(idx, w_obj): - return idx >= 0 and idx < w_obj.size() - def subscript(idx, w_obj): if isinstance(w_obj, model.W_PointersObject): return w_obj.getindexedvar(idx) @@ -30,13 +27,20 @@ return fimg.wrap_int(w_obj.getbyte(idx)) raise PrimitiveFailedError() +def assert_bounds(idx, minb, maxb): + if idx < minb or idx >= maxb: + raise PrimitiveFailedError() + +def assert_valid_index(idx, w_obj): + assert_bounds(idx, 0, w_obj.size()) + # ___________________________________________________________________________ # Primitive table: it is filled in at initialization time with the # primitive functions. Each primitive function takes a single argument, # the frame (a W_ContextFrame object); the function either completes, and # returns a result, or throws a PrimitiveFailedError. -prim_table = [None] * 127 +prim_table = [None] * 576 # Squeak has primitives all the way up to 575 def primitive(code): def decorator(func): @@ -121,25 +125,17 @@ SIZE = 62 STRING_AT = 63 STRING_AT_PUT = 64 -OBJECT_AT = 68 -OBJECT_AT_PUT = 69 -NEW = 70 -NEW_WITH_ARG = 71 -INST_VAR_AT = 73 -AS_OOP = 75 def common_at(stack): [w_idx, w_obj] = stack idx = unwrap_int(w_idx) - if not is_valid_index(idx, w_obj): - raise PrimitiveFailedError() + assert_valid_index(idx, w_obj) return idx, w_obj def common_at_put(stack): [w_val, w_idx, w_obj] = stack idx = unwrap_int(w_idx) - if not is_valid_index(idx, w_obj): - raise PrimitiveFailedError() + assert_valid_index(idx, w_obj) return w_val, idx, w_obj @primitive(AT) @@ -179,13 +175,27 @@ w_obj.setbyte(idx, fimg.ord_w_char(w_val)) return w_val +# ___________________________________________________________________________ +# Storage Management Primitives + +OBJECT_AT = 68 +OBJECT_AT_PUT = 69 +NEW = 70 +NEW_WITH_ARG = 71 +ARRAY_BECOME_ONE_WAY = 72 # Blue Book: primitiveBecome +INST_VAR_AT = 73 +INST_VAR_AT_PUT = 74 +AS_OOP = 75 +STORE_STACKP = 76 # Blue Book: primitiveAsObject +SOME_INSTANCE = 77 +NEXT_INSTANCE = 78 + @primitive(OBJECT_AT) @stack(2) def func(stack): [w_idx, w_rcvr] = stack idx = unwrap_int(w_idx) - if idx < 0 or idx >= w_rcvr.w_class.instvarsize: - raise PrimitiveFailedError() + assert_bounds(idx, 0, w_rcvr.w_class.instvarsize) return w_rcvr.getnamedvar(idx) @primitive(OBJECT_AT_PUT) @@ -193,12 +203,10 @@ def func(stack): [w_val, w_idx, w_rcvr] = stack idx = unwrap_int(w_idx) - if idx < 0 or idx >= w_rcvr.w_class.instvarsize: - raise PrimitiveFailedError() + assert_bounds(idx, 0, w_rcvr.w_class.instvarsize) w_rcvr.setnamedvar(idx, w_val) return w_val - @primitive(NEW) @stack(1) def func(stack): @@ -216,6 +224,10 @@ size = unwrap_int(w_size) return w_cls.new(size) + at primitive(ARRAY_BECOME_ONE_WAY) +def func(frame): + raise PrimitiveNotYetWrittenError + @primitive(INST_VAR_AT) @stack(2) def func(stack): @@ -233,6 +245,10 @@ return subscript(idx, w_rcvr) raise PrimitiveFailedError() + at primitive(INST_VAR_AT_PUT) +def func(frame): + raise PrimitiveNotYetWrittenError() + @primitive(AS_OOP) @stack(1) def func(stack): @@ -241,6 +257,30 @@ raise PrimitiveFailedError() return w_rcvr.w_hash + at primitive(STORE_STACKP) + at stack(2) +def func(stack): + # This primitive seems to resize the stack. I don't think this is + # really relevant in our implementation. + raise PrimitiveNotYetWrittenError() + + at primitive(SOME_INSTANCE) + at stack(1) +def func(stack): + # This primitive returns some instance of the class on the stack. + # Not sure quite how to do this; maintain a weak list of all + # existing instances or something? + [w_class] = stack + raise PrimitiveNotYetWrittenError() + + at primitive(NEXT_INSTANCE) + at stack(1) +def func(stack): + # This primitive is used to iterate through all instances of a class: + # it returns the "next" instance after w_obj. + [w_obj] = stack + raise PrimitiveNotYetWrittenError() + # ___________________________________________________________________________ # Boolean Primitives @@ -287,3 +327,38 @@ res = op(v1, v2) w_res = fimg.wrap_bool(res) return w_res + +# ___________________________________________________________________________ +# Quick Push Const Primitives + +PUSH_SELF = 256 +PUSH_TRUE = 257 +PUSH_FALSE = 258 +PUSH_NIL = 259 +PUSH_MINUS_ONE = 260 +PUSH_ZERO = 261 +PUSH_ONE = 262 +PUSH_TWO = 263 + + at primitive(PUSH_SELF) + at stack(1) +def func(stack): + [w_self] = stack + return w_self + +def define_const_primitives(): + for (code, const) in [ + (PUSH_TRUE, fimg.w_true), + (PUSH_FALSE, fimg.w_false), + (PUSH_NIL, fimg.w_nil), + (PUSH_MINUS_ONE, fimg.w_mone), + (PUSH_ZERO, fimg.w_zero), + (PUSH_ONE, fimg.w_one), + (PUSH_TWO, fimg.w_two), + ]: + @primitive(code) + @stack(1) + def func(stack, const=const): # n.b.: capture const + return const +define_const_primitives() + Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Wed Oct 24 16:18:35 2007 @@ -133,6 +133,20 @@ def test_as_oop_not_applicable_to_int(): prim_fails(p.AS_OOP, [22]) +def test_const_primitives(): + for (code, const) in [ + (p.PUSH_TRUE, fimg.w_true), + (p.PUSH_FALSE, fimg.w_false), + (p.PUSH_NIL, fimg.w_nil), + (p.PUSH_MINUS_ONE, fimg.w_mone), + (p.PUSH_ZERO, fimg.w_zero), + (p.PUSH_ONE, fimg.w_one), + (p.PUSH_TWO, fimg.w_two), + ]: + assert prim(code, [fimg.w_nil]) is const + assert prim(p.PUSH_SELF, [fimg.w_nil]) is fimg.w_nil + assert prim(p.PUSH_SELF, ["a"]) is wrap("a") + def test_boolean(): assert prim(p.LESSTHAN, [1,2]) == fimg.w_true assert prim(p.GREATERTHAN, [3,4]) == fimg.w_false From fijal at codespeak.net Wed Oct 24 16:42:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 24 Oct 2007 16:42:54 +0200 (CEST) Subject: [pypy-svn] r47834 - pypy/dist/pypy/doc/config Message-ID: <20071024144254.802F780D1@code0.codespeak.net> Author: fijal Date: Wed Oct 24 16:42:53 2007 New Revision: 47834 Added: pypy/dist/pypy/doc/config/objspace.usemodules.zipimport.txt (contents, props changed) Log: Add zipimport doc Added: pypy/dist/pypy/doc/config/objspace.usemodules.zipimport.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/config/objspace.usemodules.zipimport.txt Wed Oct 24 16:42:53 2007 @@ -0,0 +1,3 @@ +This module implements zipimport mechanise described +in PEP 302. It's supposed to work and translate, so it's included +by default \ No newline at end of file From fijal at codespeak.net Wed Oct 24 16:43:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 24 Oct 2007 16:43:46 +0200 (CEST) Subject: [pypy-svn] r47835 - pypy/dist/pypy/config Message-ID: <20071024144346.9A06880D1@code0.codespeak.net> Author: fijal Date: Wed Oct 24 16:43:45 2007 New Revision: 47835 Modified: pypy/dist/pypy/config/pypyoption.py Log: Add zipimport to --allworkingmodules Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Wed Oct 24 16:43:45 2007 @@ -24,7 +24,7 @@ working_modules.update(dict.fromkeys( ["_socket", "unicodedata", "mmap", "fcntl", "rctime", "select", "crypt", "signal", "dyngram", "readline", "termios", "zlib", - "struct", "md5", "sha", "bz2" + "struct", "md5", "sha", "bz2", 'zimimport' ] )) From fijal at codespeak.net Wed Oct 24 16:46:59 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 24 Oct 2007 16:46:59 +0200 (CEST) Subject: [pypy-svn] r47836 - in pypy/dist/pypy/module/zipimport: . test Message-ID: <20071024144659.CE78580D1@code0.codespeak.net> Author: fijal Date: Wed Oct 24 16:46:59 2007 New Revision: 47836 Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py pypy/dist/pypy/module/zipimport/test/test_zipimport.py Log: Add .archive attribute Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/interp_zipimport.py Wed Oct 24 16:46:59 2007 @@ -162,6 +162,9 @@ "Cannot find module %s in %s" % (filename, self.name))) is_package.unwrap_spec = ['self', ObjSpace, str] + def getarchive(space, self): + return space.wrap(self.name) + def descr_new_zipimporter(space, w_type, name): try: return zip_importer_cache[name] @@ -200,4 +203,5 @@ get_source = interp2app(W_ZipImporter.get_source), is_package = interp2app(W_ZipImporter.is_package), load_module = interp2app(W_ZipImporter.load_module), + archive = GetSetProperty(W_ZipImporter.getarchive), ) Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/test/test_zipimport.py Wed Oct 24 16:46:59 2007 @@ -175,3 +175,4 @@ #assert z.get_code('uu') == self.co assert z.get_code('xx') assert z.get_source('xx') == "5" + assert z.archive == self.zipfile From arigo at codespeak.net Wed Oct 24 17:19:56 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 24 Oct 2007 17:19:56 +0200 (CEST) Subject: [pypy-svn] r47837 - in pypy/dist/pypy/rpython: . lltypesystem Message-ID: <20071024151956.A72818116@code0.codespeak.net> Author: arigo Date: Wed Oct 24 17:19:56 2007 New Revision: 47837 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/lltype.py Log: Forgot to check this in. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Wed Oct 24 17:19:56 2007 @@ -699,7 +699,7 @@ self.heap.free(obj, flavor=flavor) def op_zero_gc_pointers_inside(self, obj): - pass + raise NotImplementedError("zero_gc_pointers_inside") def op_getfield(self, obj, field): checkptr(obj) Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lltype.py Wed Oct 24 17:19:56 2007 @@ -1077,8 +1077,11 @@ def __iter__(self): # this is a work-around for the 'isrpystring' hack in __getitem__, # which otherwise causes list(p) to include the extra \x00 character. - for i in range(len(self)): - yield self[i] + if isinstance(self._T, (Array, FixedSizeArray)): + for i in range(self._obj.getlength()): + yield self[i] + else: + raise TypeError("%r instance is not an array" % (self._T,)) def __repr__(self): return '<%s>' % (self,) From arigo at codespeak.net Wed Oct 24 17:41:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 24 Oct 2007 17:41:52 +0200 (CEST) Subject: [pypy-svn] r47838 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071024154152.980B28112@code0.codespeak.net> Author: arigo Date: Wed Oct 24 17:41:51 2007 New Revision: 47838 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Log: A more explicit error message than "NameError: standard_c_lib". Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Wed Oct 24 17:41:51 2007 @@ -513,6 +513,9 @@ # ____________________________________________ def get_ctypes_callable(funcptr, calling_conv): + if not ctypes: + raise ImportError("ctypes is needed to use ll2ctypes") + def get_on_lib(lib, elem): """ Wrapper to always use lib[func] instead of lib.func """ From akuhn at codespeak.net Wed Oct 24 17:49:18 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Wed, 24 Oct 2007 17:49:18 +0200 (CEST) Subject: [pypy-svn] r47839 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024154918.476CD8116@code0.codespeak.net> Author: akuhn Date: Wed Oct 24 17:49:17 2007 New Revision: 47839 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_model.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py Log: (akuhn, tverwaes) got sidetracked while implementing some image loading code. we removed knowledge about number of named inst vars from W_Class / W_Object as this is not trivially known to the VM Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Wed Oct 24 17:49:17 2007 @@ -1,5 +1,5 @@ -class W_Object: +class W_Object(object): def __init__(self, w_class): self.w_class = w_class self.w_hash = None # XXX initial value? @@ -8,17 +8,35 @@ return 0 def instvarsize(self): - return self.w_class.instvarsize + return self.w_class.instvarsize + + def invariant(self): + return (hasattr(self, 'w_class') and + self.w_class != None) + + def as_w_class_instvarsize(self): + return self.vars[3] # hardcoded position of instvar class W_SmallInteger(W_Object): def __init__(self, w_class, value): W_Object.__init__(self, w_class) self.value = value + + def invariant(self): + W_Object.invariant(self) + assert hasattr(self, 'value') + assert self.value != None class W_Float(W_Object): def __init__(self, w_class, value): W_Object.__init__(self, w_class) self.value = value + + def invariant(self): + return (W_Object.invariant(self) and + hasattr(self, 'value') and + self.value != None) + class W_PointersObject(W_Object): """ The normal object """ @@ -26,22 +44,33 @@ W_Object.__init__(self, w_class) self.vars = [None] * (w_class.instvarsize + size) - def getnamedvar(self, index): - if not index < self.w_class.instvarsize: raise IndexError + def fetch(self, index): return self.vars[index] + + def store(self, index, w_value): + self.vars[index] = w_value + + def getnamedvar(self, index): + print "deprecated" + return self.fetch(index) def setnamedvar(self, index, w_value): - if not index < self.w_class.instvarsize: raise IndexError - self.vars[index] = w_value + print "deprecated" + return self.store(index, w_value) def size(self): - return len(self.vars) - self.w_class.instvarsize + return len(self.vars) def getindexedvar(self, index): - return self.vars[index + self.w_class.instvarsize] + raise NotImplementedError def setindexedvar(self, index, w_value): - self.vars[index + self.w_class.instvarsize] = w_value + raise NotImplementedError + + def invariant(self): + return (W_Object.invariant(self) and + hasattr(self, 'vars') and + self.vars != None) class W_BytesObject(W_Object): @@ -58,6 +87,12 @@ def size(self): return len(self.bytes) + def invariant(self): + return (W_Object.invariant(self) and + hasattr(self, 'bytes') and + self.bytes != None) + + class W_WordsObject(W_Object): def __init__(self, w_class, size): W_Object.__init__(self, w_class) @@ -70,7 +105,12 @@ self.words[n] = word def size(self): - return len(self.words) + return len(self.words) + + def invariant(self): + return (W_Object.invariant(self) and + hasattr(self, 'words') and + self.words != None) class W_CompiledMethod(W_Object): """My instances are methods suitable for interpretation by the virtual machine. This is the only class in the system whose instances intermix both indexable pointer fields and indexable integer fields. @@ -109,6 +149,19 @@ assert len(arguments) == self.argsize return W_ContextFrame(None, self, receiver, arguments, sender) + def invariant(self): + return (W_Object.invariant(self) and + hasattr(self, 'literals') and + self.literals != None and + hasattr(self, 'bytes') and + self.bytes != None and + hasattr(self, 'argsize') and + self.argsize != None and + hasattr(self, 'tempsize') and + self.tempsize != None and + hasattr(self, 'primitive') and + self.primitive != None) + # ____________________________________________________________ POINTERS = 0 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Wed Oct 24 17:49:17 2007 @@ -142,13 +142,13 @@ @stack(2) def func(stack): idx, w_obj = common_at(stack) - return w_obj.getindexedvar(idx) + return w_obj.fetch(idx) @primitive(AT_PUT) @stack(3) def func(stack): w_val, idx, w_obj = common_at_put(stack) - w_obj.setindexedvar(idx, w_val) + w_obj.store(idx, w_val) return w_val @primitive(SIZE) Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Wed Oct 24 17:49:17 2007 @@ -1,14 +1,23 @@ import py -import struct from pypy.lang.smalltalk import model from pypy.lang.smalltalk import fakeimage from pypy.rlib import objectmodel def int2str(integer): - return (chr((integer & 0xff000000) >> 24) + - chr((integer & 0x00ff0000) >> 16) + - chr((integer & 0x0000ff00) >> 8) + - chr((integer & 0x000000ff))) + return (chr((integer >> 24) & 0xff) + + chr((integer >> 16) & 0xff) + + chr((integer >> 8) & 0xff) + + chr((integer >> 0) & 0xff)) + +def splitbits(integer, lengths): + assert sum(lengths) <= 32 + result = [] + for length in lengths: + result.append(integer & (2**length - 1)) + integer = integer >> length + #XXX we can later mask and unroll this + return result + # ____________________________________________________________ # @@ -26,6 +35,7 @@ self.count = 0 def peek(self): + import struct if self.pos >= len(self.data): raise IndexError if self.swap: @@ -49,15 +59,11 @@ assert (self.pos + jump) <= len(self.data) self.pos += jump self.count += jump + + def close(self): + pass # already closed + -def splitbits(integer, lengths): - assert sum(lengths) <= 32 - result = [] - for length in lengths: - result.append(integer & (2**length - 1)) - integer = integer >> length - #XXX we can later mask and unroll this - return result class CorruptImageError(Exception): pass @@ -73,24 +79,14 @@ self.read_body() self.init_compactclassesarray() self.init_g_objects() - - def init_g_objects(self): - for chunk in self.chunks.itervalues(): - chunk.as_g_object(self) - - def init_w_objects(self): - for chunk in self.chunks.itervalues(): - chunk.g_object.init_w_object() + self.init_w_objects() + self.fillin_w_objects() - def fillin_w_objects(self): - for chunk in self.chunks.itervalues(): - chunk.g_object.fillin_w_object() - def read_header(self): version = self.stream.next() - if version != 0x1966: raise NotImplementedError + if version != 0x1966: raise NotImplementedError #XXX swap here if 0x66190000 headersize = self.stream.next() - self.endofmemory = self.stream.next() + self.endofmemory = self.stream.next() # endofmemory = bodysize self.oldbaseaddress = self.stream.next() self.specialobjectspointer = self.stream.next() lasthash = self.stream.next() @@ -98,16 +94,31 @@ fullscreenflag = self.stream.next() extravmmemory = self.stream.next() self.stream.skipbytes(headersize - (9 * 4)) - + def read_body(self): self.chunks = {} self.stream.reset_count() while self.stream.count < self.endofmemory: chunk, pos = self.read_object() self.chunks[pos + self.oldbaseaddress] = chunk - return self.chunks.values() + self.stream.close() + del self.stream + return self.chunks.values() # return for testing + + def init_g_objects(self): + for chunk in self.chunks.itervalues(): + chunk.as_g_object(self) # initialized g_object + + def init_w_objects(self): + for chunk in self.chunks.itervalues(): + chunk.g_object.init_w_object() + + def fillin_w_objects(self): + for chunk in self.chunks.itervalues(): + chunk.g_object.fillin_w_object() def init_compactclassesarray(self): + """ (CompiledMethod Symbol Array PseudoContext LargePositiveInteger nil MethodDictionary Association Point Rectangle nil TranslatedMethod BlockContext MethodContext nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil ) """ special = self.chunks[self.specialobjectspointer] assert special.size > 24 #at least assert special.format == 2 @@ -116,10 +127,6 @@ assert chunk.format == 2 self.compactclasses = [self.chunks[pointer] for pointer in chunk.data] - def init_actualobjects(self): - for chunk in self.chunks.itervalues(): - chunk.get_actual() # initialization - def read_object(self): kind = self.stream.peek() & 3 # 2 bits if kind == 0: # 00 bits @@ -142,8 +149,8 @@ return ImageChunk(size, format, classid, idhash), self.stream.count - 4 def read_2wordobjectheader(self): - assert splitbits(self.stream.peek(), [2])[0] == 1 #kind - classid = self.stream.next() - 1 # remove headertype to get pointer + assert self.stream.peek() & 3 == 1 #kind + classid = self.stream.next() - 01 # remove headertype to get pointer kind, size, format, _, idhash = splitbits(self.stream.next(), [2,6,4,5,12]) assert kind == 1 return ImageChunk(size, format, classid, idhash), self.stream.count - 4 @@ -152,12 +159,28 @@ kind, size = splitbits(self.stream.next(), [2,30]) assert kind == 0 assert splitbits(self.stream.peek(), [2])[0] == 0 #kind - classid = self.stream.next() - 0 # remove headertype to get pointer + classid = self.stream.next() - 00 # remove headertype to get pointer kind, _, format, _, idhash = splitbits(self.stream.next(), [2,6,4,5,12]) assert kind == 0 return ImageChunk(size, format, classid, idhash), self.stream.count - 4 + +# ____________________________________________________________ + +class SqueakImage(object): + + def from_reader(self, reader): + self.special_objects = [g_object.w_object for g_object in + reader.chunks[reader.specialobjectspointer] + .g_object.pointers] + self.objects = [chunk.g_object.w_object for chunk in reader.chunks.itervalues()] + + def special(self, index): + return self.special_objects[index] + COMPACT_CLASSES_ARRAY = 28 +FLOAT_CLASS = 10 + # ____________________________________________________________ @@ -177,7 +200,7 @@ self.owner = reader self.value = value self.size = -1 - self.w_object = fakeimage.small_int(value) + self.w_object = fakeimage.wrap_int(value) def initialize(self, chunk, reader): self.owner = reader @@ -198,14 +221,15 @@ def init_data(self, chunk): if not self.ispointers(): return - self.pointers = [] - for pointer in chunk.data: - g_object = self.decode_pointer(pointer) - self.pointers.append(g_object) + self.pointers = [self.decode_pointer(pointer) + for pointer in chunk.data] + assert len(filter(lambda x: x is None, self.pointers)) == 0 def decode_pointer(self, pointer): if (pointer & 1) == 1: - return GenericObject().initialize_int(pointer >> 1, self.owner) + small_int = GenericObject() + small_int.initialize_int(pointer >> 1, self.owner) + return small_int else: return self.owner.chunks[pointer].g_object @@ -219,8 +243,24 @@ return self.format < 5 #TODO, what about compiled methods? def init_w_object(self): + """ 0 no fields + 1 fixed fields only (all containing pointers) + 2 indexable fields only (all containing pointers) + 3 both fixed and indexable fields (all containing pointers) + 4 both fixed and indexable weak fields (all containing pointers). + + 5 unused + 6 indexable word fields only (no pointers) + 7 indexable long (64-bit) fields (only in 64-bit images) + + 8-11 indexable byte fields only (no pointers) (low 2 bits are low 2 bits of size) + 12-15 compiled methods: + # of literal oops specified in method header, + followed by indexable bytes (same interpretation of low 2 bits as above) + """ if self.w_object is None: if self.format < 5: + # XXX self.format == 4 is weak self.w_object = objectmodel.instantiate(model.W_PointersObject) elif self.format == 5: raise CorruptImageError("Unknown format 5") @@ -240,17 +280,21 @@ # below we are using an RPython idiom to 'cast' self.w_object # and pass the casted reference to the fillin_* methods casted = self.w_object - case = type(casted) + case = casted.__class__ if case == model.W_PointersObject: - self.fillin_poingersobject(casted) + self.fillin_pointersobject(casted) elif case == model.W_WordsObject: self.fillin_wordsobject(casted) elif case == model.W_BytesObject: self.fillin_bytesobject(casted) elif case == model.W_CompiledMethod: self.fillin_compiledmethod(casted) + else: + assert 0 + assert casted.invariant() def fillin_pointersobject(self, w_pointersobject): + assert self.pointers is not None w_pointersobject.vars = [g_object.w_object for g_object in self.pointers] w_pointersobject.w_class = self.g_class.w_object @@ -260,16 +304,16 @@ def fillin_bytesobject(self, w_bytesobject): bytes = [] - for each in chunk.data: - bytes.append((integer & 0xff000000) >> 24) - bytes.append((integer & 0xff000000) >> 16) - bytes.append((integer & 0xff000000) >> 8) - bytes.append((integer & 0xff000000)) + for each in self.chunk.data: + bytes.append((each >> 24) & 0xff) + bytes.append((each >> 16) & 0xff) + bytes.append((each >> 8) & 0xff) + bytes.append((each >> 0) & 0xff) w_bytesobject.w_class = self.g_class.w_object w_bytesobject.bytes = bytes[:-(self.format & 3)] # omit odd bytes def fillin_compiledmethod(self, w_compiledmethod): - header = chunk.data[0] + header = self.chunk.data[0] #(index 0) 9 bits: main part of primitive number (#primitive) #(index 9) 8 bits: number of literals (#numLiterals) #(index 17) 1 bit: whether a large frame size is needed (#frameSize) @@ -277,18 +321,19 @@ #(index 24) 4 bits: number of arguments to the method (#numArgs) #(index 28) 1 bit: high-bit of primitive number (#primitive) #(index 29) 1 bit: flag bit, ignored by the VM (#flag) - highbit, numargs, tempsize, islarge, literalsize, primitive = ( - splitbits(header, [1,4,6,1,8,9])) + primitive, literalsize, islarge, tempsize, numargs, highbit = ( + splitbits(header, [9,8,1,6,4,1])) primitive = primitive + (highbit << 10) - assert (1 + literalsize) < len(chunk.data) + #assert literalsize <= len(self.chunk.data) l = [] - for each in chunk.data[(1 + literalsize):]: + for each in self.chunk.data[(1 + literalsize):]: l.append(int2str(each)) - l[-1] = l[-1][:-(self.format & 3)] # omit odd bytes + if len(l) > 0: + l[-1] = l[-1][:-(self.format & 3)] # omit odd bytes bytes = "".join(l) w_compiledmethod.__init__( w_class = self.g_class.w_object, - size = self.literalsize, + size = literalsize, bytes = bytes, argsize = numargs, tempsize = tempsize, Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Wed Oct 24 17:49:17 2007 @@ -17,20 +17,6 @@ w_myinstance.setnamedvar(1, w_myinstance) assert w_myinstance.getnamedvar(1) is w_myinstance -def test_new_indexednamedvars(): - w_mycls = model.W_Class(None, None, 3, format=model.VAR_POINTERS) - w_myinstance = w_mycls.new(2) - assert isinstance(w_myinstance, model.W_PointersObject) - assert w_myinstance.w_class is w_mycls - assert w_myinstance.getnamedvar(0) is None - py.test.raises(IndexError, lambda: w_myinstance.getnamedvar(3)) - w_myinstance.setnamedvar(1, w_myinstance) - assert w_myinstance.getnamedvar(1) is w_myinstance - assert w_myinstance.getindexedvar(1) is None - py.test.raises(IndexError, lambda: w_myinstance.getindexedvar(2)) - w_myinstance.setindexedvar(0, w_myinstance) - assert w_myinstance.getindexedvar(0) is w_myinstance - def test_bytes_object(): w_class = model.W_Class(None, None, format=model.BYTES) w_bytes = w_class.new(20) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Wed Oct 24 17:49:17 2007 @@ -53,7 +53,7 @@ def test_at(): w_obj = model.W_Class(None, None, 0, format=model.VAR_POINTERS).new(1) - w_obj.setindexedvar(0, "foo") + w_obj.store(0, "foo") assert prim(p.AT, [w_obj, 0]) == "foo" def test_invalid_at(): Modified: pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py Wed Oct 24 17:49:17 2007 @@ -126,52 +126,3 @@ assert pos == 8 assert chunk0 == chunk -# ----- mini.image productline ------------------------------- -# relying on order of methods - -mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') - -def test_miniimageexists(): - assert mini_image.check(dir=False) - -example = squeakimage.ImageReader(squeakimage.Stream(mini_image.open())) - -def test_read_header(): - example.read_header() - assert example.endofmemory == 0x93174 - assert example.oldbaseaddress == 0x6649000 - assert example.specialobjectspointer == 0x6668380 - -def test_read_all_header(): - next = example.stream.peek() - assert next != 0 #expects object header, which must not be 0x00000000 - -def test_read_body(): - objects = example.read_body() - assert len(objects) > 0 - assert 15000 < len(objects) < 16000 - -def test_read_all_body(): - py.test.raises(IndexError, lambda: example.stream.next()) - -def test_all_pointers_valid(): - for each in example.chunks.itervalues(): - if each.format < 5: - for pointer in each.data: - if (pointer & 1) != 1: - assert pointer in example.chunks - -def test_init_compactclasses(): - example.init_compactclassesarray() - assert len(example.compactclasses) == 31 - -def test_init_g_objects(): - example.init_g_objects() - -def test_init_w_objects(): - example.init_w_objects() - -def test_fillin_w_objects(): - example.fillin_w_objects() - - \ No newline at end of file From akuhn at codespeak.net Wed Oct 24 17:51:12 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Wed, 24 Oct 2007 17:51:12 +0200 (CEST) Subject: [pypy-svn] r47840 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071024155112.43AD48118@code0.codespeak.net> Author: akuhn Date: Wed Oct 24 17:51:11 2007 New Revision: 47840 Added: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: stupid me forgot to add test/test_miniimage.py Added: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Wed Oct 24 17:51:11 2007 @@ -0,0 +1,66 @@ +# ----- mini.image productline ------------------------------- +# NOT relying on order of methods +# one big method to rule them all +import py +from pypy.lang.smalltalk import squeakimage as sq + + +mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') + +def test_miniimageexists(): + assert mini_image.check(dir=False) + +def get_miniimage(): + return sq.ImageReader(sq.Stream(mini_image.open())) + +def test_read_header(): + example = get_miniimage() + example.read_header() + assert example.endofmemory == 0x93174 + assert example.oldbaseaddress == 0x6649000 + assert example.specialobjectspointer == 0x6668380 + +def test_read_all_header(): + example = get_miniimage() + example.read_header() + next = example.stream.peek() + assert next != 0 #expects object header, which must not be 0x00000000 + +def test_readimage_productline(): + example = get_miniimage() + example.read_header() + objects = example.read_body() + assert len(objects) > 0 + assert 15000 < len(objects) < 16000 + + # at end of file + # py.test.raises(IndexError, lambda: example.stream.next()) + + # all pointers are valid + for each in example.chunks.itervalues(): + if each.format < 5: + for pointer in each.data: + if (pointer & 1) != 1: + assert pointer in example.chunks + + # there are 31 compact classes + example.init_compactclassesarray() + assert len(example.compactclasses) == 31 + + example.init_g_objects() + + example.init_w_objects() + + example.fillin_w_objects() + + image = sq.SqueakImage() + + image.from_reader(example) + + for each in image.objects: + each.invariant() + + w_float_class = image.special(sq.FLOAT_CLASS) + + #assert w_float_class.size() == 10 + From tismer at codespeak.net Wed Oct 24 17:57:43 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Wed, 24 Oct 2007 17:57:43 +0200 (CEST) Subject: [pypy-svn] r47841 - pypy/dist/pypy/module/zipimport/test Message-ID: <20071024155743.B5BCA80DE@code0.codespeak.net> Author: tismer Date: Wed Oct 24 17:57:43 2007 New Revision: 47841 Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py Log: small fixes on zipimport tests Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/test/test_zipimport.py Wed Oct 24 17:57:43 2007 @@ -89,7 +89,7 @@ """) def test_py(self): - import sys + import sys, os self.writefile(self, "uuu.py", "def f(x): return x") mod = __import__('uuu', globals(), locals(), []) assert mod.f(3) == 3 @@ -99,11 +99,11 @@ 'f': mod.f} for key, val in expected.items(): assert mod.__dict__[key] == val - assert mod.__file__.endswith('.zip/uuu.py') + assert mod.__file__.endswith('.zip'+os.sep+'uuu.py') del sys.modules['uuu'] def test_pyc(self): - import sys + import sys, os self.writefile(self, "uuu.pyc", self.test_pyc) mod = __import__('uuu', globals(), locals(), []) expected = { @@ -114,7 +114,7 @@ } for key, val in expected.items(): assert mod.__dict__[key] == val - assert mod.__file__.endswith('.zip/uuu.pyc') + assert mod.__file__.endswith('.zip'+os.sep+'uuu.pyc') assert mod.get_file() == mod.__file__ assert mod.get_name() == mod.__name__ del sys.modules['uuu'] From lukas at codespeak.net Wed Oct 24 18:02:10 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Wed, 24 Oct 2007 18:02:10 +0200 (CEST) Subject: [pypy-svn] r47842 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024160210.4D6838108@code0.codespeak.net> Author: lukas Date: Wed Oct 24 18:02:09 2007 New Revision: 47842 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (lr) added integer primitives Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Wed Oct 24 18:02:09 2007 @@ -319,13 +319,13 @@ self.callPrimitiveAndPush(primitives.NOTEQUAL, "~=", 1, interp) def bytecodePrimMultiply(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush(primitives.MUL, "*", 1, interp) def bytecodePrimDivide(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush(primitives.DIVIDE, "/", 1, interp) def bytecodePrimMod(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush(primitives.MOD, "\\", 1, interp) def bytecodePrimMakePoint(self, interp): raise MissingBytecode @@ -334,7 +334,7 @@ raise MissingBytecode def bytecodePrimDiv(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush(primitives.DIV, "//", 1, interp) def bytecodePrimBitAnd(self, interp): raise MissingBytecode Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Wed Oct 24 18:02:09 2007 @@ -2,6 +2,7 @@ import pypy.lang.smalltalk.model as model import pypy.lang.smalltalk.classtable as ct import pypy.lang.smalltalk.fakeimage as fimg +from pypy.rlib import rarithmetic class PrimitiveFailedError(Exception): pass @@ -9,10 +10,6 @@ class PrimitiveNotYetWrittenError(PrimitiveFailedError): pass -def unwrap_int(w_v): - if isinstance(w_v, model.W_SmallInteger): return w_v.value - raise PrimitiveFailedError() - def unwrap_float(w_v): if isinstance(w_v, model.W_Float): return w_v.value elif isinstance(w_v, model.W_SmallInteger): return float(w_v.value) @@ -51,7 +48,7 @@ def stack(n): def decorator(wrapped): def result(frame): - items = [frame.peek(i) for i in range(n)] + items = frame.stack[len(frame.stack)-n:] res = wrapped(items) frame.pop_n(n) # only if no exception occurs! return res @@ -59,43 +56,91 @@ return decorator # ___________________________________________________________________________ -# Small Integer Primitives +# SmallInteger Primitives + +def unwrap_int(w_value): + if isinstance(w_value, model.W_SmallInteger): + return w_value.value + raise PrimitiveFailedError() -ADD = 1 -SUBTRACT = 2 -MAKE_POINT = 18 +def wrap_int(value): + if value > 1073741823: + raise PrimitiveFailedError() + if value < -1073741824: + raise PrimitiveFailedError() + return fimg.wrap_int(value) + +ADD = 1 +SUBTRACT = 2 +MULTIPLY = 9 +DIVIDE = 10 +MOD = 11 +DIV = 12 +QUO = 13 math_ops = { ADD: operator.add, - SUBTRACT: operator.sub + SUBTRACT: operator.sub, + MULTIPLY: operator.mul, } for (code,op) in math_ops.items(): + @primitive(code) @stack(2) def func(stack, op=op): # n.b. capture op value - [w_v2, w_v1] = stack - v1 = unwrap_int(w_v1) - v2 = unwrap_int(w_v2) - res = op(v1, v2) - - # Emulate the bounds of smalltalk tagged integers: - if res > 1073741823: raise PrimitiveFailedError() - if res < -1073741824: raise PrimitiveFailedError() - - w_res = fimg.wrap_int(res) - return w_res - prim_table[code] = func + [w_receiver, w_argument] = stack + receiver = unwrap_int(w_receiver) + argument = unwrap_int(w_argument) + try: + res = rarithmetic.ovfcheck(op(receiver, argument)) + except OverflowError: + raise PrimitiveFailedError() + return wrap_int(res) - at primitive(MAKE_POINT) -def primitivemakepoint(frame): - raise PrimitiveNotYetWrittenError(MAKE_POINT) +# #/ -- return the result of a division, only succeed if the division is exact + at primitive(DIVIDE) + at stack(2) +def func(stack): + [w_receiver, w_argument] = stack + receiver = unwrap_int(w_receiver) + argument = unwrap_int(w_argument) + if argument == 0: + raise PrimitiveFailedError() + if receiver % argument != 0: + raise PrimitiveFailedError() + return wrap_int(receiver // argument) -# ___________________________________________________________________________ -# Integer Primitives -# -# Primitives 21-37 are aliased to 1-17 for historical reasons. +# #\\ -- return the remainder of a division + at primitive(MOD) + at stack(2) +def func(stack): + [w_receiver, w_argument] = stack + receiver = unwrap_int(w_receiver) + argument = unwrap_int(w_argument) + if argument == 0: + raise PrimitiveFailedError() + return wrap_int(receiver % argument) -for i in range(21,38): - prim_table[i] = prim_table[i-20] +# #// -- return the result of a division, rounded towards negative zero + at primitive(DIV) + at stack(2) +def func(stack): + [w_receiver, w_argument] = stack + receiver = unwrap_int(w_receiver) + argument = unwrap_int(w_argument) + if argument == 0: + raise PrimitiveFailedError() + return wrap_int(receiver // argument) + +# #// -- return the result of a division, rounded towards negative infinity + at primitive(QUO) + at stack(2) +def func(stack): + [w_receiver, w_argument] = stack + receiver = unwrap_int(w_receiver) + argument = unwrap_int(w_argument) + if argument == 0: + raise PrimitiveFailedError() + return wrap_int(receiver // argument) # ___________________________________________________________________________ # Float Primitives @@ -110,7 +155,7 @@ for (code,op) in math_ops.items(): @stack(2) def func(res, op=op): # n.b. capture op value - [w_v2, w_v1] = res + [w_v1, w_v2] = res v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) w_res = fimg.wrap_float(op(v1, v2)) @@ -127,27 +172,27 @@ STRING_AT_PUT = 64 def common_at(stack): - [w_idx, w_obj] = stack + [w_obj, w_idx] = stack idx = unwrap_int(w_idx) assert_valid_index(idx, w_obj) - return idx, w_obj + return w_obj, idx def common_at_put(stack): - [w_val, w_idx, w_obj] = stack + [w_obj, w_idx, w_val] = stack idx = unwrap_int(w_idx) assert_valid_index(idx, w_obj) - return w_val, idx, w_obj + return w_obj, idx, w_val @primitive(AT) @stack(2) def func(stack): - idx, w_obj = common_at(stack) + w_obj, idx = common_at(stack) return w_obj.fetch(idx) @primitive(AT_PUT) @stack(3) def func(stack): - w_val, idx, w_obj = common_at_put(stack) + w_obj, idx, w_val = common_at_put(stack) w_obj.store(idx, w_val) return w_val @@ -162,14 +207,14 @@ @primitive(STRING_AT) @stack(2) def func(stack): - idx, w_obj = common_at(stack) + w_obj, idx = common_at(stack) byte = w_obj.getbyte(idx) return fimg.CharacterTable[byte] @primitive(STRING_AT_PUT) @stack(3) def func(stack): - w_val, idx, w_obj = common_at_put(stack) + w_obj, idx, w_val = common_at_put(stack) if w_val.w_class is not ct.w_Character: raise PrimitiveFailedError() w_obj.setbyte(idx, fimg.ord_w_char(w_val)) @@ -193,7 +238,7 @@ @primitive(OBJECT_AT) @stack(2) def func(stack): - [w_idx, w_rcvr] = stack + [w_rcvr, w_idx] = stack idx = unwrap_int(w_idx) assert_bounds(idx, 0, w_rcvr.w_class.instvarsize) return w_rcvr.getnamedvar(idx) @@ -201,7 +246,7 @@ @primitive(OBJECT_AT_PUT) @stack(3) def func(stack): - [w_val, w_idx, w_rcvr] = stack + [w_rcvr, w_idx, w_val] = stack idx = unwrap_int(w_idx) assert_bounds(idx, 0, w_rcvr.w_class.instvarsize) w_rcvr.setnamedvar(idx, w_val) @@ -218,7 +263,7 @@ @primitive(NEW_WITH_ARG) @stack(2) def func(stack): - [w_size, w_cls] = stack + [w_cls, w_size] = stack if not isinstance(w_cls, model.W_Class) or not w_cls.isvariable(): raise PrimitiveFailedError() size = unwrap_int(w_size) @@ -233,7 +278,7 @@ def func(stack): # I *think* this is the correct behavior, but I'm not quite sure. # Might be restricted to fixed length fields? - [w_idx, w_rcvr] = stack + [w_rcvr, w_idx] = stack idx = unwrap_int(w_idx) w_cls = w_rcvr.w_class if idx < 0: @@ -310,7 +355,7 @@ @primitive(code) @stack(2) def func(stack, op=op): # n.b. capture op value - [w_v2, w_v1] = stack + [w_v1, w_v2] = stack v1 = unwrap_int(w_v1) v2 = unwrap_int(w_v2) res = op(v1, v2) @@ -321,7 +366,7 @@ @primitive(code+_FLOAT_OFFSET) @stack(2) def func(stack, op=op): # n.b. capture op value - [w_v2, w_v1] = stack + [w_v1, w_v2] = stack v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) res = op(v1, v2) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Wed Oct 24 18:02:09 2007 @@ -36,17 +36,50 @@ py.test.fail("Expected PrimitiveFailedError") except PrimitiveFailedError: assert stack_w.stack == orig_stack - -def test_small_int_plus(): + +# smallinteger tests +def test_small_int_add(): assert prim(p.ADD, [1,2]).value == 3 assert prim(p.ADD, [3,4]).value == 7 +def test_small_int_add_fail(): + prim_fails(p.ADD, [1073741823,2]) + def test_small_int_minus(): assert prim(p.SUBTRACT, [5,9]).value == -4 -def test_small_int_overflow(): - prim_fails(p.ADD, [1073741823,2]) +def test_small_int_minus_fail(): + prim_fails(p.SUBTRACT, [-1073741823,2]) + +def test_small_int_divide(): + assert prim(p.DIVIDE, [6,3]).value == 2 + +def test_small_int_divide_fail(): + prim_fails(p.DIVIDE, [12, 0]) + prim_fails(p.DIVIDE, [12, 7]) + +def test_small_int_mod(): + assert prim(p.MOD, [12,7]).value == 5 + +def test_small_int_mod_fail(): + prim_fails(p.MOD, [12, 0]) + +def test_small_int_div(): + assert prim(p.DIV, [12,3]).value == 4 + assert prim(p.DIV, [12,7]).value == 1 + +def test_small_int_div_fail(): + prim_fails(p.DIV, [12, 0]) +def test_small_int_quo(): + assert prim(p.QUO, [12,3]).value == 4 + assert prim(p.QUO, [12,7]).value == 1 + +def test_small_int_quo_fail(): + prim_fails(p.QUO, [12, 0]) + + + def test_float(): assert prim(p.FLOAT_ADD, [1.0,2.0]).value == 3.0 assert prim(p.FLOAT_ADD, [3,4.5]).value == 7.5 From akuhn at codespeak.net Wed Oct 24 18:05:12 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Wed, 24 Oct 2007 18:05:12 +0200 (CEST) Subject: [pypy-svn] r47843 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024160512.267A98108@code0.codespeak.net> Author: akuhn Date: Wed Oct 24 18:05:11 2007 New Revision: 47843 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: (akuhn, tverwaes) Am Anfang war das Wort "Float" (succesfully retrieved the first string from loadedimage) Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Wed Oct 24 18:05:11 2007 @@ -179,7 +179,7 @@ return self.special_objects[index] COMPACT_CLASSES_ARRAY = 28 -FLOAT_CLASS = 10 +FLOAT_CLASS = 9 # ____________________________________________________________ @@ -305,10 +305,10 @@ def fillin_bytesobject(self, w_bytesobject): bytes = [] for each in self.chunk.data: - bytes.append((each >> 24) & 0xff) - bytes.append((each >> 16) & 0xff) - bytes.append((each >> 8) & 0xff) - bytes.append((each >> 0) & 0xff) + bytes.append(chr((each >> 24) & 0xff)) + bytes.append(chr((each >> 16) & 0xff)) + bytes.append(chr((each >> 8) & 0xff)) + bytes.append(chr((each >> 0) & 0xff)) w_bytesobject.w_class = self.g_class.w_object w_bytesobject.bytes = bytes[:-(self.format & 3)] # omit odd bytes Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Wed Oct 24 18:05:11 2007 @@ -3,6 +3,7 @@ # one big method to rule them all import py from pypy.lang.smalltalk import squeakimage as sq +from pypy.lang.smalltalk import model as sqm mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') @@ -62,5 +63,11 @@ w_float_class = image.special(sq.FLOAT_CLASS) - #assert w_float_class.size() == 10 + assert w_float_class.size() == 9 + + w_float_class_name = w_float_class.fetch(6) + + assert isinstance(w_float_class_name, sqm.W_BytesObject) + + assert w_float_class_name.bytes == list("Float") From arigo at codespeak.net Wed Oct 24 18:06:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 24 Oct 2007 18:06:04 +0200 (CEST) Subject: [pypy-svn] r47844 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024160604.B8F1A8115@code0.codespeak.net> Author: arigo Date: Wed Oct 24 18:06:04 2007 New Revision: 47844 Modified: pypy/dist/pypy/lang/smalltalk/fakeimage.py pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_model.py Log: Rename away the deprecated get/setnamedvar(). Modified: pypy/dist/pypy/lang/smalltalk/fakeimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/fakeimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/fakeimage.py Wed Oct 24 18:06:04 2007 @@ -23,7 +23,7 @@ def ord_w_char(w_c): assert w_c.w_class is ct.w_Character - w_ord = w_c.getnamedvar(CHARACTER_VALUE_INDEX) + w_ord = w_c.fetch(CHARACTER_VALUE_INDEX) assert w_ord.w_class is ct.w_SmallInteger assert isinstance(w_ord, model.W_SmallInteger) return w_ord.value @@ -51,7 +51,7 @@ global CharacterTable def bld_char(i): w_cinst = ct.w_Character.new() - w_cinst.setnamedvar(CHARACTER_VALUE_INDEX, wrap_int(i)) + w_cinst.store(CHARACTER_VALUE_INDEX, wrap_int(i)) return w_cinst CharacterTable = [bld_char(i) for i in range(256)] wrap_char_table() Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Wed Oct 24 18:06:04 2007 @@ -54,7 +54,7 @@ # push bytecodes def pushReceiverVariableBytecode(self, interp): index = self.currentBytecode & 15 - self.push(self.receiver.getnamedvar(index)) + self.push(self.receiver.fetch(index)) def pushTemporaryVariableBytecode(self, interp): index = self.currentBytecode & 15 @@ -70,11 +70,11 @@ # named var (the value). index = self.currentBytecode & 31 association = self.method.literals[index] - self.push(association.getnamedvar(1)) + self.push(association.fetch(1)) def storeAndPopReceiverVariableBytecode(self, interp): index = self.currentBytecode & 7 - self.receiver.setnamedvar(index, self.pop()) + self.receiver.store(index, self.pop()) def storeAndPopTemporaryVariableBytecode(self, interp): index = self.currentBytecode & 7 @@ -179,26 +179,26 @@ def extendedPushBytecode(self, interp): variableType, variableIndex = self.extendedVariableTypeAndIndex() if variableType == 0: - self.push(self.receiver.getnamedvar(variableIndex)) + self.push(self.receiver.fetch(variableIndex)) elif variableType == 1: self.push(self.gettemp(variableIndex)) elif variableType == 2: self.push(self.method.literals[variableIndex]) elif variableType == 3: association = self.method.literals[variableIndex] - self.push(association.getnamedvar(1)) + self.push(association.fetch(1)) def extendedStoreBytecode(self, interp): variableType, variableIndex = self.extendedVariableTypeAndIndex() if variableType == 0: - self.receiver.setnamedvar(variableIndex, self.top()) + self.receiver.store(variableIndex, self.top()) elif variableType == 1: self.settemp(variableIndex, self.top()) elif variableType == 2: raise IllegalStoreError elif variableType == 3: association = self.method.literals[variableIndex] - association.setnamedvar(1,self.top()) + association.store(1,self.top()) def extendedStoreAndPopBytecode(self, interp): self.extendedStoreBytecode(interp) @@ -226,21 +226,21 @@ second & 31, interp) elif opType == 2: # pushReceiver - self.push(self.receiver.getnamedvar(third)) + self.push(self.receiver.fetch(third)) elif opType == 3: # pushLiteralConstant self.push(self.method.literals[third]) elif opType == 4: # pushLiteralVariable association = self.method.literals[third] - self.push(association.getnamedvar(1)) + self.push(association.fetch(1)) elif opType == 5: - self.receiver.setnamedvar(third, self.top()) + self.receiver.store(third, self.top()) elif opType == 6: - self.receiver.setnamedvar(third, self.pop()) + self.receiver.store(third, self.pop()) elif opType == 7: association = self.method.literals[third] - association.setnamedvar(1,self.top()) + association.store(1,self.top()) def singleExtendedSuperBytecode(self, interp): selector, argcount = self.getExtendedSelectorArgcount() Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Wed Oct 24 18:06:04 2007 @@ -49,23 +49,9 @@ def store(self, index, w_value): self.vars[index] = w_value - - def getnamedvar(self, index): - print "deprecated" - return self.fetch(index) - - def setnamedvar(self, index, w_value): - print "deprecated" - return self.store(index, w_value) def size(self): return len(self.vars) - - def getindexedvar(self, index): - raise NotImplementedError - - def setindexedvar(self, index, w_value): - raise NotImplementedError def invariant(self): return (W_Object.invariant(self) and Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Wed Oct 24 18:06:04 2007 @@ -241,7 +241,7 @@ [w_rcvr, w_idx] = stack idx = unwrap_int(w_idx) assert_bounds(idx, 0, w_rcvr.w_class.instvarsize) - return w_rcvr.getnamedvar(idx) + return w_rcvr.fetch(idx) @primitive(OBJECT_AT_PUT) @stack(3) @@ -249,7 +249,7 @@ [w_rcvr, w_idx, w_val] = stack idx = unwrap_int(w_idx) assert_bounds(idx, 0, w_rcvr.w_class.instvarsize) - w_rcvr.setnamedvar(idx, w_val) + w_rcvr.store(idx, w_val) return w_val @primitive(NEW) @@ -284,7 +284,7 @@ if idx < 0: raise PrimitiveFailedError() if idx < w_cls.instvarsize: - return w_rcvr.getnamedvar(idx) + return w_rcvr.fetch(idx) idx -= w_cls.instvarsize if idx < w_rcvr.size(): return subscript(idx, w_rcvr) Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Wed Oct 24 18:06:04 2007 @@ -75,9 +75,9 @@ pushReceiverVariableBytecode(2))): w_democlass = model.W_Class(None, None, 3) w_demo = w_democlass.new() - w_demo.setnamedvar(0, "egg") - w_demo.setnamedvar(1, "bar") - w_demo.setnamedvar(2, "baz") + w_demo.store(0, "egg") + w_demo.store(1, "bar") + w_demo.store(2, "baz") interp = new_interpreter(bytecode, receiver = w_demo) interp.step() interp.step() @@ -107,8 +107,8 @@ def test_pushLiteralVariableBytecode(bytecode=pushLiteralVariableBytecode(0)): w_associationclass = model.W_Class(None, None, 2) w_association = w_associationclass.new() - w_association.setnamedvar(0, "mykey") - w_association.setnamedvar(1, "myvalue") + w_association.store(0, "mykey") + w_association.store(1, "myvalue") interp = new_interpreter(bytecode) interp.activeContext.method.literals = [w_association] interp.step() @@ -130,9 +130,9 @@ for test_index in range(8): if test_index == index: - assert w_object.getnamedvar(test_index) == interp.TRUE + assert w_object.fetch(test_index) == interp.TRUE else: - assert w_object.getnamedvar(test_index) == None + assert w_object.fetch(test_index) == None def test_storeAndPopTemporaryVariableBytecode(bytecode=storeAndPopTemporaryVariableBytecode): for index in range(8): @@ -332,13 +332,13 @@ def storeAssociation(bytecode): w_associationclass = model.W_Class(None, None, 2) w_association = w_associationclass.new() - w_association.setnamedvar(0, "mykey") - w_association.setnamedvar(1, "myvalue") + w_association.store(0, "mykey") + w_association.store(1, "myvalue") interp = new_interpreter(pushConstantOneBytecode + bytecode) interp.activeContext.method.literals = [w_association] interp.step() interp.step() - assert w_association.getnamedvar(1) == interp.ONE + assert w_association.fetch(1) == interp.ONE def test_extendedStoreAndPopBytecode(): test_storeAndPopReceiverVariableBytecode(lambda index: extendedStoreAndPopBytecode + chr((0<<6) + index)) Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Wed Oct 24 18:06:04 2007 @@ -12,10 +12,10 @@ w_myinstance = w_mycls.new() assert isinstance(w_myinstance, model.W_PointersObject) assert w_myinstance.w_class is w_mycls - assert w_myinstance.getnamedvar(0) is None - py.test.raises(IndexError, lambda: w_myinstance.getnamedvar(3)) - w_myinstance.setnamedvar(1, w_myinstance) - assert w_myinstance.getnamedvar(1) is w_myinstance + assert w_myinstance.fetch(0) is None + py.test.raises(IndexError, lambda: w_myinstance.fetch(3)) + w_myinstance.store(1, w_myinstance) + assert w_myinstance.fetch(1) is w_myinstance def test_bytes_object(): w_class = model.W_Class(None, None, format=model.BYTES) From elmom at codespeak.net Wed Oct 24 18:12:48 2007 From: elmom at codespeak.net (elmom at codespeak.net) Date: Wed, 24 Oct 2007 18:12:48 +0200 (CEST) Subject: [pypy-svn] r47845 - pypy/dist/pypy/config Message-ID: <20071024161248.CF57780E9@code0.codespeak.net> Author: elmom Date: Wed Oct 24 18:12:48 2007 New Revision: 47845 Modified: pypy/dist/pypy/config/pypyoption.py Log: zimimport should be zipimport :) Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Wed Oct 24 18:12:48 2007 @@ -24,7 +24,7 @@ working_modules.update(dict.fromkeys( ["_socket", "unicodedata", "mmap", "fcntl", "rctime", "select", "crypt", "signal", "dyngram", "readline", "termios", "zlib", - "struct", "md5", "sha", "bz2", 'zimimport' + "struct", "md5", "sha", "bz2", 'zipimport' ] )) From tismer at codespeak.net Wed Oct 24 18:16:48 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Wed, 24 Oct 2007 18:16:48 +0200 (CEST) Subject: [pypy-svn] r47846 - pypy/dist/pypy/module/zipimport/test Message-ID: <20071024161648.8BB5780E9@code0.codespeak.net> Author: tismer Date: Wed Oct 24 18:16:48 2007 New Revision: 47846 Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py Log: more os dependencies on file names in test for zipimporter. Still there is one test failing Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/test/test_zipimport.py Wed Oct 24 18:16:48 2007 @@ -152,17 +152,19 @@ assert mod(3) == 4 def test_package(self): - self.writefile(self, "xx/__init__.py", "") - self.writefile(self, "xx/yy.py", "def f(x): return x") + import os + self.writefile(self, "xx"+os.sep+"__init__.py", "") + self.writefile(self, "xx"+os.sep+"yy.py", "def f(x): return x") mod = __import__("xx", globals(), locals(), ['yy']) assert mod.__path__ assert mod.yy.f(3) == 3 def test_functions(self): + import os import zipimport data = "saddsadsa" self.writefile(self, "xxx", data) - self.writefile(self, "xx/__init__.py", "5") + self.writefile(self, "xx"+os.sep+"__init__.py", "5") self.writefile(self, "yy.py", "3") self.writefile(self, 'uu.pyc', self.test_pyc) z = zipimport.zipimporter(self.zipfile) From arigo at codespeak.net Wed Oct 24 18:35:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 24 Oct 2007 18:35:17 +0200 (CEST) Subject: [pypy-svn] r47847 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071024163517.6B2228111@code0.codespeak.net> Author: arigo Date: Wed Oct 24 18:35:15 2007 New Revision: 47847 Added: pypy/dist/pypy/lang/smalltalk/constants.py (contents, props changed) Modified: pypy/dist/pypy/lang/smalltalk/fakeimage.py Log: Move the constants to a new module. Please use it for all "magic constants" needed by the Smalltalk VM or image. Added: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/constants.py Wed Oct 24 18:35:15 2007 @@ -0,0 +1,14 @@ + +# ___________________________________________________________________________ +# Slot Names + +CHARACTER_VALUE_INDEX = 0 # Page 630 of the blue book + +STREAM_ARRAY_INDEX = 0 # Page 631 of the blue book +STREAM_INDEX_INDEX = 1 +STREAM_READ_LIMIT_INDEX = 2 +STREAM_WRITE_LIMIT_INDEX = 3 + +CLASS_SUPERCLASS_INDEX = 0 +CLASS_METHODDICT_INDEX = 1 +CLASS_FORMAT_INDEX = 2 Modified: pypy/dist/pypy/lang/smalltalk/fakeimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/fakeimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/fakeimage.py Wed Oct 24 18:35:15 2007 @@ -1,4 +1,5 @@ import pypy.lang.smalltalk.classtable as ct +from pypy.lang.smalltalk.constants import CHARACTER_VALUE_INDEX from pypy.lang.smalltalk import model # ___________________________________________________________________________ @@ -35,16 +36,6 @@ return w_false # ___________________________________________________________________________ -# Slot Names - -CHARACTER_VALUE_INDEX = 0 # Page 630 of the blue book - -STREAM_ARRAY_INDEX = 0 # Page 631 of the blue book -STREAM_INDEX_INDEX = 1 -STREAM_READ_LIMIT_INDEX = 2 -STREAM_WRITE_LIMIT_INDEX = 3 - -# ___________________________________________________________________________ # Global Data def wrap_char_table(): From tverwaes at codespeak.net Wed Oct 24 18:43:27 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 18:43:27 +0200 (CEST) Subject: [pypy-svn] r47848 - pypy/dist/pypy/lang/smalltalk/tools Message-ID: <20071024164327.C3A9580EE@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 18:43:27 2007 New Revision: 47848 Added: pypy/dist/pypy/lang/smalltalk/tools/ pypy/dist/pypy/lang/smalltalk/tools/analyseimage.py pypy/dist/pypy/lang/smalltalk/tools/autopath.py - copied unchanged from r47799, pypy/dist/pypy/bin/autopath.py Log: Adding nice smalltalk tools (print all strings in image) Added: pypy/dist/pypy/lang/smalltalk/tools/analyseimage.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/tools/analyseimage.py Wed Oct 24 18:43:27 2007 @@ -0,0 +1,26 @@ +import autopath +import py +from pypy.lang.smalltalk import squeakimage as sq +from pypy.lang.smalltalk import model as sqm + +mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') + +def test_miniimageexists(): + assert mini_image.check(dir=False) + +def get_miniimage(): + return sq.ImageReader(sq.Stream(mini_image.open())) + +def printStringsInImage(): + example = get_miniimage() + example.initialize() + + image = sq.SqueakImage() + image.from_reader(example) + + for each in image.objects: + if isinstance(each,sqm.W_BytesObject): + print ''.join(each.bytes) + +if __name__ == '__main__': + printStringsInImage() From akuhn at codespeak.net Wed Oct 24 18:45:14 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Wed, 24 Oct 2007 18:45:14 +0200 (CEST) Subject: [pypy-svn] r47849 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071024164514.6C42180DE@code0.codespeak.net> Author: akuhn Date: Wed Oct 24 18:45:14 2007 New Revision: 47849 Modified: pypy/dist/pypy/lang/smalltalk/constants.py Log: added constants for special object indices Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Wed Oct 24 18:45:14 2007 @@ -12,3 +12,50 @@ CLASS_SUPERCLASS_INDEX = 0 CLASS_METHODDICT_INDEX = 1 CLASS_FORMAT_INDEX = 2 + + +# ----- special objects indices ------- + +SO_NIL = 0 +SO_FALSE = 1 +SO_TRUE = 2 +SO_SCHEDULERASSOCIATIONPOINTER = 3 +SO_BITMAP_CLASS = 4 +SO_SMALLINTEGER_CLASS = 5 +SO_STRING_CLASS = 6 +SO_ARRAY_CLASS = 7 +SO_SMALLTALK = 8 +SO_FLOAT_CLASS = 9 +SO_METHODCONTEXT_CLASS = 10 +SO_BLOCKCONTEXT_CLASS = 11 +SO_POINT_CLASS = 12 +SO_LARGEPOSITIVEINTEGER_CLASS = 13 +SO_DISPLAY_CLASS = 14 +SO_MESSAGE_CLASS = 15 +SO_COMPILEDMETHOD_CLASS = 16 +SO_LOW_SPACE_SEMAPHORE = 17 +SO_SEMAPHORE_CLASS = 18 +SO_CHARACTER_CLASS = 19 +SO_DOES_NOT_UNDERSTAND = 20 +SO_CANNOT_RETURN = 21 +# no clue what 22 is doing +SO_SPECIAL SELECTORS ARRAY = 23 +SO_CHARACTER TABLE ARRAY = 24 +SO_MUST_BE_BOOLEAN = 25 +SO_BYTEARRAY_CLASS = 26 +SO_PROCESS_CLASS = 27 +SO_COMPACT_CLASSES_ARRAY = 28 +SO_DELAY_SEMAPHORE = 29 +SO_USER_INTERRUPT_SEMAPHORE = 30 +SO_FLOAT_ZERO = 31 +SO_LARGEPOSITIVEINTEGER_ZERO = 32 +SO_A_POINT = 33 +SO_CANNOT_INTERPRET = 34 +SO_A_METHODCONTEXT = 35 +# no clue what 36 is doing +SO_A_BLOCKCONTEXT = 37 +SO_AN_ARRAY = 38 +SO_PSEUDOCONTEXT_CLASS = 39 +SO_TRANSLATEDMETHOD_CLASS = 40 +SO_FINALIZATION_SEMPAHORE = 41 +SO_LARGENEGATIVEINTEGER_CLASS = 42 From akuhn at codespeak.net Wed Oct 24 18:51:29 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Wed, 24 Oct 2007 18:51:29 +0200 (CEST) Subject: [pypy-svn] r47850 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071024165129.9673580DE@code0.codespeak.net> Author: akuhn Date: Wed Oct 24 18:51:29 2007 New Revision: 47850 Modified: pypy/dist/pypy/lang/smalltalk/constants.py Log: fixed typo in constants Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Wed Oct 24 18:51:29 2007 @@ -39,8 +39,8 @@ SO_DOES_NOT_UNDERSTAND = 20 SO_CANNOT_RETURN = 21 # no clue what 22 is doing -SO_SPECIAL SELECTORS ARRAY = 23 -SO_CHARACTER TABLE ARRAY = 24 +SO_SPECIAL_SELECTORS_ARRAY = 23 +SO_CHARACTER_TABLE_ARRAY = 24 SO_MUST_BE_BOOLEAN = 25 SO_BYTEARRAY_CLASS = 26 SO_PROCESS_CLASS = 27 From lukas at codespeak.net Wed Oct 24 18:51:29 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Wed, 24 Oct 2007 18:51:29 +0200 (CEST) Subject: [pypy-svn] r47851 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024165129.EADE980DE@code0.codespeak.net> Author: lukas Date: Wed Oct 24 18:51:29 2007 New Revision: 47851 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (lr) bit shift primitive Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Wed Oct 24 18:51:29 2007 @@ -331,16 +331,16 @@ raise MissingBytecode def bytecodePrimBitShift(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush(primitives.BIT_SHIFT, "bitShift:", 1, interp) def bytecodePrimDiv(self, interp): self.callPrimitiveAndPush(primitives.DIV, "//", 1, interp) def bytecodePrimBitAnd(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush(primitives.BIT_AND, "&&", 1, interp) def bytecodePrimBitOr(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush(primitives.BIT_OR, "||", 1, interp) def bytecodePrimAt(self, interp): raise MissingBytecode Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Wed Oct 24 18:51:29 2007 @@ -77,11 +77,18 @@ MOD = 11 DIV = 12 QUO = 13 +BIT_AND = 14 +BIT_OR = 15 +BIT_XOR = 16 +BIT_SHIFT = 17 math_ops = { ADD: operator.add, SUBTRACT: operator.sub, MULTIPLY: operator.mul, + BIT_AND: operator.and_, + BIT_OR: operator.or_, + BIT_XOR: operator.xor } for (code,op) in math_ops.items(): @primitive(code) @@ -141,6 +148,30 @@ if argument == 0: raise PrimitiveFailedError() return wrap_int(receiver // argument) + +# #bitShift: -- return the shifted value + at primitive(BIT_SHIFT) + at stack(2) +def func(stack): + [w_receiver, w_argument] = stack + receiver = unwrap_int(w_receiver) + argument = unwrap_int(w_argument) + + # heh, no shifting at all + if argument == 0: + return w_receiver + + # left shift, must fail if we loose bits beyond 32 + if argument > 0: + shifted = receiver << argument + if (shifted >> argument) != receiver: + raise PrimitiveFailedError() + return wrap_int(shifted) + + # right shift, ok to lose bits + else: + return wrap_int(receiver >> -argument) + # ___________________________________________________________________________ # Float Primitives Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Wed Oct 24 18:51:29 2007 @@ -78,7 +78,57 @@ def test_small_int_quo_fail(): prim_fails(p.QUO, [12, 0]) - +def test_small_int_bit_and(): + assert prim(p.BIT_AND, [2, 4]).value == 0 + assert prim(p.BIT_AND, [2, 3]).value == 2 + assert prim(p.BIT_AND, [3, 4]).value == 0 + assert prim(p.BIT_AND, [4, 4]).value == 4 + +def test_small_int_bit_or(): + assert prim(p.BIT_OR, [2, 4]).value == 6 + assert prim(p.BIT_OR, [2, 3]).value == 3 + assert prim(p.BIT_OR, [3, 4]).value == 7 + assert prim(p.BIT_OR, [4, 4]).value == 4 + +def test_small_int_bit_xor(): + assert prim(p.BIT_XOR, [2, 4]).value == 6 + assert prim(p.BIT_XOR, [2, 3]).value == 1 + assert prim(p.BIT_XOR, [3, 4]).value == 7 + assert prim(p.BIT_XOR, [4, 4]).value == 0 + +def test_small_int_bit_shift(): + assert prim(p.BIT_SHIFT, [0, -3]).value == 0 + assert prim(p.BIT_SHIFT, [0, -2]).value == 0 + assert prim(p.BIT_SHIFT, [0, -1]).value == 0 + assert prim(p.BIT_SHIFT, [0, 0]).value == 0 + assert prim(p.BIT_SHIFT, [0, 1]).value == 0 + assert prim(p.BIT_SHIFT, [0, 2]).value == 0 + assert prim(p.BIT_SHIFT, [0, 3]).value == 0 + +def test_small_int_bit_shift_positive(): + assert prim(p.BIT_SHIFT, [4, -3]).value == 0 + assert prim(p.BIT_SHIFT, [4, -2]).value == 1 + assert prim(p.BIT_SHIFT, [4, -1]).value == 2 + assert prim(p.BIT_SHIFT, [4, 0]).value == 4 + assert prim(p.BIT_SHIFT, [4, 1]).value == 8 + assert prim(p.BIT_SHIFT, [4, 2]).value == 16 + assert prim(p.BIT_SHIFT, [4, 3]).value == 32 + +def test_small_int_bit_shift_negative(): + assert prim(p.BIT_SHIFT, [-4, -3]).value == -1 + assert prim(p.BIT_SHIFT, [-4, -2]).value == -1 + assert prim(p.BIT_SHIFT, [-4, -1]).value == -2 + assert prim(p.BIT_SHIFT, [-4, 0]).value == -4 + assert prim(p.BIT_SHIFT, [-4, 1]).value == -8 + assert prim(p.BIT_SHIFT, [-4, 2]).value == -16 + assert prim(p.BIT_SHIFT, [-4, 3]).value == -32 + +def test_small_int_bit_shift_fail(): + prim_fails(p.BIT_SHIFT, [4, 32]) + prim_fails(p.BIT_SHIFT, [4, 31]) + prim_fails(p.BIT_SHIFT, [4, 30]) + prim_fails(p.BIT_SHIFT, [4, 29]) + prim_fails(p.BIT_SHIFT, [4, 28]) def test_float(): assert prim(p.FLOAT_ADD, [1.0,2.0]).value == 3.0 From lukas at codespeak.net Wed Oct 24 18:54:56 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Wed, 24 Oct 2007 18:54:56 +0200 (CEST) Subject: [pypy-svn] r47852 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071024165456.AF4BB80EE@code0.codespeak.net> Author: lukas Date: Wed Oct 24 18:54:56 2007 New Revision: 47852 Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (lr) some more bit shift tests Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Wed Oct 24 18:54:56 2007 @@ -113,6 +113,7 @@ assert prim(p.BIT_SHIFT, [4, 1]).value == 8 assert prim(p.BIT_SHIFT, [4, 2]).value == 16 assert prim(p.BIT_SHIFT, [4, 3]).value == 32 + assert prim(p.BIT_SHIFT, [4, 27]).value == 536870912 def test_small_int_bit_shift_negative(): assert prim(p.BIT_SHIFT, [-4, -3]).value == -1 @@ -122,6 +123,7 @@ assert prim(p.BIT_SHIFT, [-4, 1]).value == -8 assert prim(p.BIT_SHIFT, [-4, 2]).value == -16 assert prim(p.BIT_SHIFT, [-4, 3]).value == -32 + assert prim(p.BIT_SHIFT, [-4, 27]).value == -536870912 def test_small_int_bit_shift_fail(): prim_fails(p.BIT_SHIFT, [4, 32]) From lukas at codespeak.net Wed Oct 24 19:10:53 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Wed, 24 Oct 2007 19:10:53 +0200 (CEST) Subject: [pypy-svn] r47853 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071024171053.DCA4480E9@code0.codespeak.net> Author: lukas Date: Wed Oct 24 19:10:53 2007 New Revision: 47853 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py Log: added some more primitive bytecodes Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Wed Oct 24 19:10:53 2007 @@ -337,19 +337,19 @@ self.callPrimitiveAndPush(primitives.DIV, "//", 1, interp) def bytecodePrimBitAnd(self, interp): - self.callPrimitiveAndPush(primitives.BIT_AND, "&&", 1, interp) + self.callPrimitiveAndPush(primitives.BIT_AND, "&", 1, interp) def bytecodePrimBitOr(self, interp): - self.callPrimitiveAndPush(primitives.BIT_OR, "||", 1, interp) + self.callPrimitiveAndPush(primitives.BIT_OR, "|", 1, interp) def bytecodePrimAt(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush(primitives.AT, "at:", 1, interp) def bytecodePrimAtPut(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush(primitives.AT_PUT, "at:put:", 2, interp) def bytecodePrimSize(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush(primitives.SIZE, "size", 0, interp) def bytecodePrimNext(self, interp): raise MissingBytecode @@ -376,19 +376,19 @@ raise MissingBytecode def bytecodePrimDo(self, interp): - raise MissingBytecode + self._sendSelfSelector("do:", 1, interp) def bytecodePrimNew(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush(primitives.NEW, "new", 0, interp) def bytecodePrimNewWithArg(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush(primitives.NEW_WITH_ARG, "new:", 1, interp) def bytecodePrimPointX(self, interp): - raise MissingBytecode + self._sendSelfSelector("x", 0, interp) def bytecodePrimPointY(self, interp): - raise MissingBytecode + self._sendSelfSelector("y", 0, interp) class Interpreter: From tverwaes at codespeak.net Wed Oct 24 19:16:06 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 19:16:06 +0200 (CEST) Subject: [pypy-svn] r47854 - pypy/dist/pypy/lang/smalltalk/tools Message-ID: <20071024171606.895FE80E9@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 19:16:06 2007 New Revision: 47854 Modified: pypy/dist/pypy/lang/smalltalk/tools/analyseimage.py Log: testing methods Modified: pypy/dist/pypy/lang/smalltalk/tools/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tools/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tools/analyseimage.py Wed Oct 24 19:16:06 2007 @@ -2,6 +2,7 @@ import py from pypy.lang.smalltalk import squeakimage as sq from pypy.lang.smalltalk import model as sqm +from pypy.lang.smalltalk import interpreter as sqi mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') @@ -11,16 +12,44 @@ def get_miniimage(): return sq.ImageReader(sq.Stream(mini_image.open())) -def printStringsInImage(): +def create_squeakimage(): example = get_miniimage() example.initialize() image = sq.SqueakImage() - image.from_reader(example) - + image.from_reader(example) + return image + +def printStringsInImage(): + image = create_squeakimage() for each in image.objects: if isinstance(each,sqm.W_BytesObject): - print ''.join(each.bytes) + print repr(''.join(each.bytes)) + +def testCompiledMethods(): + image = create_squeakimage() + amethod = None + skip = 0 + + for each in image.objects: + if isinstance(each,sqm.W_CompiledMethod): + if (amethod == None and + each.argsize == 0 and + each.tempsize == 0 and + each.primitive == 1 and skip >= 0): + amethod = each + else: + skip += 1 + #print "%d %d %d" % (each.argsize, each.tempsize, each.primitive) + + # receiver, arguments + w_frame = amethod.createFrame("receiver", []) + interp = sqi.Interpreter() + interp.activeContext = w_frame + interp.interpret() + +def test_do(): + testCompiledMethods() if __name__ == '__main__': - printStringsInImage() + test_do() From tverwaes at codespeak.net Wed Oct 24 19:48:25 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 19:48:25 +0200 (CEST) Subject: [pypy-svn] r47855 - pypy/dist/pypy/lang/smalltalk/tools Message-ID: <20071024174825.2685D80DE@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 19:48:24 2007 New Revision: 47855 Modified: pypy/dist/pypy/lang/smalltalk/tools/analyseimage.py Log: hacky hacky Modified: pypy/dist/pypy/lang/smalltalk/tools/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tools/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tools/analyseimage.py Wed Oct 24 19:48:24 2007 @@ -38,15 +38,31 @@ each.tempsize == 0 and each.primitive == 1 and skip >= 0): amethod = each + + if len(amethod.bytes) == 0: + print "Found method with bodylenght 0" + amethod = None else: skip += 1 #print "%d %d %d" % (each.argsize, each.tempsize, each.primitive) # receiver, arguments - w_frame = amethod.createFrame("receiver", []) interp = sqi.Interpreter() + + anObject = sqm.W_PointersObject(sqm.W_Class(None,None,100),0) + for i in range(0,99): + anObject.store(i, interp.ONE) + + w_frame = amethod.createFrame(anObject, []) interp.activeContext = w_frame - interp.interpret() + w_frame.push(interp.TRUE) + + while True: + try: + interp.step() + print interp.activeContext.stack + except sqi.ReturnFromTopLevel, e: + return e.object def test_do(): testCompiledMethods() From cfbolz at codespeak.net Wed Oct 24 19:51:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 24 Oct 2007 19:51:22 +0200 (CEST) Subject: [pypy-svn] r47856 - in pypy/dist/pypy/lang/smalltalk: tool tools Message-ID: <20071024175122.1E63080BB@code0.codespeak.net> Author: cfbolz Date: Wed Oct 24 19:51:21 2007 New Revision: 47856 Added: pypy/dist/pypy/lang/smalltalk/tool/ - copied from r47855, pypy/dist/pypy/lang/smalltalk/tools/ Removed: pypy/dist/pypy/lang/smalltalk/tools/ Log: consistent names of the tools directory across the PyPy project From tverwaes at codespeak.net Wed Oct 24 20:13:05 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 20:13:05 +0200 (CEST) Subject: [pypy-svn] r47857 - pypy/dist/pypy/lang/smalltalk/tool Message-ID: <20071024181305.08FEB80F4@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 20:13:04 2007 New Revision: 47857 Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: hacky hacky, we made out first loaded method pass (apparently some primitive call on self) Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Wed Oct 24 20:13:04 2007 @@ -33,17 +33,17 @@ for each in image.objects: if isinstance(each,sqm.W_CompiledMethod): - if (amethod == None and - each.argsize == 0 and + if (each.argsize == 0 and amethod == None and each.tempsize == 0 and - each.primitive == 1 and skip >= 0): - amethod = each - - if len(amethod.bytes) == 0: - print "Found method with bodylenght 0" - amethod = None - else: - skip += 1 + each.primitive == 1): + + if len(each.bytes) == 0: + pass + else: + if skip >= SKIPMETHODS: + amethod = each + else: + skip += 1 #print "%d %d %d" % (each.argsize, each.tempsize, each.primitive) # receiver, arguments @@ -55,7 +55,9 @@ w_frame = amethod.createFrame(anObject, []) interp.activeContext = w_frame - w_frame.push(interp.TRUE) + #w_frame.push(interp.TRUE) + w_frame.push(interp.ONE) + w_frame.push(interp.TWO) while True: try: @@ -64,6 +66,9 @@ except sqi.ReturnFromTopLevel, e: return e.object +# apply to Xth method +SKIPMETHODS=3 #X + def test_do(): testCompiledMethods() From tverwaes at codespeak.net Wed Oct 24 20:21:30 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 20:21:30 +0200 (CEST) Subject: [pypy-svn] r47858 - pypy/dist/pypy/lang/smalltalk/tool Message-ID: <20071024182130.6E79180F4@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 20:21:30 2007 New Revision: 47858 Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: now prints bytecode representation... this doesn't seem right at all... a 2-arg prim pushed without pushing an argument first?? Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Wed Oct 24 20:21:30 2007 @@ -59,6 +59,8 @@ w_frame.push(interp.ONE) w_frame.push(interp.TWO) + print "Going to execute code: " + print [ord(i) for i in amethod.bytes] while True: try: interp.step() From akuhn at codespeak.net Wed Oct 24 20:40:17 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Wed, 24 Oct 2007 20:40:17 +0200 (CEST) Subject: [pypy-svn] r47859 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071024184017.DC97180DE@code0.codespeak.net> Author: akuhn Date: Wed Oct 24 20:40:17 2007 New Revision: 47859 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: fixed off-by-one bug in deciphering header of compiled methods Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Wed Oct 24 20:40:17 2007 @@ -314,6 +314,7 @@ def fillin_compiledmethod(self, w_compiledmethod): header = self.chunk.data[0] + #---!!!---- 1 tagged pointer! #(index 0) 9 bits: main part of primitive number (#primitive) #(index 9) 8 bits: number of literals (#numLiterals) #(index 17) 1 bit: whether a large frame size is needed (#frameSize) @@ -321,12 +322,15 @@ #(index 24) 4 bits: number of arguments to the method (#numArgs) #(index 28) 1 bit: high-bit of primitive number (#primitive) #(index 29) 1 bit: flag bit, ignored by the VM (#flag) - primitive, literalsize, islarge, tempsize, numargs, highbit = ( - splitbits(header, [9,8,1,6,4,1])) - primitive = primitive + (highbit << 10) - #assert literalsize <= len(self.chunk.data) + _, primitive, literalsize, islarge, tempsize, numargs, highbit = ( + splitbits(header, [1,9,8,1,6,4,1])) + primitive = primitive + (highbit << 10) ##XXX todo, check this + # -------------------- + literals = [self.decode_pointer(pointer) + for pointer in self.chunk.data[:literalsize+1]] + # -------------------- l = [] - for each in self.chunk.data[(1 + literalsize):]: + for each in self.chunk.data[(literalsize+1):]: l.append(int2str(each)) if len(l) > 0: l[-1] = l[-1][:-(self.format & 3)] # omit odd bytes From akuhn at codespeak.net Wed Oct 24 20:57:45 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Wed, 24 Oct 2007 20:57:45 +0200 (CEST) Subject: [pypy-svn] r47860 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071024185745.D30A180EB@code0.codespeak.net> Author: akuhn Date: Wed Oct 24 20:57:45 2007 New Revision: 47860 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: omitting odd bytes in bytes and c'methods in the same way Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Wed Oct 24 20:57:45 2007 @@ -332,9 +332,7 @@ l = [] for each in self.chunk.data[(literalsize+1):]: l.append(int2str(each)) - if len(l) > 0: - l[-1] = l[-1][:-(self.format & 3)] # omit odd bytes - bytes = "".join(l) + bytes = "".join(l)[:-(self.format & 3)] w_compiledmethod.__init__( w_class = self.g_class.w_object, size = literalsize, From akuhn at codespeak.net Wed Oct 24 21:06:34 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Wed, 24 Oct 2007 21:06:34 +0200 (CEST) Subject: [pypy-svn] r47861 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071024190634.8FAA180F4@code0.codespeak.net> Author: akuhn Date: Wed Oct 24 21:06:31 2007 New Revision: 47861 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: keeping objects in same order as in image file Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Wed Oct 24 21:06:31 2007 @@ -73,7 +73,9 @@ class ImageReader(object): def __init__(self, stream): self.stream = stream - + self.chunks = {} + self.chunklist = [] + def initialize(self): self.read_header() self.read_body() @@ -96,14 +98,14 @@ self.stream.skipbytes(headersize - (9 * 4)) def read_body(self): - self.chunks = {} self.stream.reset_count() while self.stream.count < self.endofmemory: chunk, pos = self.read_object() + self.chunklist.append(chunk) self.chunks[pos + self.oldbaseaddress] = chunk self.stream.close() del self.stream - return self.chunks.values() # return for testing + return self.chunklist # return for testing def init_g_objects(self): for chunk in self.chunks.itervalues(): @@ -173,7 +175,7 @@ self.special_objects = [g_object.w_object for g_object in reader.chunks[reader.specialobjectspointer] .g_object.pointers] - self.objects = [chunk.g_object.w_object for chunk in reader.chunks.itervalues()] + self.objects = [chunk.g_object.w_object for chunk in reader.chunklist] def special(self, index): return self.special_objects[index] @@ -330,9 +332,10 @@ for pointer in self.chunk.data[:literalsize+1]] # -------------------- l = [] - for each in self.chunk.data[(literalsize+1):]: + for each in self.chunk.data[literalsize+1:]: l.append(int2str(each)) - bytes = "".join(l)[:-(self.format & 3)] + bytes = "".join(l) + bytes = bytes[:-(self.format & 3)] w_compiledmethod.__init__( w_class = self.g_class.w_object, size = literalsize, From tverwaes at codespeak.net Wed Oct 24 21:16:08 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 21:16:08 +0200 (CEST) Subject: [pypy-svn] r47862 - pypy/dist/pypy/lang/smalltalk/tool Message-ID: <20071024191608.5CC1A80E5@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 21:16:07 2007 New Revision: 47862 Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: trying to get the right bytecode from compiledmethods. committing in order to be able to switch machine :) Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Wed Oct 24 21:16:07 2007 @@ -6,9 +6,6 @@ mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') -def test_miniimageexists(): - assert mini_image.check(dir=False) - def get_miniimage(): return sq.ImageReader(sq.Stream(mini_image.open())) @@ -26,6 +23,12 @@ if isinstance(each,sqm.W_BytesObject): print repr(''.join(each.bytes)) +def printReadableBytecode(bytecode): + print "\n\nBytecode:\n---------------------" + print "\n".join([sqi.BYTECODE_TABLE[ord(i)].__name__ for i in bytecode]) + print "---------------------\n" + + def testCompiledMethods(): image = create_squeakimage() amethod = None @@ -35,7 +38,7 @@ if isinstance(each,sqm.W_CompiledMethod): if (each.argsize == 0 and amethod == None and each.tempsize == 0 and - each.primitive == 1): + each.primitive == 0): if len(each.bytes) == 0: pass @@ -56,11 +59,11 @@ w_frame = amethod.createFrame(anObject, []) interp.activeContext = w_frame #w_frame.push(interp.TRUE) - w_frame.push(interp.ONE) - w_frame.push(interp.TWO) + #w_frame.push(interp.ONE) + #w_frame.push(interp.TWO) + + printReadableBytecode(amethod.bytes) - print "Going to execute code: " - print [ord(i) for i in amethod.bytes] while True: try: interp.step() @@ -69,10 +72,11 @@ return e.object # apply to Xth method -SKIPMETHODS=3 #X +SKIPMETHODS=42 #X def test_do(): - testCompiledMethods() + #testCompiledMethods() + printStringsInImage() if __name__ == '__main__': test_do() From arigo at codespeak.net Wed Oct 24 22:24:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 24 Oct 2007 22:24:09 +0200 (CEST) Subject: [pypy-svn] r47863 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024202409.42FE080F0@code0.codespeak.net> Author: arigo Date: Wed Oct 24 22:24:08 2007 New Revision: 47863 Added: pypy/dist/pypy/lang/smalltalk/mirror.py (contents, props changed) Modified: pypy/dist/pypy/lang/smalltalk/classtable.py pypy/dist/pypy/lang/smalltalk/fakeimage.py pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_classtable.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_model.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (cfbolz, arigo) Big refactoring. Every Smalltalk object used as the class of another Smalltalk object now gets a "mirror" object that only exists in the interpreter. The mirror decodes the mind-bogglingly complicated bitfields and the method dict and stores it in nicely readable RPython fields. The internal class field of each Smalltalk object is now poiting to the mirror directly. This changes mostly everything done so far, but should be much more future-proof and cleaner from various points of view. Modified: pypy/dist/pypy/lang/smalltalk/classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/classtable.py Wed Oct 24 22:24:08 2007 @@ -1,69 +1,89 @@ -from pypy.lang.smalltalk.model import \ - W_Class, W_MetaClass, W_SmallInteger, POINTERS, BYTES +from pypy.lang.smalltalk import mirror + +def bootstrap_classmirror(instsize, m_superclass=None, m_metaclass=None, + name='?', format=mirror.POINTERS, varsized=False): + from pypy.lang.smalltalk import model + w_class = model.W_Object() # a dummy placeholder for testing + m = mirror.ClassMirror(w_class) + m.methoddict = {} + m.m_superclass = m_superclass + m.m_metaclass = m_metaclass + m.name = name + m.instance_size = instsize + m.instance_kind = format + m.instance_varsized = varsized or format != mirror.POINTERS + m.invalid = False + mirror.mirrorcache.cache[w_class] = m + return m # ___________________________________________________________________________ # Core Bootstrapping Objects classtable = {} def create_classtable(): - def define_core_cls(clsnm, clsobj): - clsobj.name = clsnm - classtable[clsnm] = clsobj - globals()[clsnm] = clsobj - return clsobj + def define_core_cls(name, m_superclass, m_metaclass): + assert name.startswith('m_') + mirror = bootstrap_classmirror(instsize=0, # XXX + m_superclass=m_superclass, + m_metaclass=m_metaclass, + name=name[2:]) + classtable[name] = mirror + globals()[name] = mirror + return mirror # Class Name Super class name cls_nm_tbl = [ - ["w_Object", "w_ProtoObject"], - ["w_Behavior", "w_Object"], - ["w_ClassDescription", "w_Behavior"], - ["w_Class", "w_ClassDescription"], - ["w_Metaclass", "w_ClassDescription"], + ["m_Object", "m_ProtoObject"], + ["m_Behavior", "m_Object"], + ["m_ClassDescription", "m_Behavior"], + ["m_Class", "m_ClassDescription"], + ["m_Metaclass", "m_ClassDescription"], ] - define_core_cls("w_ProtoObjectClass", W_MetaClass(None, None)) - define_core_cls("w_ProtoObject", W_Class(w_ProtoObjectClass, None)) + define_core_cls("m_ProtoObjectClass", None, None) + define_core_cls("m_ProtoObject", None, m_ProtoObjectClass) for (cls_nm, super_cls_nm) in cls_nm_tbl: meta_nm = cls_nm + "Class" meta_super_nm = super_cls_nm + "Class" - metacls = define_core_cls( - meta_nm, W_MetaClass(None, classtable[meta_super_nm], name=meta_nm)) - define_core_cls( - cls_nm, W_Class(metacls, classtable[super_cls_nm], name=cls_nm)) - w_ProtoObjectClass.w_superclass = w_Class - for nm, w_cls_obj in classtable.items(): - if w_cls_obj.ismetaclass(): - w_cls_obj.w_class = w_Metaclass + m_metacls = define_core_cls(meta_nm, classtable[meta_super_nm], None) + define_core_cls(cls_nm, classtable[super_cls_nm], m_metacls) + m_ProtoObjectClass.m_superclass = m_Class + # at this point, all classes that still lack a m_metaclass are themselves + # metaclasses + for nm, m_cls_obj in classtable.items(): + if m_cls_obj.m_metaclass is None: + m_cls_obj.m_metaclass = m_Metaclass create_classtable() # ___________________________________________________________________________ # Other classes -def define_cls(cls_nm, supercls_nm, instvarsize=0, format=POINTERS): +def define_cls(cls_nm, supercls_nm, instvarsize=0, format=mirror.POINTERS): meta_nm = cls_nm + "Class" meta_super_nm = supercls_nm + "Class" - w_meta_cls = globals()[meta_nm] = classtable[meta_nm] = \ - W_MetaClass(w_Metaclass, - classtable[meta_super_nm], - name=meta_nm) - w_cls = globals()[cls_nm] = classtable[cls_nm] = \ - W_Class(w_meta_cls, - classtable[supercls_nm], - name=cls_nm, - instvarsize=instvarsize, - format=format) - -define_cls("w_Magnitude", "w_Object") -define_cls("w_Character", "w_Magnitude", instvarsize=1) -define_cls("w_Number", "w_Magnitude") -define_cls("w_Integer", "w_Number") -define_cls("w_SmallInteger", "w_Integer") -define_cls("w_Float", "w_Number") -define_cls("w_Collection", "w_Object") -define_cls("w_SequencableCollection", "w_Collection") -define_cls("w_ArrayedCollection", "w_SequencableCollection") -define_cls("w_String", "w_ArrayedCollection") -define_cls("w_ByteString", "w_String", format=BYTES) -define_cls("w_UndefinedObject", "w_Object") -define_cls("w_Boolean", "w_Object") -define_cls("w_True", "w_Boolean") -define_cls("w_False", "w_Boolean") + m_meta_cls = globals()[meta_nm] = classtable[meta_nm] = \ + bootstrap_classmirror(0, # XXX + classtable[meta_super_nm], + m_Metaclass) + m_cls = globals()[cls_nm] = classtable[cls_nm] = \ + bootstrap_classmirror(instvarsize, + classtable[supercls_nm], + m_meta_cls, + format=format) + +define_cls("m_Magnitude", "m_Object") +define_cls("m_Character", "m_Magnitude", instvarsize=1) +define_cls("m_Number", "m_Magnitude") +define_cls("m_Integer", "m_Number") +define_cls("m_SmallInteger", "m_Integer") +define_cls("m_Float", "m_Number", format=mirror.BYTES) +define_cls("m_Collection", "m_Object") +define_cls("m_SequencableCollection", "m_Collection") +define_cls("m_ArrayedCollection", "m_SequencableCollection") +define_cls("m_String", "m_ArrayedCollection") +define_cls("m_ByteString", "m_String", format=mirror.BYTES) +define_cls("m_UndefinedObject", "m_Object") +define_cls("m_Boolean", "m_Object") +define_cls("m_True", "m_Boolean") +define_cls("m_False", "m_Boolean") +define_cls("m_ByteArray", "m_ArrayedCollection", format=mirror.BYTES) +define_cls("m_CompiledMethod", "m_ByteArray", format=mirror.COMPILED_METHOD) Modified: pypy/dist/pypy/lang/smalltalk/fakeimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/fakeimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/fakeimage.py Wed Oct 24 22:24:08 2007 @@ -7,14 +7,14 @@ def wrap_int(i): if i <= 0x3FFFFFFF and i >= -0x40000000: - return model.W_SmallInteger(ct.w_SmallInteger, i) + return model.W_SmallInteger(i) raise NotImplementedError def wrap_float(i): - return model.W_Float(ct.w_Float,i) + return model.W_Float(i) def wrap_string(str): - w_inst = ct.w_ByteString.new(len(str)) + w_inst = ct.m_ByteString.new(len(str)) for i in range(len(str)): w_inst.setbyte(i, ord(str[i])) return w_inst @@ -23,9 +23,9 @@ return CharacterTable[ord(c)] def ord_w_char(w_c): - assert w_c.w_class is ct.w_Character + assert w_c.getclassmirror() is ct.m_Character w_ord = w_c.fetch(CHARACTER_VALUE_INDEX) - assert w_ord.w_class is ct.w_SmallInteger + assert w_ord.getclassmirror() is ct.m_SmallInteger assert isinstance(w_ord, model.W_SmallInteger) return w_ord.value @@ -41,15 +41,15 @@ def wrap_char_table(): global CharacterTable def bld_char(i): - w_cinst = ct.w_Character.new() + w_cinst = ct.m_Character.new() w_cinst.store(CHARACTER_VALUE_INDEX, wrap_int(i)) return w_cinst CharacterTable = [bld_char(i) for i in range(256)] wrap_char_table() -w_true = ct.w_True.new() -w_false = ct.w_False.new() -w_nil = ct.w_UndefinedObject.new() +w_true = ct.m_True.new() +w_false = ct.m_False.new() +w_nil = ct.m_UndefinedObject.new() w_mone = wrap_int(-1) w_zero = wrap_int(0) w_one = wrap_int(1) Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Wed Oct 24 22:24:08 2007 @@ -10,9 +10,8 @@ """Illegal Store.""" -class W_ContextFrame(model.W_Object): - def __init__(self, w_class, method, receiver, arguments, sender = None): - model.W_Object.__init__(self, w_class) +class W_MethodContext(model.W_Object): + def __init__(self, method, receiver, arguments, sender = None): self.method = method self.receiver = receiver self.sender = sender @@ -20,6 +19,13 @@ self.temps = arguments + [None] * method.tempsize self.pc = 0 + def getclassmirror(self): + from pypy.lang.smalltalk.classtable import m_MethodContext # XXX do me + return m_MethodContext + + def gethash(self): + return 44 # XXX + def pop(self): return self.stack.pop() @@ -120,15 +126,15 @@ def _sendSelfSelector(self, selector, argcount, interp): receiver = self.peek(argcount) self._sendSelector(selector, argcount, interp, - receiver, receiver.w_class) + receiver, receiver.getclassmirror()) def _sendSuperSelector(self, selector, argcount, interp): self._sendSelector(selector, argcount, interp, self.receiver, - self.method.w_compiledin.w_superclass) + self.method.m_compiledin.m_superclass) def _sendSelector(self, selector, argcount, interp, - receiver, receiverclass): - method = receiverclass.lookup(selector) + receiver, m_receiverclass): + method = m_receiverclass.lookup(selector) assert method if method.primitive: func = primitives.prim_table[method.primitive] @@ -422,77 +428,77 @@ self.object = object BYTECODE_RANGES = [ - ( 0, 15, W_ContextFrame.pushReceiverVariableBytecode), - ( 16, 31, W_ContextFrame.pushTemporaryVariableBytecode), - ( 32, 63, W_ContextFrame.pushLiteralConstantBytecode), - ( 64, 95, W_ContextFrame.pushLiteralVariableBytecode), - ( 96, 103, W_ContextFrame.storeAndPopReceiverVariableBytecode), - (104, 111, W_ContextFrame.storeAndPopTemporaryVariableBytecode), - (112, W_ContextFrame.pushReceiverBytecode), - (113, W_ContextFrame.pushConstantTrueBytecode), - (114, W_ContextFrame.pushConstantFalseBytecode), - (115, W_ContextFrame.pushConstantNilBytecode), - (116, W_ContextFrame.pushConstantMinusOneBytecode), - (117, W_ContextFrame.pushConstantZeroBytecode), - (118, W_ContextFrame.pushConstantOneBytecode), - (119, W_ContextFrame.pushConstantTwoBytecode), - (120, W_ContextFrame.returnReceiver), - (121, W_ContextFrame.returnTrue), - (122, W_ContextFrame.returnFalse), - (123, W_ContextFrame.returnNil), - (124, W_ContextFrame.returnTopFromMethod), - (125, W_ContextFrame.returnTopFromBlock), - (126, W_ContextFrame.unknownBytecode), - (127, W_ContextFrame.unknownBytecode), - (128, W_ContextFrame.extendedPushBytecode), - (129, W_ContextFrame.extendedStoreBytecode), - (130, W_ContextFrame.extendedStoreAndPopBytecode), - (131, W_ContextFrame.singleExtendedSendBytecode), - (132, W_ContextFrame.doubleExtendedDoAnythingBytecode), - (133, W_ContextFrame.singleExtendedSuperBytecode), - (134, W_ContextFrame.secondExtendedSendBytecode), - (135, W_ContextFrame.popStackBytecode), - (136, W_ContextFrame.duplicateTopBytecode), - (137, W_ContextFrame.pushActiveContextBytecode), - (138, 143, W_ContextFrame.experimentalBytecode), - (144, 151, W_ContextFrame.shortUnconditionalJump), - (152, 159, W_ContextFrame.shortConditionalJump), - (160, 167, W_ContextFrame.longUnconditionalJump), - (168, 171, W_ContextFrame.longJumpIfTrue), - (172, 175, W_ContextFrame.longJumpIfFalse), - (176, W_ContextFrame.bytecodePrimAdd), - (177, W_ContextFrame.bytecodePrimSubtract), - (178, W_ContextFrame.bytecodePrimLessThan), - (179, W_ContextFrame.bytecodePrimGreaterThan), - (180, W_ContextFrame.bytecodePrimLessOrEqual), - (181, W_ContextFrame.bytecodePrimGreaterOrEqual), - (182, W_ContextFrame.bytecodePrimEqual), - (183, W_ContextFrame.bytecodePrimNotEqual), - (184, W_ContextFrame.bytecodePrimMultiply), - (185, W_ContextFrame.bytecodePrimDivide), - (186, W_ContextFrame.bytecodePrimMod), - (187, W_ContextFrame.bytecodePrimMakePoint), - (188, W_ContextFrame.bytecodePrimBitShift), - (189, W_ContextFrame.bytecodePrimDiv), - (190, W_ContextFrame.bytecodePrimBitAnd), - (191, W_ContextFrame.bytecodePrimBitOr), - (192, W_ContextFrame.bytecodePrimAt), - (193, W_ContextFrame.bytecodePrimAtPut), - (194, W_ContextFrame.bytecodePrimSize), - (195, W_ContextFrame.bytecodePrimNext), - (196, W_ContextFrame.bytecodePrimNextPut), - (197, W_ContextFrame.bytecodePrimAtEnd), - (198, W_ContextFrame.bytecodePrimEquivalent), - (199, W_ContextFrame.bytecodePrimClass), - (200, W_ContextFrame.bytecodePrimBlockCopy), - (201, W_ContextFrame.bytecodePrimValue), - (202, W_ContextFrame.bytecodePrimValueWithArg), - (203, W_ContextFrame.bytecodePrimDo), - (204, W_ContextFrame.bytecodePrimNew), - (205, W_ContextFrame.bytecodePrimNewWithArg), - (206, W_ContextFrame.bytecodePrimPointX), - (207, W_ContextFrame.bytecodePrimPointY), - (208, 255, W_ContextFrame.sendLiteralSelectorBytecode), + ( 0, 15, W_MethodContext.pushReceiverVariableBytecode), + ( 16, 31, W_MethodContext.pushTemporaryVariableBytecode), + ( 32, 63, W_MethodContext.pushLiteralConstantBytecode), + ( 64, 95, W_MethodContext.pushLiteralVariableBytecode), + ( 96, 103, W_MethodContext.storeAndPopReceiverVariableBytecode), + (104, 111, W_MethodContext.storeAndPopTemporaryVariableBytecode), + (112, W_MethodContext.pushReceiverBytecode), + (113, W_MethodContext.pushConstantTrueBytecode), + (114, W_MethodContext.pushConstantFalseBytecode), + (115, W_MethodContext.pushConstantNilBytecode), + (116, W_MethodContext.pushConstantMinusOneBytecode), + (117, W_MethodContext.pushConstantZeroBytecode), + (118, W_MethodContext.pushConstantOneBytecode), + (119, W_MethodContext.pushConstantTwoBytecode), + (120, W_MethodContext.returnReceiver), + (121, W_MethodContext.returnTrue), + (122, W_MethodContext.returnFalse), + (123, W_MethodContext.returnNil), + (124, W_MethodContext.returnTopFromMethod), + (125, W_MethodContext.returnTopFromBlock), + (126, W_MethodContext.unknownBytecode), + (127, W_MethodContext.unknownBytecode), + (128, W_MethodContext.extendedPushBytecode), + (129, W_MethodContext.extendedStoreBytecode), + (130, W_MethodContext.extendedStoreAndPopBytecode), + (131, W_MethodContext.singleExtendedSendBytecode), + (132, W_MethodContext.doubleExtendedDoAnythingBytecode), + (133, W_MethodContext.singleExtendedSuperBytecode), + (134, W_MethodContext.secondExtendedSendBytecode), + (135, W_MethodContext.popStackBytecode), + (136, W_MethodContext.duplicateTopBytecode), + (137, W_MethodContext.pushActiveContextBytecode), + (138, 143, W_MethodContext.experimentalBytecode), + (144, 151, W_MethodContext.shortUnconditionalJump), + (152, 159, W_MethodContext.shortConditionalJump), + (160, 167, W_MethodContext.longUnconditionalJump), + (168, 171, W_MethodContext.longJumpIfTrue), + (172, 175, W_MethodContext.longJumpIfFalse), + (176, W_MethodContext.bytecodePrimAdd), + (177, W_MethodContext.bytecodePrimSubtract), + (178, W_MethodContext.bytecodePrimLessThan), + (179, W_MethodContext.bytecodePrimGreaterThan), + (180, W_MethodContext.bytecodePrimLessOrEqual), + (181, W_MethodContext.bytecodePrimGreaterOrEqual), + (182, W_MethodContext.bytecodePrimEqual), + (183, W_MethodContext.bytecodePrimNotEqual), + (184, W_MethodContext.bytecodePrimMultiply), + (185, W_MethodContext.bytecodePrimDivide), + (186, W_MethodContext.bytecodePrimMod), + (187, W_MethodContext.bytecodePrimMakePoint), + (188, W_MethodContext.bytecodePrimBitShift), + (189, W_MethodContext.bytecodePrimDiv), + (190, W_MethodContext.bytecodePrimBitAnd), + (191, W_MethodContext.bytecodePrimBitOr), + (192, W_MethodContext.bytecodePrimAt), + (193, W_MethodContext.bytecodePrimAtPut), + (194, W_MethodContext.bytecodePrimSize), + (195, W_MethodContext.bytecodePrimNext), + (196, W_MethodContext.bytecodePrimNextPut), + (197, W_MethodContext.bytecodePrimAtEnd), + (198, W_MethodContext.bytecodePrimEquivalent), + (199, W_MethodContext.bytecodePrimClass), + (200, W_MethodContext.bytecodePrimBlockCopy), + (201, W_MethodContext.bytecodePrimValue), + (202, W_MethodContext.bytecodePrimValueWithArg), + (203, W_MethodContext.bytecodePrimDo), + (204, W_MethodContext.bytecodePrimNew), + (205, W_MethodContext.bytecodePrimNewWithArg), + (206, W_MethodContext.bytecodePrimPointX), + (207, W_MethodContext.bytecodePrimPointY), + (208, 255, W_MethodContext.sendLiteralSelectorBytecode), ] def initialize_bytecode_table(): Added: pypy/dist/pypy/lang/smalltalk/mirror.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/mirror.py Wed Oct 24 22:24:08 2007 @@ -0,0 +1,150 @@ +import weakref +from pypy.lang.smalltalk import model, constants + +POINTERS = 0 +BYTES = 1 +WORDS = 2 +WEAK_POINTERS = 3 +COMPILED_METHOD = 4 + +class ClassMirrorError(Exception): + pass + +def unwrap_int(w_value): + if isinstance(w_value, model.W_SmallInteger): + return w_value.value + raise ClassMirrorError("expected a W_SmallInteger, got %s" % (w_value,)) + + +class ClassMirror(object): + def __init__(self, w_self): + self.w_self = w_self + self.invalidate() + + def invalidate(self): + self.methoddict = {} + self.m_superclass = None + self.m_metaclass = None + self.name = '?' + self.invalid = True + + def check(self): + if self.invalid: + self.update_mirror() + + def update_mirror(self): + "Update the ClassMirror with data from the w_self class." + w_self = self.w_self + # read and painfully decode the format + classformat = unwrap_int(w_self.fetch(constants.CLASS_FORMAT_INDEX)) + # The classformat in Squeak, as an integer value, is: + # <2 bits=instSize//64><5 bits=cClass><4 bits=instSpec> + # <6 bits=instSize\\64><1 bit=0> + # In Slang the value is read directly as a boxed integer, so that + # the code gets a "pointer" whose bits are set as above, but + # shifted one bit to the left and with the lowest bit set to 1. + + # compute the instance size (really the size, not the number of bytes) + instsize_lo = (classformat >> 1) & 0x3F + instsize_hi = (classformat >> (9 + 1)) & 0xC0 + self.instance_size = instsize_lo | instsize_hi + # decode the instSpec + format = (classformat >> 7) & 15 + self.instance_varsized = format >= 2 + if format < 4: + self.instance_kind = POINTERS + elif format == 4: + self.instance_kind = WEAK_POINTERS + elif format == 6: + self.instance_kind = WORDS + if self.instance_kind != 0: + raise ClassMirrorError("can't have both words and a non-zero " + "base instance size") + elif 8 <= format <= 11: + self.instance_kind = BYTES + if self.instance_kind != 0: + raise ClassMirrorError("can't have both bytes and a non-zero " + "base instance size") + elif 12 <= format <= 15: + self.instance_kind = COMPILED_METHOD + else: + raise ClassMirrorError("unknown format %d" % (format,)) + self.invalid = False + + def new(self, extrasize=0): + self.check() + if self.instance_kind == POINTERS: + return model.W_PointersObject(self, self.instance_size + extrasize) + elif self.instance_kind == WORDS: + return model.W_WordsObject(self, extrasize) + elif self.instance_kind == BYTES: + return model.W_BytesObject(self, extrasize) + else: + raise NotImplementedError(self.instance_kind) + + # _______________________________________________________________ + # Methods for querying the format word, taken from the blue book: + # + # included so that we can reproduce code from the reference impl + # more easily + + def ispointers(self): + " True if instances of this class have data stored as pointers " + XXX # what about weak pointers? + return self.format == POINTERS + + def iswords(self): + " True if instances of this class have data stored as numerical words " + XXX # what about weak pointers? + return self.format in (POINTERS, WORDS) + + def isbytes(self): + " True if instances of this class have data stored as numerical bytes " + return self.format == BYTES + + def isvariable(self): + " True if instances of this class have indexed inst variables " + return self.instance_varsized + + def instsize(self): + " Number of named instance variables for each instance of this class " + return self.instance_size + + # _______________________________________________________________ + # Methods for querying the format word, taken from the blue book: + + def __repr__(self): + return "" % (self.name,) + + def lookup(self, selector): + if selector in self.methoddict: + return self.methoddict[selector] + elif self.m_superclass != None: + return self.m_superclass.lookup(selector) + else: + return None + + def installmethod(self, selector, method): + "NOT_RPYTHON" # this is only for testing. + assert isinstance(method, model.W_CompiledMethod) + self.methoddict[selector] = method + method.m_compiledin = self + + +class MirrorCache: + def __init__(self): + self.cache = weakref.WeakKeyDictionary() + + def get_or_build(self, w_class): + try: + mirror = self.cache[w_class] + except KeyError: + mirror = self.cache[w_class] = ClassMirror(w_class) + return mirror + + def getmirror(self, w_class): + mirror = self.get_or_build(w_class) + mirror.check() + return mirror + +mirrorcache = MirrorCache() Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Wed Oct 24 22:24:08 2007 @@ -1,48 +1,69 @@ class W_Object(object): - def __init__(self, w_class): - self.w_class = w_class - self.w_hash = None # XXX initial value? def size(self): return 0 - def instvarsize(self): - return self.w_class.instvarsize - + def getclassmirror(self): + raise NotImplementedError + + def gethash(self): + raise NotImplementedError + def invariant(self): - return (hasattr(self, 'w_class') and - self.w_class != None) - - def as_w_class_instvarsize(self): - return self.vars[3] # hardcoded position of instvar + return True class W_SmallInteger(W_Object): - def __init__(self, w_class, value): - W_Object.__init__(self, w_class) + def __init__(self, value): self.value = value - + + def getclassmirror(self): + from pypy.lang.smalltalk.classtable import m_SmallInteger + return m_SmallInteger + + def gethash(self): + return self.value # XXX check this + def invariant(self): - W_Object.invariant(self) - assert hasattr(self, 'value') - assert self.value != None + return isinstance(self.value, int) class W_Float(W_Object): - def __init__(self, w_class, value): - W_Object.__init__(self, w_class) + def __init__(self, value): self.value = value - + + def getclassmirror(self): + from pypy.lang.smalltalk.classtable import m_Float + return m_Float + + def gethash(self): + return XXX # check this + def invariant(self): - return (W_Object.invariant(self) and - hasattr(self, 'value') and - self.value != None) - + return self.value is not None # XXX but later: + #return isinstance(self.value, float) + +class W_ObjectWithStoredClass(W_Object): + """ The base class of objects that store 'm_class' explicitly. """ + def __init__(self, m_class): + self.m_class = m_class + self.hash = 42 # XXX + + def getclassmirror(self): + return self.m_class -class W_PointersObject(W_Object): + def gethash(self): + return self.hash + + def invariant(self): + from pypy.lang.smalltalk.mirror import ClassMirror + return (isinstance(self.m_class, ClassMirror) and + 1)#isinstance(self.hash, int)) XXX + +class W_PointersObject(W_ObjectWithStoredClass): """ The normal object """ - def __init__(self, w_class, size=0): - W_Object.__init__(self, w_class) - self.vars = [None] * (w_class.instvarsize + size) + def __init__(self, m_class, size): + W_ObjectWithStoredClass.__init__(self, m_class) + self.vars = [None] * size def fetch(self, index): return self.vars[index] @@ -54,14 +75,12 @@ return len(self.vars) def invariant(self): - return (W_Object.invariant(self) and - hasattr(self, 'vars') and - self.vars != None) - + return (W_ObjectWithStoredClass.invariant(self) and + isinstance(self.vars, list)) -class W_BytesObject(W_Object): - def __init__(self, w_class, size): - W_Object.__init__(self, w_class) +class W_BytesObject(W_ObjectWithStoredClass): + def __init__(self, m_class, size): + W_ObjectWithStoredClass.__init__(self, m_class) self.bytes = ['\x00'] * size def getbyte(self, n): @@ -74,14 +93,16 @@ return len(self.bytes) def invariant(self): - return (W_Object.invariant(self) and - hasattr(self, 'bytes') and - self.bytes != None) - + if not W_ObjectWithStoredClass.invariant(self): + return False + for c in self.bytes: + if not isinstance(c, str) or len(c) != 1: + return False + return True -class W_WordsObject(W_Object): - def __init__(self, w_class, size): - W_Object.__init__(self, w_class) +class W_WordsObject(W_ObjectWithStoredClass): + def __init__(self, m_class, size): + W_ObjectWithStoredClass.__init__(self, m_class) self.words = [0] * size def getword(self, n): @@ -94,9 +115,8 @@ return len(self.words) def invariant(self): - return (W_Object.invariant(self) and - hasattr(self, 'words') and - self.words != None) + return (W_ObjectWithStoredClass.invariant(self) and + isinstance(self.words, list)) class W_CompiledMethod(W_Object): """My instances are methods suitable for interpretation by the virtual machine. This is the only class in the system whose instances intermix both indexable pointer fields and indexable integer fields. @@ -120,112 +140,36 @@ The trailer has two variant formats. In the first variant, the last byte is at least 252 and the last four bytes represent a source pointer into one of the sources files (see #sourcePointer). In the second variant, the last byte is less than 252, and the last several bytes are a compressed version of the names of the method's temporary variables. The number of bytes used for this purpose is the value of the last byte in the method. """ - def __init__(self, w_class, size, bytes="", argsize=0, - tempsize=0, primitive=0, w_compiledin=None): - W_Object.__init__(self, w_class) + def __init__(self, size, bytes, argsize=0, + tempsize=0, primitive=0, m_compiledin=None): self.literals = [None] * size - self.w_compiledin = w_compiledin + self.m_compiledin = m_compiledin self.bytes = bytes self.argsize = argsize self.tempsize = tempsize self.primitive = primitive + def getclassmirror(self): + from pypy.lang.smalltalk.classtable import m_CompiledMethod + return m_CompiledMethod + + def gethash(self): + return 43 # XXX + def createFrame(self, receiver, arguments, sender = None): - from pypy.lang.smalltalk.interpreter import W_ContextFrame + from pypy.lang.smalltalk.interpreter import W_MethodContext assert len(arguments) == self.argsize - return W_ContextFrame(None, self, receiver, arguments, sender) + return W_MethodContext(self, receiver, arguments, sender) def invariant(self): return (W_Object.invariant(self) and hasattr(self, 'literals') and - self.literals != None and + self.literals is not None and hasattr(self, 'bytes') and - self.bytes != None and + self.bytes is not None and hasattr(self, 'argsize') and - self.argsize != None and + self.argsize is not None and hasattr(self, 'tempsize') and - self.tempsize != None and + self.tempsize is not None and hasattr(self, 'primitive') and - self.primitive != None) - -# ____________________________________________________________ - -POINTERS = 0 -VAR_POINTERS = 1 -BYTES = 2 -WORDS = 3 -WEAK_POINTERS = 4 - -class W_Class(W_Object): - def __init__(self, w_class, w_superclass, instvarsize=0, - format=POINTERS, name="?"): - W_Object.__init__(self, w_class) - self.name = name - self.w_superclass = w_superclass - self.methoddict = {} - self.instvarsize = instvarsize - self.format = format - - # _______________________________________________________________ - # Methods for querying the format word, taken from the blue book: - # - # included so that we can reproduce code from the reference impl - # more easily - - def ispointers(self): - " True if instances of this class have data stored as pointers " - return self.format in (POINTERS, VAR_POINTERS) - - def iswords(self): - " True if instances of this class have data stored as numerical words " - return self.format in (POINTERS, VAR_POINTERS, WORDS) - - def isbytes(self): - " True if instances of this class have data stored as numerical bytes " - return self.format == BYTES - - def isvariable(self): - " True if instances of this class have indexed inst variables " - return self.format in (VAR_POINTERS, WORDS, BYTES) - - def instsize(self): - " Number of named instance variables for each instance of this class " - return self.instvarsize - - def ismetaclass(self): - return False - - # _______________________________________________________________ - # Methods for querying the format word, taken from the blue book: - - def new(self, size=0): - if self.format == POINTERS: - assert size == 0 - return W_PointersObject(self) - elif self.format == VAR_POINTERS: - return W_PointersObject(w_class=self, size=size) - elif self.format == WORDS: - return W_WordsObject(w_class=self, size=size) - elif self.format == BYTES: - return W_BytesObject(w_class=self, size=size) - else: - raise NotImplementedError(self.format) - - def __repr__(self): - return "W_Class(%s)" % self.name - - def lookup(self, selector): - if selector in self.methoddict: - return self.methoddict[selector] - elif self.w_superclass != None: - return self.w_superclass.lookup(selector) - else: - return None - - def installmethod(self, selector, method): - self.methoddict[selector] = method - method.w_compiledin = self - -class W_MetaClass(W_Class): - def ismetaclass(self): - return True + self.primitive is not None) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Wed Oct 24 22:24:08 2007 @@ -1,5 +1,5 @@ import operator -import pypy.lang.smalltalk.model as model +from pypy.lang.smalltalk import model, mirror import pypy.lang.smalltalk.classtable as ct import pypy.lang.smalltalk.fakeimage as fimg from pypy.rlib import rarithmetic @@ -205,12 +205,14 @@ def common_at(stack): [w_obj, w_idx] = stack idx = unwrap_int(w_idx) + # XXX should be idx-1, probably assert_valid_index(idx, w_obj) return w_obj, idx def common_at_put(stack): [w_obj, w_idx, w_val] = stack idx = unwrap_int(w_idx) + # XXX should be idx-1, probably assert_valid_index(idx, w_obj) return w_obj, idx, w_val @@ -231,7 +233,7 @@ @stack(1) def func(stack): [w_obj] = stack - if not w_obj.w_class.isvariable(): + if not w_obj.getclassmirror().isvariable(): raise PrimitiveFailedError() return w_obj.size() @@ -246,7 +248,7 @@ @stack(3) def func(stack): w_obj, idx, w_val = common_at_put(stack) - if w_val.w_class is not ct.w_Character: + if w_val.getclassmirror() is not ct.m_Character: raise PrimitiveFailedError() w_obj.setbyte(idx, fimg.ord_w_char(w_val)) return w_val @@ -271,7 +273,8 @@ def func(stack): [w_rcvr, w_idx] = stack idx = unwrap_int(w_idx) - assert_bounds(idx, 0, w_rcvr.w_class.instvarsize) + # XXX should be idx-1, probably + assert_bounds(idx, 0, w_rcvr.getclassmirror().instance_size) return w_rcvr.fetch(idx) @primitive(OBJECT_AT_PUT) @@ -279,7 +282,8 @@ def func(stack): [w_rcvr, w_idx, w_val] = stack idx = unwrap_int(w_idx) - assert_bounds(idx, 0, w_rcvr.w_class.instvarsize) + # XXX should be idx-1, probably + assert_bounds(idx, 0, w_rcvr.getclassmirror().instance_size) w_rcvr.store(idx, w_val) return w_val @@ -287,18 +291,20 @@ @stack(1) def func(stack): [w_cls] = stack - if not isinstance(w_cls, model.W_Class) or w_cls.isvariable(): + m_cls = mirror.mirrorcache.getmirror(w_cls) + if m_cls.isvariable(): raise PrimitiveFailedError() - return w_cls.new() + return m_cls.new() @primitive(NEW_WITH_ARG) @stack(2) def func(stack): [w_cls, w_size] = stack - if not isinstance(w_cls, model.W_Class) or not w_cls.isvariable(): + m_cls = mirror.mirrorcache.getmirror(w_cls) + if not m_cls.isvariable(): raise PrimitiveFailedError() size = unwrap_int(w_size) - return w_cls.new(size) + return m_cls.new(size) @primitive(ARRAY_BECOME_ONE_WAY) def func(frame): @@ -311,12 +317,12 @@ # Might be restricted to fixed length fields? [w_rcvr, w_idx] = stack idx = unwrap_int(w_idx) - w_cls = w_rcvr.w_class + m_cls = w_rcvr.getclassmirror() if idx < 0: raise PrimitiveFailedError() - if idx < w_cls.instvarsize: + if idx < m_cls.instsize(): return w_rcvr.fetch(idx) - idx -= w_cls.instvarsize + idx -= m_cls.instsize() if idx < w_rcvr.size(): return subscript(idx, w_rcvr) raise PrimitiveFailedError() Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Wed Oct 24 22:24:08 2007 @@ -1,6 +1,7 @@ import py from pypy.lang.smalltalk import model from pypy.lang.smalltalk import fakeimage +from pypy.lang.smalltalk.mirror import mirrorcache from pypy.rlib import objectmodel def int2str(integer): @@ -298,11 +299,11 @@ def fillin_pointersobject(self, w_pointersobject): assert self.pointers is not None w_pointersobject.vars = [g_object.w_object for g_object in self.pointers] - w_pointersobject.w_class = self.g_class.w_object + w_pointersobject.m_class = mirrorcache.get_or_build(self.g_class.w_object) def fillin_wordsobject(self, w_wordsobject): w_wordsobject.words = self.chunk.data - w_wordsobject.w_class = self.g_class.w_object + w_wordsobject.m_class = mirrorcache.get_or_build(self.g_class.w_object) def fillin_bytesobject(self, w_bytesobject): bytes = [] @@ -311,7 +312,7 @@ bytes.append(chr((each >> 16) & 0xff)) bytes.append(chr((each >> 8) & 0xff)) bytes.append(chr((each >> 0) & 0xff)) - w_bytesobject.w_class = self.g_class.w_object + w_bytesobject.m_class = mirrorcache.get_or_build(self.g_class.w_object) w_bytesobject.bytes = bytes[:-(self.format & 3)] # omit odd bytes def fillin_compiledmethod(self, w_compiledmethod): @@ -336,8 +337,9 @@ l.append(int2str(each)) bytes = "".join(l) bytes = bytes[:-(self.format & 3)] + # XXX assert mirrorcache.get_or_build(self.g_class.w_object) is + # ct.m_CompiledMethod w_compiledmethod.__init__( - w_class = self.g_class.w_object, size = literalsize, bytes = bytes, argsize = numargs, Modified: pypy/dist/pypy/lang/smalltalk/test/test_classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_classtable.py Wed Oct 24 22:24:08 2007 @@ -1,30 +1,22 @@ from pypy.lang.smalltalk.classtable import classtable import pypy.lang.smalltalk.classtable as ct -def inherits_from(w_cls, w_superclass): - w_p = w_cls - while w_p and w_p != w_superclass: - w_p = w_p.w_superclass - return w_p == w_superclass +def inherits_from(m_cls, m_superclass): + m_p = m_cls + while m_p and m_p != m_superclass: + m_p = m_p.m_superclass + return m_p == m_superclass def test_every_class_is_an_instance_of_a_metaclass(): - for (nm, w_cls) in classtable.items(): - assert w_cls.ismetaclass() or w_cls.w_class.ismetaclass() # ? + for (nm, m_cls) in classtable.items(): + assert (m_cls.m_metaclass is ct.m_Metaclass or + m_cls.m_metaclass.m_metaclass is ct.m_Metaclass) def test_every_metaclass_inherits_from_class_and_behavior(): - for (nm, w_cls) in classtable.items(): - if w_cls.ismetaclass(): - assert inherits_from(w_cls, ct.w_Class) - assert inherits_from(ct.w_Class, ct.w_Behavior) - -def test_every_metaclass_is_an_instance_of_metaclass(): - for (nm, w_cls) in classtable.items(): - if w_cls.ismetaclass(): - assert w_cls.w_class is ct.w_Metaclass + for (nm, m_cls) in classtable.items(): + if m_cls.m_metaclass is ct.m_Metaclass: + assert inherits_from(m_cls, ct.m_Class) + assert inherits_from(ct.m_Class, ct.m_Behavior) def test_metaclass_of_metaclass_is_an_instance_of_metaclass(): - assert ct.w_Metaclass.w_class.w_class is ct.w_Metaclass - - - - + assert ct.m_Metaclass.m_metaclass.m_metaclass is ct.m_Metaclass Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Wed Oct 24 22:24:08 2007 @@ -1,7 +1,10 @@ import py -from pypy.lang.smalltalk import model, interpreter, primitives +from pypy.lang.smalltalk import model, interpreter, primitives, mirror +from pypy.lang.smalltalk.fakeimage import wrap_int import pypy.lang.smalltalk.classtable as ct +mockclassmirror = ct.bootstrap_classmirror + # expose the bytecode's values as global constants. # Bytecodes that have a whole range are exposed as global functions: # call them with an argument 'n' to get the bytecode number 'base + n'. @@ -24,7 +27,7 @@ def new_interpreter(bytes, receiver="receiver"): assert isinstance(bytes, str) - w_method = model.W_CompiledMethod(None, 0, bytes=bytes, + w_method = model.W_CompiledMethod(0, bytes=bytes, argsize=2, tempsize=1) w_frame = w_method.createFrame(receiver, ["foo", "bar"]) interp = interpreter.Interpreter() @@ -32,7 +35,7 @@ return interp def test_create_frame(): - w_method = model.W_CompiledMethod(None, 0, bytes="hello", + w_method = model.W_CompiledMethod(0, bytes="hello", argsize=2, tempsize=1) w_frame = w_method.createFrame("receiver", ["foo", "bar"]) assert w_frame.receiver == "receiver" @@ -73,8 +76,7 @@ def test_pushReceiverVariableBytecode(bytecode = (pushReceiverVariableBytecode(0) + pushReceiverVariableBytecode(1) + pushReceiverVariableBytecode(2))): - w_democlass = model.W_Class(None, None, 3) - w_demo = w_democlass.new() + w_demo = mockclassmirror(3).new() w_demo.store(0, "egg") w_demo.store(1, "bar") w_demo.store(2, "baz") @@ -105,8 +107,7 @@ assert interp.activeContext.stack == ["a", "b", "c"] def test_pushLiteralVariableBytecode(bytecode=pushLiteralVariableBytecode(0)): - w_associationclass = model.W_Class(None, None, 2) - w_association = w_associationclass.new() + w_association = mockclassmirror(2).new() w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(bytecode) @@ -116,9 +117,9 @@ def test_storeAndPopReceiverVariableBytecode(bytecode=storeAndPopReceiverVariableBytecode, popped=True): - w_class = model.W_Class(None, None, 8) + m_class = mockclassmirror(8) for index in range(8): - w_object = w_class.new() + w_object = m_class.new() interp = new_interpreter(pushConstantTrueBytecode + bytecode(index)) interp.activeContext.receiver = w_object interp.step() @@ -194,18 +195,18 @@ interp.step() assert interp.activeContext.stack == [interp.ZERO, interp.ZERO] -# w_class - the class from which the method is going to be called +# m_class - the class from which the method is going to be called # (and on which it is going to be installed) # w_object - the actual object we will be sending the method to # bytecodes - the bytecode to be executed -def sendBytecodesTest(w_class, w_object, bytecodes): +def sendBytecodesTest(m_class, w_object, bytecodes): for bytecode, result in [ (returnReceiver, w_object), (returnTrue, interpreter.Interpreter.TRUE), (returnFalse, interpreter.Interpreter.FALSE), (returnNil, interpreter.Interpreter.NIL), (returnTopFromMethod, interpreter.Interpreter.ONE) ]: - w_class.installmethod("foo", - model.W_CompiledMethod(None, 0, pushConstantOneBytecode + bytecode)) + m_class.installmethod("foo", + model.W_CompiledMethod(0, pushConstantOneBytecode + bytecode)) interp = new_interpreter(bytecodes) interp.activeContext.method.literals = ["foo"] interp.activeContext.push(w_object) @@ -214,7 +215,7 @@ assert interp.activeContext.sender == callerContext assert interp.activeContext.stack == [] assert interp.activeContext.receiver == w_object - assert interp.activeContext.method == w_class.methoddict["foo"] + assert interp.activeContext.method == m_class.methoddict["foo"] assert callerContext.stack == [] interp.step() interp.step() @@ -222,42 +223,42 @@ assert interp.activeContext.stack == [result] def test_sendLiteralSelectorBytecode(): - w_class = model.W_Class(None, None) - w_object = w_class.new() - sendBytecodesTest(w_class, w_object, sendLiteralSelectorBytecode(0)) + m_class = mockclassmirror(0) + w_object = m_class.new() + sendBytecodesTest(m_class, w_object, sendLiteralSelectorBytecode(0)) def test_fibWithArgument(): bytecode = ''.join(map(chr, [ 16, 119, 178, 154, 118, 164, 11, 112, 16, 118, 177, 224, 112, 16, 119, 177, 224, 176, 124 ])) - w_class = model.W_Class(None, None) - w_class.installmethod("fib:", model.W_CompiledMethod(None, 1, bytecode, 1)) - w_class.methoddict["fib:"].literals[0] = "fib:" - w_object = w_class.new() + m_class = mockclassmirror(0) + method = model.W_CompiledMethod(1, bytecode, 1) + method.literals[0] = "fib:" + m_class.installmethod("fib:", method) + w_object = m_class.new() interp = new_interpreter(sendLiteralSelectorBytecode(16) + returnTopFromMethod) interp.activeContext.method.literals = ["fib:"] interp.activeContext.push(w_object) - interp.activeContext.push(model.W_SmallInteger(None, 8)) + interp.activeContext.push(wrap_int(8)) result = interp.interpret() assert primitives.unwrap_int(result) == 34 def test_send_to_primitive(): - w_smallintclass = model.W_Class(None, None) - prim_meth = model.W_CompiledMethod(None, 0, "", argsize=1, + m_smallintclass = ct.m_SmallInteger + prim_meth = model.W_CompiledMethod(0, "", argsize=1, primitive=primitives.SUBTRACT) - w_smallintclass.installmethod("sub", prim_meth) - w_50 = model.W_SmallInteger(w_smallintclass, 50) - w_50.w_class = w_smallintclass - w_8 = model.W_SmallInteger(w_smallintclass, 8) - w_8.w_class = w_smallintclass - interp = new_interpreter(sendLiteralSelectorBytecode(1 + 16)) - interp.activeContext.method.literals = ["foo", "sub"] - interp.activeContext.push(w_50) - interp.activeContext.push(w_8) - callerContext = interp.activeContext - interp.step() - assert interp.activeContext is callerContext - assert len(interp.activeContext.stack) == 1 - w_result = interp.activeContext.pop() - assert primitives.unwrap_int(w_result) == 42 + m_smallintclass.installmethod("sub", prim_meth) + try: + interp = new_interpreter(sendLiteralSelectorBytecode(1 + 16)) + interp.activeContext.method.literals = ["foo", "sub"] + interp.activeContext.push(wrap_int(50)) + interp.activeContext.push(wrap_int(8)) + callerContext = interp.activeContext + interp.step() + assert interp.activeContext is callerContext + assert len(interp.activeContext.stack) == 1 + w_result = interp.activeContext.pop() + assert primitives.unwrap_int(w_result) == 42 + finally: + del m_smallintclass.methoddict['sub'] # clean up after you def test_longJumpIfTrue(): interp = new_interpreter(longJumpIfTrue(0) + chr(15) + longJumpIfTrue(0) + chr(15)) @@ -330,8 +331,7 @@ test_pushLiteralVariableBytecode(extendedPushBytecode + chr((3<<6) + 0)) def storeAssociation(bytecode): - w_associationclass = model.W_Class(None, None, 2) - w_association = w_associationclass.new() + w_association = mockclassmirror(2).new() w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(pushConstantOneBytecode + bytecode) @@ -353,13 +353,13 @@ def test_callPrimitiveAndPush_fallback(): interp = new_interpreter(bytecodePrimAdd) - w_class = model.W_Class(None, None) - w_class.installmethod("+", model.W_CompiledMethod(None, 1, "", 1)) - w_object = w_class.new() + m_class = mockclassmirror(0) + m_class.installmethod("+", model.W_CompiledMethod(1, "", 1)) + w_object = m_class.new() interp.activeContext.push(w_object) interp.activeContext.push(interp.ONE) interp.step() - assert interp.activeContext.method == w_class.methoddict["+"] + assert interp.activeContext.method == m_class.methoddict["+"] assert interp.activeContext.receiver is w_object assert interp.activeContext.gettemp(0) == interp.ONE assert interp.activeContext.stack == [] @@ -380,49 +380,49 @@ interp.FALSE, interp.TRUE] def test_singleExtendedSendBytecode(): - w_class = model.W_Class(None, None) - w_object = w_class.new() - sendBytecodesTest(w_class, w_object, singleExtendedSendBytecode + chr((0<<5)+0)) + m_class = mockclassmirror(0) + w_object = m_class.new() + sendBytecodesTest(m_class, w_object, singleExtendedSendBytecode + chr((0<<5)+0)) def test_singleExtendedSuperBytecode(bytecode=singleExtendedSuperBytecode + chr((0<<5) + 0)): - w_supersuper = model.W_Class(None, None) - w_super = model.W_Class(None, w_supersuper) - w_class = model.W_Class(None, w_super) - w_object = w_class.new() - # first call method installed in w_class + m_supersuper = mockclassmirror(0) + m_super = mockclassmirror(0, m_superclass=m_supersuper) + m_class = mockclassmirror(0, m_superclass=m_super) + w_object = m_class.new() + # first call method installed in m_class bytecodes = singleExtendedSendBytecode + chr(0) # which does a call to its super - w_class.installmethod("foo", - model.W_CompiledMethod(None, 0, bytecode)) + m_class.installmethod("foo", + model.W_CompiledMethod(0, bytecode)) # and that one again to its super - w_super.installmethod("foo", - model.W_CompiledMethod(None, 0, bytecode)) - w_supersuper.installmethod("foo", - model.W_CompiledMethod(None, 0, "")) - w_class.methoddict["foo"].literals = ["foo"] - w_super.methoddict["foo"].literals = ["foo"] + m_super.installmethod("foo", + model.W_CompiledMethod(0, bytecode)) + m_supersuper.installmethod("foo", + model.W_CompiledMethod(0, "")) + m_class.methoddict["foo"].literals = ["foo"] + m_super.methoddict["foo"].literals = ["foo"] interp = new_interpreter(bytecodes) interp.activeContext.method.literals = ["foo"] interp.activeContext.push(w_object) - for w_specificclass in [w_class, w_super, w_supersuper]: + for m_specificclass in [m_class, m_super, m_supersuper]: callerContext = interp.activeContext interp.step() assert interp.activeContext.sender == callerContext assert interp.activeContext.stack == [] assert interp.activeContext.receiver == w_object - assert interp.activeContext.method == w_specificclass.methoddict["foo"] + assert interp.activeContext.method == m_specificclass.methoddict["foo"] assert callerContext.stack == [] def test_secondExtendedSendBytecode(): - w_class = model.W_Class(None, None) - w_object = w_class.new() - sendBytecodesTest(w_class, w_object, secondExtendedSendBytecode + chr(0)) + m_class = mockclassmirror(0) + w_object = m_class.new() + sendBytecodesTest(m_class, w_object, secondExtendedSendBytecode + chr(0)) def test_doubleExtendedDoAnythinBytecode(): - w_class = model.W_Class(None, None) - w_object = w_class.new() + m_class = mockclassmirror(0) + w_object = m_class.new() - sendBytecodesTest(w_class, w_object, doubleExtendedDoAnythingBytecode + chr((0<<5) + 0) + chr(0)) + sendBytecodesTest(m_class, w_object, doubleExtendedDoAnythingBytecode + chr((0<<5) + 0) + chr(0)) test_singleExtendedSuperBytecode(doubleExtendedDoAnythingBytecode + (chr((1<<5) + 0) + chr(0))) Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Wed Oct 24 22:24:08 2007 @@ -1,28 +1,31 @@ import py -from pypy.lang.smalltalk import model +from pypy.lang.smalltalk import model, mirror +import pypy.lang.smalltalk.classtable as ct + +mockclassmirror = ct.bootstrap_classmirror def test_new(): - w_mycls = model.W_Class(None, None) - w_myinstance = w_mycls.new() - assert isinstance(w_myinstance, model.W_Object) - assert w_myinstance.w_class is w_mycls + m_mycls = mockclassmirror(0) + w_myinstance = m_mycls.new() + assert isinstance(w_myinstance, model.W_PointersObject) + assert w_myinstance.getclassmirror() is m_mycls def test_new_namedvars(): - w_mycls = model.W_Class(None, None, 3) - w_myinstance = w_mycls.new() + m_mycls = mockclassmirror(3) + w_myinstance = m_mycls.new() assert isinstance(w_myinstance, model.W_PointersObject) - assert w_myinstance.w_class is w_mycls + assert w_myinstance.getclassmirror() is m_mycls assert w_myinstance.fetch(0) is None py.test.raises(IndexError, lambda: w_myinstance.fetch(3)) w_myinstance.store(1, w_myinstance) assert w_myinstance.fetch(1) is w_myinstance def test_bytes_object(): - w_class = model.W_Class(None, None, format=model.BYTES) - w_bytes = w_class.new(20) - assert w_bytes.w_class is w_class + m_class = mockclassmirror(0, format=mirror.BYTES) + w_bytes = m_class.new(20) + assert w_bytes.getclassmirror() is m_class assert w_bytes.size() == 20 - assert w_bytes.instvarsize() == 0 + assert m_class.instsize() == 0 assert w_bytes.getbyte(3) == 00 w_bytes.setbyte(3, 0xAA) assert w_bytes.getbyte(3) == 0xAA @@ -30,11 +33,11 @@ py.test.raises(IndexError, lambda: w_bytes.getbyte(20)) def test_word_object(): - w_class = model.W_Class(None, None, format=model.WORDS) - w_bytes = w_class.new(20) - assert w_bytes.w_class is w_class + m_class = mockclassmirror(0, format=mirror.WORDS) + w_bytes = m_class.new(20) + assert w_bytes.getclassmirror() is m_class assert w_bytes.size() == 20 - assert w_bytes.instvarsize() == 0 + assert m_class.instsize() == 0 assert w_bytes.getword(3) == 0 w_bytes.setword(3, 42) assert w_bytes.getword(3) == 42 @@ -42,21 +45,20 @@ py.test.raises(IndexError, lambda: w_bytes.getword(20)) def test_method_lookup(): - w_class = model.W_Class(None, None) - w_class.methoddict["foo"] = 1 - w_class.methoddict["bar"] = 2 - w_subclass = model.W_Class(None, w_class) - w_subclass.methoddict["foo"] = 3 - assert w_class.lookup("foo") == 1 - assert w_class.lookup("bar") == 2 - assert w_class.lookup("zork") == None - assert w_subclass.lookup("foo") == 3 - assert w_subclass.lookup("bar") == 2 - assert w_subclass.lookup("zork") == None - -def test_w_compiledin(): - w_super = model.W_Class(None,None) - w_class = model.W_Class(None,w_super) - w_super.installmethod("foo", - model.W_CompiledMethod(None, 0)) - assert w_class.lookup("foo").w_compiledin == w_super + m_class = mockclassmirror(0) + m_class.methoddict["foo"] = 1 + m_class.methoddict["bar"] = 2 + m_subclass = mockclassmirror(0, m_superclass=m_class) + m_subclass.methoddict["foo"] = 3 + assert m_class.lookup("foo") == 1 + assert m_class.lookup("bar") == 2 + assert m_class.lookup("zork") == None + assert m_subclass.lookup("foo") == 3 + assert m_subclass.lookup("bar") == 2 + assert m_subclass.lookup("zork") == None + +def test_m_compiledin(): + m_super = mockclassmirror(0) + m_class = mockclassmirror(0, m_superclass=m_super) + m_super.installmethod("foo", model.W_CompiledMethod(0, "")) + assert m_class.lookup("foo").m_compiledin is m_super Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Wed Oct 24 22:24:08 2007 @@ -1,12 +1,14 @@ import py from pypy.lang.smalltalk.primitives import prim_table, PrimitiveFailedError import pypy.lang.smalltalk.primitives as p -import pypy.lang.smalltalk.model as model +from pypy.lang.smalltalk import model, mirror import pypy.lang.smalltalk.interpreter as interp import pypy.lang.smalltalk.classtable as ct import pypy.lang.smalltalk.fakeimage as fimg -class MockFrame(interp.W_ContextFrame): +mockclassmirror = ct.bootstrap_classmirror + +class MockFrame(interp.W_MethodContext): def __init__(self, stack): self.stack = stack @@ -16,6 +18,7 @@ if isinstance(x, model.W_Object): return x if isinstance(x, str) and len(x) == 1: return fimg.wrap_char(x) if isinstance(x, str): return fimg.wrap_string(x) + if isinstance(x, mirror.ClassMirror): return x.w_self raise NotImplementedError def mock(stack): @@ -137,21 +140,21 @@ assert prim(p.FLOAT_ADD, [3,4.5]).value == 7.5 def test_at(): - w_obj = model.W_Class(None, None, 0, format=model.VAR_POINTERS).new(1) + w_obj = mockclassmirror(0, varsized=True).new(1) w_obj.store(0, "foo") assert prim(p.AT, [w_obj, 0]) == "foo" def test_invalid_at(): - w_obj = model.W_Class(None, None, 0, format=model.POINTERS).new() + w_obj = mockclassmirror(0).new() prim_fails(p.AT, [w_obj, 0]) def test_at_put(): - w_obj = model.W_Class(None, None, 0, format=model.VAR_POINTERS).new(1) + w_obj = mockclassmirror(0, varsized=1).new(1) assert prim(p.AT_PUT, [w_obj, 0, 22]).value == 22 assert prim(p.AT, [w_obj, 0]).value == 22 def test_invalid_at_put(): - w_obj = model.W_Class(None, None, 0, format=model.POINTERS).new() + w_obj = mockclassmirror(0).new() prim_fails(p.AT_PUT, [w_obj, 0, 22]) def test_string_at(): @@ -172,12 +175,12 @@ prim_fails(p.OBJECT_AT, ["q", fimg.CHARACTER_VALUE_INDEX+1]) def test_object_at_put(): - w_obj = model.W_Class(None, None, 1, format=model.POINTERS).new() + w_obj = mockclassmirror(1).new() assert prim(p.OBJECT_AT_PUT, [w_obj, 0, "q"]) is wrap("q") assert prim(p.OBJECT_AT, [w_obj, 0]) is wrap("q") def test_invalid_object_at_put(): - w_obj = model.W_Class(None, None, 1, format=model.POINTERS).new() + w_obj = mockclassmirror(1).new() prim_fails(p.OBJECT_AT, [w_obj, 1, 1]) def test_string_at_put(): @@ -188,19 +191,19 @@ assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i]) def test_new(): - w_res = prim(p.NEW, [ct.w_Object]) - assert w_res.w_class == ct.w_Object + w_res = prim(p.NEW, [ct.m_Object]) + assert w_res.getclassmirror() == ct.m_Object def test_invalid_new(): - prim_fails(p.NEW, [ct.w_ByteString]) + prim_fails(p.NEW, [ct.m_ByteString]) def test_new_with_arg(): - w_res = prim(p.NEW_WITH_ARG, [ct.w_ByteString, 20]) - assert w_res.w_class == ct.w_ByteString + w_res = prim(p.NEW_WITH_ARG, [ct.m_ByteString, 20]) + assert w_res.getclassmirror() == ct.m_ByteString assert w_res.size() == 20 def test_invalid_new_with_arg(): - prim_fails(p.NEW_WITH_ARG, [ct.w_Object, 20]) + prim_fails(p.NEW_WITH_ARG, [ct.m_Object, 20]) def test_inst_var_at(): # I am not entirely sure if this is what this primitive is @@ -211,7 +214,7 @@ assert w_v.value == ord("b") def test_as_oop(): - w_obj = model.W_Class(None, None, 0, format=model.POINTERS).new() + w_obj = mockclassmirror(0).new() w_obj.w_hash = wrap(22) assert prim(p.AS_OOP, [w_obj]).value == 22 From tverwaes at codespeak.net Wed Oct 24 22:46:21 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 22:46:21 +0200 (CEST) Subject: [pypy-svn] r47864 - in pypy/dist/pypy/lang/smalltalk: . test tool Message-ID: <20071024204621.3CC6880EB@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 22:46:20 2007 New Revision: 47864 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: a starting point for method lookup in the analysis tool Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Wed Oct 24 22:46:20 2007 @@ -514,3 +514,17 @@ return result BYTECODE_TABLE = initialize_bytecode_table() + +def initialize_readable_bytecode_table(): + result = [None] * 256 + for entry in BYTECODE_RANGES: + if len(entry) == 2: + positions = [entry[0]] + else: + positions = range(entry[0], entry[1]+1) + for pos in positions: + result[pos] = "%s(%d)" % (entry[-1],pos) + assert None not in result + return result + +READABLE_BYTECODE_TABLE = initialize_readable_bytecode_table() Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Wed Oct 24 22:46:20 2007 @@ -27,6 +27,9 @@ def invariant(self): return isinstance(self.value, int) + def __str__(self): + return "SmallInt(%d)" % self.value + class W_Float(W_Object): def __init__(self, value): self.value = value @@ -41,6 +44,8 @@ def invariant(self): return self.value is not None # XXX but later: #return isinstance(self.value, float) + def __str__(self): + return "Float(%f)" % self.value class W_ObjectWithStoredClass(W_Object): """ The base class of objects that store 'm_class' explicitly. """ @@ -92,6 +97,9 @@ def size(self): return len(self.bytes) + def __str__(self): + return repr("".join(self.bytes)) + def invariant(self): if not W_ObjectWithStoredClass.invariant(self): return False @@ -161,6 +169,12 @@ assert len(arguments) == self.argsize return W_MethodContext(self, receiver, arguments, sender) + def __str__(self): + from pypy.lang.smalltalk.interpreter import BYTECODE_TABLE + return ("\n\nBytecode:\n---------------------\n" + + "\n".join([BYTECODE_TABLE[ord(i)].__name__ + " " + str(ord(i)) for i in self.bytes]) + + "\n---------------------\n") + def invariant(self): return (W_Object.invariant(self) and hasattr(self, 'literals') and Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Wed Oct 24 22:46:20 2007 @@ -70,4 +70,3 @@ assert isinstance(w_float_class_name, sqm.W_BytesObject) assert w_float_class_name.bytes == list("Float") - Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Wed Oct 24 22:46:20 2007 @@ -21,48 +21,34 @@ image = create_squeakimage() for each in image.objects: if isinstance(each,sqm.W_BytesObject): - print repr(''.join(each.bytes)) + print each.bytes def printReadableBytecode(bytecode): print "\n\nBytecode:\n---------------------" print "\n".join([sqi.BYTECODE_TABLE[ord(i)].__name__ for i in bytecode]) print "---------------------\n" +def getMethodFromClass(w_class,methodname): + w_methoddict = w_class.fetch(1) + for var in w_methoddict.vars: + if isinstance(var,sqm.W_BytesObject): + if str(var) == repr(methodname): + return w_methoddict.vars[1].vars[w_methoddict.vars.index(var)-2] def testCompiledMethods(): image = create_squeakimage() amethod = None - skip = 0 - for each in image.objects: - if isinstance(each,sqm.W_CompiledMethod): - if (each.argsize == 0 and amethod == None and - each.tempsize == 0 and - each.primitive == 0): - - if len(each.bytes) == 0: - pass - else: - if skip >= SKIPMETHODS: - amethod = each - else: - skip += 1 - #print "%d %d %d" % (each.argsize, each.tempsize, each.primitive) - - # receiver, arguments - interp = sqi.Interpreter() - - anObject = sqm.W_PointersObject(sqm.W_Class(None,None,100),0) - for i in range(0,99): - anObject.store(i, interp.ONE) + w_float_class = image.special(sq.FLOAT_CLASS) + interp = sqi.Interpreter() + anObject = sqm.W_Float(1.5) + amethod = getMethodFromClass(w_float_class,"abs") + # receiver, arguments w_frame = amethod.createFrame(anObject, []) interp.activeContext = w_frame - #w_frame.push(interp.TRUE) - #w_frame.push(interp.ONE) - #w_frame.push(interp.TWO) - printReadableBytecode(amethod.bytes) + print amethod while True: try: @@ -75,8 +61,8 @@ SKIPMETHODS=42 #X def test_do(): - #testCompiledMethods() - printStringsInImage() + testCompiledMethods() + #printStringsInImage() if __name__ == '__main__': test_do() From cfbolz at codespeak.net Wed Oct 24 23:05:25 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 24 Oct 2007 23:05:25 +0200 (CEST) Subject: [pypy-svn] r47865 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024210525.D7FA480EB@code0.codespeak.net> Author: cfbolz Date: Wed Oct 24 23:05:25 2007 New Revision: 47865 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_model.py Log: a small integer is its own hash Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Wed Oct 24 23:05:25 2007 @@ -22,7 +22,7 @@ return m_SmallInteger def gethash(self): - return self.value # XXX check this + return self.value def invariant(self): return isinstance(self.value, int) Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Wed Oct 24 23:05:25 2007 @@ -62,3 +62,7 @@ m_class = mockclassmirror(0, m_superclass=m_super) m_super.installmethod("foo", model.W_CompiledMethod(0, "")) assert m_class.lookup("foo").m_compiledin is m_super + +def test_hashes(): + w_five = W_SmallInteger(5) + assert w_five.gethash() == 5 From cfbolz at codespeak.net Wed Oct 24 23:08:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 24 Oct 2007 23:08:28 +0200 (CEST) Subject: [pypy-svn] r47866 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071024210828.0263180EB@code0.codespeak.net> Author: cfbolz Date: Wed Oct 24 23:08:28 2007 New Revision: 47866 Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py Log: garumpf. I nicely wrote a test but didn't run it. I should go to bed. Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Wed Oct 24 23:08:28 2007 @@ -64,5 +64,5 @@ assert m_class.lookup("foo").m_compiledin is m_super def test_hashes(): - w_five = W_SmallInteger(5) + w_five = model.W_SmallInteger(5) assert w_five.gethash() == 5 From cfbolz at codespeak.net Wed Oct 24 23:11:24 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 24 Oct 2007 23:11:24 +0200 (CEST) Subject: [pypy-svn] r47867 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071024211124.B434480EB@code0.codespeak.net> Author: cfbolz Date: Wed Oct 24 23:11:24 2007 New Revision: 47867 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: fill in the identity hash field for object loaded from the image Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Wed Oct 24 23:11:24 2007 @@ -300,10 +300,12 @@ assert self.pointers is not None w_pointersobject.vars = [g_object.w_object for g_object in self.pointers] w_pointersobject.m_class = mirrorcache.get_or_build(self.g_class.w_object) + w_pointersobject.hash = self.chunk.hash12 def fillin_wordsobject(self, w_wordsobject): w_wordsobject.words = self.chunk.data w_wordsobject.m_class = mirrorcache.get_or_build(self.g_class.w_object) + w_wordsobject.hash = self.chunk.hash12 # XXX check this def fillin_bytesobject(self, w_bytesobject): bytes = [] @@ -314,6 +316,7 @@ bytes.append(chr((each >> 0) & 0xff)) w_bytesobject.m_class = mirrorcache.get_or_build(self.g_class.w_object) w_bytesobject.bytes = bytes[:-(self.format & 3)] # omit odd bytes + w_bytesobject.hash = self.chunk.hash12 # XXX check this def fillin_compiledmethod(self, w_compiledmethod): header = self.chunk.data[0] From fijal at codespeak.net Wed Oct 24 23:28:50 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 24 Oct 2007 23:28:50 +0200 (CEST) Subject: [pypy-svn] r47868 - in pypy/dist/pypy/module/zipimport: . test Message-ID: <20071024212850.519E580EB@code0.codespeak.net> Author: fijal Date: Wed Oct 24 23:28:47 2007 New Revision: 47868 Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py pypy/dist/pypy/module/zipimport/test/test_zipimport.py Log: A present for twisted folks - implement obscure and undocumented behavior of cpython. Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/interp_zipimport.py Wed Oct 24 23:28:47 2007 @@ -170,12 +170,24 @@ return zip_importer_cache[name] except KeyError: pass - try: - s = os.stat(name) - except OSError: - return space.w_None - if stat.S_ISDIR(s.st_mode): - return space.w_None + ok = False + parts = name.split(os.path.sep) + filename = "" # make annotator happy + for i in range(1, len(parts) + 1): + filename = os.path.sep.join(parts[:i]) + if not filename: + filename = os.path.sep + try: + s = os.stat(filename) + except OSError: + raise OperationError(space.w_ImportError, space.wrap( + "Cannot find name %s" % (filename,))) + if not stat.S_ISDIR(s.st_mode): + ok = True + break + if not ok: + raise OperationError(space.w_ImportError, space.wrap( + "Did not find %s to be a valid zippath" % (name,))) w_import = space.builtin.get('__import__') w_zipfile = space.call(w_import, space.newlist([ space.wrap('zipfile'), @@ -184,12 +196,12 @@ space.newlist([])])) w_ZipFile = space.getattr(w_zipfile, space.wrap('ZipFile')) try: - w_dir = space.call(w_ZipFile, space.newlist([space.wrap(name)])) + w_dir = space.call(w_ZipFile, space.newlist([space.wrap(filename)])) except OperationError: # we catch everything as this function - # should not raise - return space.w_None + raise OperationError(space.w_ImportError, space.wrap( + "%s seems not to be a zipfile" % (filename,))) result = space.wrap(W_ZipImporter(space, name, w_dir, w_zipfile)) - zip_importer_cache[name] = result + zip_importer_cache[filename] = result return result descr_new_zipimporter.unwrap_spec = [ObjSpace, W_Root, str] Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/test/test_zipimport.py Wed Oct 24 23:28:47 2007 @@ -88,6 +88,21 @@ sys.path.pop(0) """) + def test_good_bad_arguments(self): + from zipimport import zipimporter + import os + self.writefile(self, "x.py", "y") + zipimporter(self.zipfile) # should work + raises(ImportError, "zipimporter(os.path.dirname(self.zipfile))") + raises(ImportError, 'zipimporter("fsafdetrssffdsagadfsafdssadasa")') + name = os.path.join(os.path.dirname(self.zipfile), "x.zip") + f = open(name, "w") + f.write("zzz") + f.close() + raises(ImportError, 'zipimporter(name)') + # this should work as well :-/ + zipimporter(os.path.join(self.zipfile, 'x')) + def test_py(self): import sys, os self.writefile(self, "uuu.py", "def f(x): return x") From cfbolz at codespeak.net Wed Oct 24 23:46:43 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 24 Oct 2007 23:46:43 +0200 (CEST) Subject: [pypy-svn] r47869 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024214643.1EDF58108@code0.codespeak.net> Author: cfbolz Date: Wed Oct 24 23:46:42 2007 New Revision: 47869 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_model.py Log: (arigo, cfbolz, akuhn advising) be incredibly advanced: an object's identity hash is computed by using our extremely good mersenne twister random number generator. Also introduce a very smalltalkish class hierarchy. Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Wed Oct 24 23:46:42 2007 @@ -1,3 +1,7 @@ +import sys +from pypy.rlib import rrandom +from pypy.rlib.rarithmetic import intmask + class W_Object(object): @@ -47,27 +51,41 @@ def __str__(self): return "Float(%f)" % self.value -class W_ObjectWithStoredClass(W_Object): +class W_AbstractObjectWithIdentityHash(W_Object): + #XXX maybe this is too extreme, but it's very random + hash_generator = rrandom.Random() + UNASSIGNED_HASH = sys.maxint + + hash = UNASSIGNED_HASH # default value + + def gethash(self): + if self.hash == self.UNASSIGNED_HASH: + self.hash = hash = intmask(self.hash_generator.genrand32()) // 2 + return hash + return self.hash + + def invariant(self): + return isinstance(self.hash, int) + +class W_AbstractObjectWithClassReference(W_AbstractObjectWithIdentityHash): """ The base class of objects that store 'm_class' explicitly. """ + def __init__(self, m_class): self.m_class = m_class - self.hash = 42 # XXX def getclassmirror(self): return self.m_class - def gethash(self): - return self.hash - def invariant(self): from pypy.lang.smalltalk.mirror import ClassMirror - return (isinstance(self.m_class, ClassMirror) and - 1)#isinstance(self.hash, int)) XXX + return (W_AbstractObjectWithIdentityHash.invariant(self) and + isinstance(self.m_class, ClassMirror)) + -class W_PointersObject(W_ObjectWithStoredClass): +class W_PointersObject(W_AbstractObjectWithClassReference): """ The normal object """ def __init__(self, m_class, size): - W_ObjectWithStoredClass.__init__(self, m_class) + W_AbstractObjectWithClassReference.__init__(self, m_class) self.vars = [None] * size def fetch(self, index): @@ -80,12 +98,12 @@ return len(self.vars) def invariant(self): - return (W_ObjectWithStoredClass.invariant(self) and + return (W_AbstractObjectWithClassReference.invariant(self) and isinstance(self.vars, list)) -class W_BytesObject(W_ObjectWithStoredClass): +class W_BytesObject(W_AbstractObjectWithClassReference): def __init__(self, m_class, size): - W_ObjectWithStoredClass.__init__(self, m_class) + W_AbstractObjectWithClassReference.__init__(self, m_class) self.bytes = ['\x00'] * size def getbyte(self, n): @@ -101,16 +119,16 @@ return repr("".join(self.bytes)) def invariant(self): - if not W_ObjectWithStoredClass.invariant(self): + if not W_AbstractObjectWithClassReference.invariant(self): return False for c in self.bytes: if not isinstance(c, str) or len(c) != 1: return False return True -class W_WordsObject(W_ObjectWithStoredClass): +class W_WordsObject(W_AbstractObjectWithClassReference): def __init__(self, m_class, size): - W_ObjectWithStoredClass.__init__(self, m_class) + W_AbstractObjectWithClassReference.__init__(self, m_class) self.words = [0] * size def getword(self, n): @@ -123,10 +141,10 @@ return len(self.words) def invariant(self): - return (W_ObjectWithStoredClass.invariant(self) and + return (W_AbstractObjectWithClassReference.invariant(self) and isinstance(self.words, list)) -class W_CompiledMethod(W_Object): +class W_CompiledMethod(W_AbstractObjectWithIdentityHash): """My instances are methods suitable for interpretation by the virtual machine. This is the only class in the system whose instances intermix both indexable pointer fields and indexable integer fields. The current format of a CompiledMethod is as follows: Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Wed Oct 24 23:46:42 2007 @@ -66,3 +66,10 @@ def test_hashes(): w_five = model.W_SmallInteger(5) assert w_five.gethash() == 5 + m_class = mockclassmirror(0) + w_inst = m_class.new() + assert w_inst.hash == w_inst.UNASSIGNED_HASH + h1 = w_inst.gethash() + h2 = w_inst.gethash() + assert h1 == h2 + assert h1 == w_inst.hash From tverwaes at codespeak.net Wed Oct 24 23:50:48 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 23:50:48 +0200 (CEST) Subject: [pypy-svn] r47870 - in pypy/dist/pypy/lang/smalltalk: . tool Message-ID: <20071024215048.52C2D8118@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 23:50:47 2007 New Revision: 47870 Modified: pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: Added methodlookup and hacky test in tool/analysis Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Wed Oct 24 23:50:47 2007 @@ -13,6 +13,8 @@ CLASS_METHODDICT_INDEX = 1 CLASS_FORMAT_INDEX = 2 +METHODDICT_VALUES_INDEX = 1 +METHODDICT_NAMES_INDEX = 2 # ----- special objects indices ------- Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Wed Oct 24 23:50:47 2007 @@ -139,11 +139,14 @@ if method.primitive: func = primitives.prim_table[method.primitive] try: + print "Going to send primitive" w_result = func(self) except primitives.PrimitiveFailedError: + print "Primitive failed" pass # ignore this error and fall back to the Smalltalk version else: # the primitive succeeded + print "Pushing primitive result on stack" self.push(w_result) return arguments = self.stack[len(self.stack)-argcount:] @@ -296,9 +299,13 @@ def callPrimitiveAndPush(self, primitive, selector, argcount, interp): try: + print "Pushing result" self.push(primitives.prim_table[primitive](self)) + print "Pushed result" except primitives.PrimitiveFailedError: + print "Falling back to smalltalk version" self._sendSelfSelector(selector, argcount, interp) + print "Fallback succeeded" def bytecodePrimAdd(self, interp): self.callPrimitiveAndPush(primitives.ADD, "+", 1, interp) @@ -514,17 +521,3 @@ return result BYTECODE_TABLE = initialize_bytecode_table() - -def initialize_readable_bytecode_table(): - result = [None] * 256 - for entry in BYTECODE_RANGES: - if len(entry) == 2: - positions = [entry[0]] - else: - positions = range(entry[0], entry[1]+1) - for pos in positions: - result[pos] = "%s(%d)" % (entry[-1],pos) - assert None not in result - return result - -READABLE_BYTECODE_TABLE = initialize_readable_bytecode_table() Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Wed Oct 24 23:50:47 2007 @@ -1,7 +1,10 @@ import sys from pypy.rlib import rrandom from pypy.rlib.rarithmetic import intmask +from pypy.lang.smalltalk import constants as sqc +class MethodNotFound(Exception): + pass class W_Object(object): @@ -101,6 +104,33 @@ return (W_AbstractObjectWithClassReference.invariant(self) and isinstance(self.vars, list)) + def compiledmethodnamed(self, methodname): + w_methoddict = self.fetch(sqc.CLASS_METHODDICT_INDEX).vars + names = w_methoddict[sqc.METHODDICT_NAMES_INDEX:] + values = w_methoddict[sqc.METHODDICT_VALUES_INDEX].vars + for var in names: + if isinstance(var, W_BytesObject): + if str(var) == repr(methodname): + return values[names.index(var)] + raise MethodNotFound + + def lookup(self, methodname): + in_class = self + while in_class != None: + try: + return in_class.compiledmethodnamed(methodname) + except MethodNotFound: + # Current hack because we don't have a ref to the real + # nil yet... XXX XXX XXX + try: + new_class = in_class.vars[sqc.CLASS_SUPERCLASS_INDEX] + if in_class == new_class: + raise Exception + else: + in_class = new_class + except: + return self.lookup("doesNotUnderstand") + class W_BytesObject(W_AbstractObjectWithClassReference): def __init__(self, m_class, size): W_AbstractObjectWithClassReference.__init__(self, m_class) Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Wed Oct 24 23:50:47 2007 @@ -332,7 +332,7 @@ splitbits(header, [1,9,8,1,6,4,1])) primitive = primitive + (highbit << 10) ##XXX todo, check this # -------------------- - literals = [self.decode_pointer(pointer) + literals = [self.decode_pointer(pointer).w_object for pointer in self.chunk.data[:literalsize+1]] # -------------------- l = [] @@ -348,6 +348,8 @@ argsize = numargs, tempsize = tempsize, primitive = primitive) + + w_compiledmethod.literals = literals Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Wed Oct 24 23:50:47 2007 @@ -1,6 +1,7 @@ import autopath import py from pypy.lang.smalltalk import squeakimage as sq +from pypy.lang.smalltalk import constants as sqc from pypy.lang.smalltalk import model as sqm from pypy.lang.smalltalk import interpreter as sqi @@ -23,11 +24,6 @@ if isinstance(each,sqm.W_BytesObject): print each.bytes -def printReadableBytecode(bytecode): - print "\n\nBytecode:\n---------------------" - print "\n".join([sqi.BYTECODE_TABLE[ord(i)].__name__ for i in bytecode]) - print "---------------------\n" - def getMethodFromClass(w_class,methodname): w_methoddict = w_class.fetch(1) for var in w_methoddict.vars: @@ -39,12 +35,12 @@ image = create_squeakimage() amethod = None - w_float_class = image.special(sq.FLOAT_CLASS) + w_smallint_class = image.special(sqc.SO_SMALLINTEGER_CLASS) interp = sqi.Interpreter() - anObject = sqm.W_Float(1.5) - amethod = getMethodFromClass(w_float_class,"abs") - # receiver, arguments + anObject = sqm.W_PointersObject(w_smallint_class, 1) + anObject.vars[0] = 3 + amethod = w_smallint_class.lookup("abs") w_frame = amethod.createFrame(anObject, []) interp.activeContext = w_frame From tverwaes at codespeak.net Wed Oct 24 23:51:39 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 23:51:39 +0200 (CEST) Subject: [pypy-svn] r47871 - pypy/dist/pypy/lang/smalltalk/tool Message-ID: <20071024215139.0A7148117@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 23:51:38 2007 New Revision: 47871 Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: removed double methodlookup from tool/analysis Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Wed Oct 24 23:51:38 2007 @@ -24,13 +24,6 @@ if isinstance(each,sqm.W_BytesObject): print each.bytes -def getMethodFromClass(w_class,methodname): - w_methoddict = w_class.fetch(1) - for var in w_methoddict.vars: - if isinstance(var,sqm.W_BytesObject): - if str(var) == repr(methodname): - return w_methoddict.vars[1].vars[w_methoddict.vars.index(var)-2] - def testCompiledMethods(): image = create_squeakimage() amethod = None @@ -53,9 +46,6 @@ except sqi.ReturnFromTopLevel, e: return e.object -# apply to Xth method -SKIPMETHODS=42 #X - def test_do(): testCompiledMethods() #printStringsInImage() From tverwaes at codespeak.net Wed Oct 24 23:55:37 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Wed, 24 Oct 2007 23:55:37 +0200 (CEST) Subject: [pypy-svn] r47872 - in pypy/dist/pypy/lang/smalltalk: . tool Message-ID: <20071024215537.1D8F98117@code0.codespeak.net> Author: tverwaes Date: Wed Oct 24 23:55:36 2007 New Revision: 47872 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: removed prints and fixed first real method interpretation Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Wed Oct 24 23:55:36 2007 @@ -139,14 +139,11 @@ if method.primitive: func = primitives.prim_table[method.primitive] try: - print "Going to send primitive" w_result = func(self) except primitives.PrimitiveFailedError: - print "Primitive failed" pass # ignore this error and fall back to the Smalltalk version else: # the primitive succeeded - print "Pushing primitive result on stack" self.push(w_result) return arguments = self.stack[len(self.stack)-argcount:] Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Wed Oct 24 23:55:36 2007 @@ -31,10 +31,11 @@ w_smallint_class = image.special(sqc.SO_SMALLINTEGER_CLASS) interp = sqi.Interpreter() - anObject = sqm.W_PointersObject(w_smallint_class, 1) - anObject.vars[0] = 3 + amethod = w_smallint_class.lookup("abs") - w_frame = amethod.createFrame(anObject, []) + # First literal of the abs method is + # a real smalltalk int + w_frame = amethod.createFrame(amethod.literals[0], []) interp.activeContext = w_frame print amethod From cfbolz at codespeak.net Wed Oct 24 23:58:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 24 Oct 2007 23:58:53 +0200 (CEST) Subject: [pypy-svn] r47873 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071024215853.3AF758117@code0.codespeak.net> Author: cfbolz Date: Wed Oct 24 23:58:52 2007 New Revision: 47873 Modified: pypy/dist/pypy/lang/smalltalk/model.py Log: those two are really nice reprs, so rename the methods Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Wed Oct 24 23:58:52 2007 @@ -34,8 +34,8 @@ def invariant(self): return isinstance(self.value, int) - def __str__(self): - return "SmallInt(%d)" % self.value + def __repr__(self): + return "W_SmallInteger(%d)" % self.value class W_Float(W_Object): def __init__(self, value): @@ -51,8 +51,8 @@ def invariant(self): return self.value is not None # XXX but later: #return isinstance(self.value, float) - def __str__(self): - return "Float(%f)" % self.value + def __repr__(self): + return "W_Float(%f)" % self.value class W_AbstractObjectWithIdentityHash(W_Object): #XXX maybe this is too extreme, but it's very random From tverwaes at codespeak.net Thu Oct 25 00:06:35 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 00:06:35 +0200 (CEST) Subject: [pypy-svn] r47875 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024220635.8057C80EB@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 00:06:34 2007 New Revision: 47875 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: Moved abs-test from tool/analysis to miniimage test Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 00:06:34 2007 @@ -182,8 +182,6 @@ return self.special_objects[index] COMPACT_CLASSES_ARRAY = 28 -FLOAT_CLASS = 9 - # ____________________________________________________________ Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 00:06:34 2007 @@ -4,6 +4,8 @@ import py from pypy.lang.smalltalk import squeakimage as sq from pypy.lang.smalltalk import model as sqm +from pypy.lang.smalltalk import constants as sqc +from pypy.lang.smalltalk import interpreter as sqi mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') @@ -14,6 +16,14 @@ def get_miniimage(): return sq.ImageReader(sq.Stream(mini_image.open())) +def create_squeakimage(): + example = get_miniimage() + example.initialize() + + image = sq.SqueakImage() + image.from_reader(example) + return image + def test_read_header(): example = get_miniimage() example.read_header() @@ -61,7 +71,7 @@ for each in image.objects: each.invariant() - w_float_class = image.special(sq.FLOAT_CLASS) + w_float_class = image.special(sqc.SO_FLOAT_CLASS) assert w_float_class.size() == 9 @@ -70,3 +80,26 @@ assert isinstance(w_float_class_name, sqm.W_BytesObject) assert w_float_class_name.bytes == list("Float") + +def test_lookup_abs_in_integer(): + image = create_squeakimage() + amethod = None + + w_smallint_class = image.special(sqc.SO_SMALLINTEGER_CLASS) + + interp = sqi.Interpreter() + + amethod = w_smallint_class.lookup("abs") + # First literal of the abs method is + # a real smalltalk int + w_frame = amethod.createFrame(amethod.literals[0], []) + interp.activeContext = w_frame + + print amethod + + while True: + try: + interp.step() + print interp.activeContext.stack + except sqi.ReturnFromTopLevel, e: + return e.object From tverwaes at codespeak.net Thu Oct 25 00:08:01 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 00:08:01 +0200 (CEST) Subject: [pypy-svn] r47876 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071024220801.4C2AA80EB@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 00:08:01 2007 New Revision: 47876 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py Log: removed some debugprints Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 00:08:01 2007 @@ -296,13 +296,9 @@ def callPrimitiveAndPush(self, primitive, selector, argcount, interp): try: - print "Pushing result" self.push(primitives.prim_table[primitive](self)) - print "Pushed result" except primitives.PrimitiveFailedError: - print "Falling back to smalltalk version" self._sendSelfSelector(selector, argcount, interp) - print "Fallback succeeded" def bytecodePrimAdd(self, interp): self.callPrimitiveAndPush(primitives.ADD, "+", 1, interp) From tverwaes at codespeak.net Thu Oct 25 00:22:10 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 00:22:10 +0200 (CEST) Subject: [pypy-svn] r47877 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071024222210.445058116@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 00:22:10 2007 New Revision: 47877 Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: works for positive smallints Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 00:22:10 2007 @@ -92,7 +92,7 @@ amethod = w_smallint_class.lookup("abs") # First literal of the abs method is # a real smalltalk int - w_frame = amethod.createFrame(amethod.literals[0], []) + w_frame = amethod.createFrame(sqm.W_SmallInteger(10), []) interp.activeContext = w_frame print amethod From tverwaes at codespeak.net Thu Oct 25 01:44:59 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 01:44:59 +0200 (CEST) Subject: [pypy-svn] r47878 - in pypy/dist/pypy/lang/smalltalk: . test tool Message-ID: <20071024234459.6CBFE8108@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 01:44:57 2007 New Revision: 47878 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: fixing literals (offset + 1, skipping methodheader) and according tests Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 01:44:57 2007 @@ -68,14 +68,14 @@ def pushLiteralConstantBytecode(self, interp): index = self.currentBytecode & 31 - self.push(self.method.literals[index]) + self.push(self.method.getliteral(index)) def pushLiteralVariableBytecode(self, interp): # this bytecode assumes that literals[index] is an Association # which is an object with two named vars, and fetches the second # named var (the value). index = self.currentBytecode & 31 - association = self.method.literals[index] + association = self.method.getliteral(index) self.push(association.fetch(1)) def storeAndPopReceiverVariableBytecode(self, interp): @@ -119,7 +119,7 @@ # send, return bytecodes def sendLiteralSelectorBytecode(self, interp): - selector = self.method.literals[self.currentBytecode & 15] + selector = self.method.getliteral(self.currentBytecode & 15) argcount = ((self.currentBytecode >> 4) & 3) - 1 self._sendSelfSelector(selector, argcount, interp) @@ -189,9 +189,9 @@ elif variableType == 1: self.push(self.gettemp(variableIndex)) elif variableType == 2: - self.push(self.method.literals[variableIndex]) + self.push(self.method.getliteral(variableIndex)) elif variableType == 3: - association = self.method.literals[variableIndex] + association = self.method.getliteral(variableIndex) self.push(association.fetch(1)) def extendedStoreBytecode(self, interp): @@ -203,7 +203,7 @@ elif variableType == 2: raise IllegalStoreError elif variableType == 3: - association = self.method.literals[variableIndex] + association = self.method.getliteral(variableIndex) association.store(1,self.top()) def extendedStoreAndPopBytecode(self, interp): @@ -212,7 +212,7 @@ def getExtendedSelectorArgcount(self): descriptor = self.getByte() - return (self.method.literals[descriptor & 31]), (descriptor >> 5) + return (self.method.getliteral(descriptor & 31)), (descriptor >> 5) def singleExtendedSendBytecode(self, interp): selector, argcount = self.getExtendedSelectorArgcount() @@ -224,28 +224,28 @@ opType = second >> 5 if opType == 0: # selfsend - self._sendSelfSelector(self.method.literals[third], + self._sendSelfSelector(self.method.getliteral(third), second & 31, interp) elif opType == 1: # supersend - self._sendSuperSelector(self.method.literals[third], + self._sendSuperSelector(self.method.getliteral(third), second & 31, interp) elif opType == 2: # pushReceiver self.push(self.receiver.fetch(third)) elif opType == 3: # pushLiteralConstant - self.push(self.method.literals[third]) + self.push(self.method.getliteral(third)) elif opType == 4: # pushLiteralVariable - association = self.method.literals[third] + association = self.method.getliteral(third) self.push(association.fetch(1)) elif opType == 5: self.receiver.store(third, self.top()) elif opType == 6: self.receiver.store(third, self.pop()) elif opType == 7: - association = self.method.literals[third] + association = self.method.getliteral(third) association.store(1,self.top()) def singleExtendedSuperBytecode(self, interp): @@ -254,7 +254,7 @@ def secondExtendedSendBytecode(self, interp): descriptor = self.getByte() - selector = self.method.literals[descriptor & 63] + selector = self.method.getliteral(descriptor & 63) argcount = descriptor >> 6 self._sendSelfSelector(selector, argcount, interp) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 01:44:57 2007 @@ -196,9 +196,10 @@ The trailer has two variant formats. In the first variant, the last byte is at least 252 and the last four bytes represent a source pointer into one of the sources files (see #sourcePointer). In the second variant, the last byte is less than 252, and the last several bytes are a compressed version of the names of the method's temporary variables. The number of bytes used for this purpose is the value of the last byte in the method. """ - def __init__(self, size, bytes, argsize=0, + def __init__(self, literalsize, bytes, argsize=0, tempsize=0, primitive=0, m_compiledin=None): - self.literals = [None] * size + # self.literals = [None] * size + self.literalsize = literalsize self.m_compiledin = m_compiledin self.bytes = bytes self.argsize = argsize @@ -212,6 +213,9 @@ def gethash(self): return 43 # XXX + def getliteral(self, index): + return self.literals[index + 1] # header of compiledmethod at index 0 + def createFrame(self, receiver, arguments, sender = None): from pypy.lang.smalltalk.interpreter import W_MethodContext assert len(arguments) == self.argsize Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 01:44:57 2007 @@ -341,7 +341,7 @@ # XXX assert mirrorcache.get_or_build(self.g_class.w_object) is # ct.m_CompiledMethod w_compiledmethod.__init__( - size = literalsize, + literalsize = literalsize, bytes = bytes, argsize = numargs, tempsize = tempsize, Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Thu Oct 25 01:44:57 2007 @@ -25,6 +25,9 @@ setup() +def fakeliterals(*literals): + return ["methodheader"] + list(literals) + def new_interpreter(bytes, receiver="receiver"): assert isinstance(bytes, str) w_method = model.W_CompiledMethod(0, bytes=bytes, @@ -100,7 +103,7 @@ pushLiteralConstantBytecode(1) + pushLiteralConstantBytecode(2)): interp = new_interpreter(bytecode) - interp.activeContext.method.literals = ["a", "b", "c"] + interp.activeContext.method.literals = fakeliterals("a", "b", "c") interp.step() interp.step() interp.step() @@ -111,7 +114,7 @@ w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(bytecode) - interp.activeContext.method.literals = [w_association] + interp.activeContext.method.literals = fakeliterals(w_association) interp.step() assert interp.activeContext.stack == ["myvalue"] @@ -208,7 +211,7 @@ m_class.installmethod("foo", model.W_CompiledMethod(0, pushConstantOneBytecode + bytecode)) interp = new_interpreter(bytecodes) - interp.activeContext.method.literals = ["foo"] + interp.activeContext.method.literals = fakeliterals("foo") interp.activeContext.push(w_object) callerContext = interp.activeContext interp.step() @@ -231,11 +234,11 @@ bytecode = ''.join(map(chr, [ 16, 119, 178, 154, 118, 164, 11, 112, 16, 118, 177, 224, 112, 16, 119, 177, 224, 176, 124 ])) m_class = mockclassmirror(0) method = model.W_CompiledMethod(1, bytecode, 1) - method.literals[0] = "fib:" + method.literals = fakeliterals("fib:") m_class.installmethod("fib:", method) w_object = m_class.new() interp = new_interpreter(sendLiteralSelectorBytecode(16) + returnTopFromMethod) - interp.activeContext.method.literals = ["fib:"] + interp.activeContext.method.literals = fakeliterals("fib:") interp.activeContext.push(w_object) interp.activeContext.push(wrap_int(8)) result = interp.interpret() @@ -248,7 +251,7 @@ m_smallintclass.installmethod("sub", prim_meth) try: interp = new_interpreter(sendLiteralSelectorBytecode(1 + 16)) - interp.activeContext.method.literals = ["foo", "sub"] + interp.activeContext.method.literals = fakeliterals("foo", "sub") interp.activeContext.push(wrap_int(50)) interp.activeContext.push(wrap_int(8)) callerContext = interp.activeContext @@ -335,7 +338,7 @@ w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(pushConstantOneBytecode + bytecode) - interp.activeContext.method.literals = [w_association] + interp.activeContext.method.literals = fakeliterals(w_association) interp.step() interp.step() assert w_association.fetch(1) == interp.ONE @@ -399,10 +402,10 @@ model.W_CompiledMethod(0, bytecode)) m_supersuper.installmethod("foo", model.W_CompiledMethod(0, "")) - m_class.methoddict["foo"].literals = ["foo"] - m_super.methoddict["foo"].literals = ["foo"] + m_class.methoddict["foo"].literals = fakeliterals("foo") + m_super.methoddict["foo"].literals = fakeliterals("foo") interp = new_interpreter(bytecodes) - interp.activeContext.method.literals = ["foo"] + interp.activeContext.method.literals = fakeliterals("foo") interp.activeContext.push(w_object) for m_specificclass in [m_class, m_super, m_supersuper]: callerContext = interp.activeContext Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Thu Oct 25 01:44:57 2007 @@ -35,7 +35,7 @@ amethod = w_smallint_class.lookup("abs") # First literal of the abs method is # a real smalltalk int - w_frame = amethod.createFrame(amethod.literals[0], []) + w_frame = amethod.createFrame(sqm.W_SmallInteger(3), []) interp.activeContext = w_frame print amethod From akuhn at codespeak.net Thu Oct 25 01:47:33 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Thu, 25 Oct 2007 01:47:33 +0200 (CEST) Subject: [pypy-svn] r47879 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071024234733.1F60E8108@code0.codespeak.net> Author: akuhn Date: Thu Oct 25 01:47:32 2007 New Revision: 47879 Modified: pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/mirror.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_classtable.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: nice __str__ for W_AbstractObjectWithClassReference that prints 'a Object' for instances and 'Object class' for classes. Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Thu Oct 25 01:47:32 2007 @@ -12,6 +12,7 @@ CLASS_SUPERCLASS_INDEX = 0 CLASS_METHODDICT_INDEX = 1 CLASS_FORMAT_INDEX = 2 +CLASS_NAME_INDEX = 6 METHODDICT_VALUES_INDEX = 1 METHODDICT_NAMES_INDEX = 2 Modified: pypy/dist/pypy/lang/smalltalk/mirror.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/mirror.py (original) +++ pypy/dist/pypy/lang/smalltalk/mirror.py Thu Oct 25 01:47:32 2007 @@ -25,7 +25,7 @@ self.methoddict = {} self.m_superclass = None self.m_metaclass = None - self.name = '?' + self.name = '?' # take care when initing this, metaclasses do not have a name! self.invalid = True def check(self): Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 01:47:32 2007 @@ -79,6 +79,17 @@ def getclassmirror(self): return self.m_class + def __str__(self): + self.getclassmirror().check() + if self.size() >= 9: + return ''.join(self.fetch(sqc.CLASS_NAME_INDEX).bytes) + " class" + else: + return "a " + ''.join(self.getclass().fetch(sqc.CLASS_NAME_INDEX).bytes) + + def getclass(self): + self.getclassmirror().check() + return self.getclassmirror().w_self + def invariant(self): from pypy.lang.smalltalk.mirror import ClassMirror return (W_AbstractObjectWithIdentityHash.invariant(self) and Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 01:47:32 2007 @@ -333,16 +333,18 @@ literals = [self.decode_pointer(pointer).w_object for pointer in self.chunk.data[:literalsize+1]] # -------------------- - l = [] - for each in self.chunk.data[literalsize+1:]: - l.append(int2str(each)) - bytes = "".join(l) - bytes = bytes[:-(self.format & 3)] + bbytes = [] + for each in self.chunk.data: + bbytes.append(chr((each >> 24) & 0xff)) + bbytes.append(chr((each >> 16) & 0xff)) + bbytes.append(chr((each >> 8) & 0xff)) + bbytes.append(chr((each >> 0) & 0xff)) + bbytes = bbytes[(literalsize + 1)*4:-(self.format & 3)] # omit literals & odd bytes # XXX assert mirrorcache.get_or_build(self.g_class.w_object) is # ct.m_CompiledMethod w_compiledmethod.__init__( literalsize = literalsize, - bytes = bytes, + bytes = ''.join(bbytes), argsize = numargs, tempsize = tempsize, primitive = primitive) Modified: pypy/dist/pypy/lang/smalltalk/test/test_classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_classtable.py Thu Oct 25 01:47:32 2007 @@ -20,3 +20,4 @@ def test_metaclass_of_metaclass_is_an_instance_of_metaclass(): assert ct.m_Metaclass.m_metaclass.m_metaclass is ct.m_Metaclass + Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 01:47:32 2007 @@ -80,6 +80,13 @@ assert isinstance(w_float_class_name, sqm.W_BytesObject) assert w_float_class_name.bytes == list("Float") + + assert str(w_float_class) == "Float class" + + assert str(w_float_class.getclass()) == "a Metaclass" # yes, with article here. + + assert str(w_float_class.getclass().getclass()) == "Metaclass class" + def test_lookup_abs_in_integer(): image = create_squeakimage() @@ -103,3 +110,4 @@ print interp.activeContext.stack except sqi.ReturnFromTopLevel, e: return e.object + From tverwaes at codespeak.net Thu Oct 25 01:57:36 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 01:57:36 +0200 (CEST) Subject: [pypy-svn] r47880 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071024235736.539E7809D@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 01:57:36 2007 New Revision: 47880 Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: TOFIX: failing miniimage test on abs with negative smallints Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 01:57:36 2007 @@ -88,7 +88,7 @@ assert str(w_float_class.getclass().getclass()) == "Metaclass class" -def test_lookup_abs_in_integer(): +def test_lookup_abs_in_integer(int=10): image = create_squeakimage() amethod = None @@ -99,7 +99,7 @@ amethod = w_smallint_class.lookup("abs") # First literal of the abs method is # a real smalltalk int - w_frame = amethod.createFrame(sqm.W_SmallInteger(10), []) + w_frame = amethod.createFrame(sqm.W_SmallInteger(int), []) interp.activeContext = w_frame print amethod @@ -111,3 +111,6 @@ except sqi.ReturnFromTopLevel, e: return e.object +def test_lookup_neg_abs_in_integer(): + py.test.fail("TOFIX methodlookup 'negated' fails in mirror SmallInteger") + test_lookup_abs_in_integer(-3) From akuhn at codespeak.net Thu Oct 25 02:11:30 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Thu, 25 Oct 2007 02:11:30 +0200 (CEST) Subject: [pypy-svn] r47881 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071025001130.E57F280FD@code0.codespeak.net> Author: akuhn Date: Thu Oct 25 02:11:30 2007 New Revision: 47881 Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: using setup_module(module) in test/test_miniimage.py Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 02:11:30 2007 @@ -1,95 +1,100 @@ # ----- mini.image productline ------------------------------- # NOT relying on order of methods -# one big method to rule them all +# using setup_module(module) now import py from pypy.lang.smalltalk import squeakimage as sq from pypy.lang.smalltalk import model as sqm from pypy.lang.smalltalk import constants as sqc from pypy.lang.smalltalk import interpreter as sqi +# lazy initialization of test data, ie ImageReader and Float class -mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') - -def test_miniimageexists(): - assert mini_image.check(dir=False) - -def get_miniimage(): +def setup_module(module): + global mini_image + global reader + global image + mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') + reader = open_miniimage() + reader.initialize() + image = sq.SqueakImage() + image.from_reader(get_reader()) + +def open_miniimage(): return sq.ImageReader(sq.Stream(mini_image.open())) -def create_squeakimage(): - example = get_miniimage() - example.initialize() +def get_reader(): + return reader - image = sq.SqueakImage() - image.from_reader(example) +def get_image(): return image + +def get_float_class(): + image = get_image() + return image.special(sqc.SO_FLOAT_CLASS) + +# ------ tests ------------------------------------------ + +def test_miniimageexists(): + assert mini_image.check(dir=False) def test_read_header(): - example = get_miniimage() - example.read_header() - assert example.endofmemory == 0x93174 - assert example.oldbaseaddress == 0x6649000 - assert example.specialobjectspointer == 0x6668380 + reader = open_miniimage() + reader.read_header() + assert reader.endofmemory == 0x93174 + assert reader.oldbaseaddress == 0x6649000 + assert reader.specialobjectspointer == 0x6668380 def test_read_all_header(): - example = get_miniimage() - example.read_header() - next = example.stream.peek() + reader = open_miniimage() + reader.read_header() + next = reader.stream.peek() assert next != 0 #expects object header, which must not be 0x00000000 -def test_readimage_productline(): - example = get_miniimage() - example.read_header() - objects = example.read_body() + + +def test_number_of_objects(): + image = get_image() + objects = image.objects assert len(objects) > 0 assert 15000 < len(objects) < 16000 - # at end of file - # py.test.raises(IndexError, lambda: example.stream.next()) - - # all pointers are valid - for each in example.chunks.itervalues(): +def test_all_pointers_are_valid(): + reader = get_reader() + for each in reader.chunks.itervalues(): if each.format < 5: for pointer in each.data: if (pointer & 1) != 1: - assert pointer in example.chunks + assert pointer in reader.chunks - # there are 31 compact classes - example.init_compactclassesarray() - assert len(example.compactclasses) == 31 - example.init_g_objects() - - example.init_w_objects() - - example.fillin_w_objects() - - image = sq.SqueakImage() - - image.from_reader(example) +def test_there_are_31_compact_classes(): + reader = get_reader() + assert len(reader.compactclasses) == 31 +def test_invariant(): + image = get_image() for each in image.objects: each.invariant() - w_float_class = image.special(sqc.SO_FLOAT_CLASS) - +def test_float_class_size(): + w_float_class = get_float_class() assert w_float_class.size() == 9 - + +def test_float_class_name(): + w_float_class = get_float_class() w_float_class_name = w_float_class.fetch(6) - assert isinstance(w_float_class_name, sqm.W_BytesObject) - assert w_float_class_name.bytes == list("Float") +def test_str_w_object(): + w_float_class = get_float_class() assert str(w_float_class) == "Float class" - assert str(w_float_class.getclass()) == "a Metaclass" # yes, with article here. - assert str(w_float_class.getclass().getclass()) == "Metaclass class" def test_lookup_abs_in_integer(int=10): - image = create_squeakimage() + image = get_image() amethod = None w_smallint_class = image.special(sqc.SO_SMALLINTEGER_CLASS) @@ -112,5 +117,5 @@ return e.object def test_lookup_neg_abs_in_integer(): - py.test.fail("TOFIX methodlookup 'negated' fails in mirror SmallInteger") + py.test.skip("TOFIX methodlookup 'negated' fails in mirror SmallInteger") test_lookup_abs_in_integer(-3) From akuhn at codespeak.net Thu Oct 25 02:36:17 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Thu, 25 Oct 2007 02:36:17 +0200 (CEST) Subject: [pypy-svn] r47882 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025003617.BF6E38112@code0.codespeak.net> Author: akuhn Date: Thu Oct 25 02:36:17 2007 New Revision: 47882 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: very strange, for example range(4)[:0] returns [] instead of [0,1,2,3]! Hence what we have to write list[:-odd] as list[:len(list)-odd] instead :( found this bug when testing the name of True class :) Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 02:36:17 2007 @@ -306,14 +306,16 @@ w_wordsobject.hash = self.chunk.hash12 # XXX check this def fillin_bytesobject(self, w_bytesobject): - bytes = [] + bbytes = [] for each in self.chunk.data: - bytes.append(chr((each >> 24) & 0xff)) - bytes.append(chr((each >> 16) & 0xff)) - bytes.append(chr((each >> 8) & 0xff)) - bytes.append(chr((each >> 0) & 0xff)) + bbytes.append(chr((each >> 24) & 0xff)) + bbytes.append(chr((each >> 16) & 0xff)) + bbytes.append(chr((each >> 8) & 0xff)) + bbytes.append(chr((each >> 0) & 0xff)) w_bytesobject.m_class = mirrorcache.get_or_build(self.g_class.w_object) - w_bytesobject.bytes = bytes[:-(self.format & 3)] # omit odd bytes + #strange, for example range(4)[:0] returns [] instead of [0,1,2,3]! + #hence what we have to write list[:-odd] as list[:len(list)-odd] instead :( + w_bytesobject.bytes = bbytes[:len(bbytes)-(self.format & 3)] # omit odd bytes w_bytesobject.hash = self.chunk.hash12 # XXX check this def fillin_compiledmethod(self, w_compiledmethod): @@ -339,7 +341,9 @@ bbytes.append(chr((each >> 16) & 0xff)) bbytes.append(chr((each >> 8) & 0xff)) bbytes.append(chr((each >> 0) & 0xff)) - bbytes = bbytes[(literalsize + 1)*4:-(self.format & 3)] # omit literals & odd bytes + #strange, for example range(4)[:0] returns [] instead of [0,1,2,3]! + #hence what we have to write list[:-odd] as list[:len(list)-odd] instead :( + bbytes = bbytes[(literalsize + 1)*4:len(bbytes)-(self.format & 3)] # omit literals & odd bytes # XXX assert mirrorcache.get_or_build(self.g_class.w_object) is # ct.m_CompiledMethod w_compiledmethod.__init__( Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 02:36:17 2007 @@ -89,9 +89,81 @@ def test_str_w_object(): w_float_class = get_float_class() assert str(w_float_class) == "Float class" - assert str(w_float_class.getclass()) == "a Metaclass" # yes, with article here. + assert str(w_float_class.getclass()) == "a Metaclass" #yes, with article assert str(w_float_class.getclass().getclass()) == "Metaclass class" +def test_nil_true_false(): + image = get_image() + w = image.special(sqc.SO_NIL) + assert str(w) == "a UndefinedObject" #yes, with article + w = image.special(sqc.SO_FALSE) + assert str(w) == "a False" #yes, with article + w = image.special(sqc.SO_TRUE) + assert str(w) == "a True" #yes, with article + +def test_scheduler(): + image = get_image() + w = image.special(sqc.SO_SCHEDULERASSOCIATIONPOINTER) + w0 = w.fetch(0) + assert str(w0) == "'Processor'" + w0 = w.fetch(1) + assert str(w0) == "a ProcessorScheduler" + +def test_special_classes0(): + image = get_image() + w = image.special(sqc.SO_BITMAP_CLASS) + assert str(w) == "Bitmap class" + w = image.special(sqc.SO_SMALLINTEGER_CLASS) + assert str(w) == "SmallInteger class" + w = image.special(sqc.SO_STRING_CLASS) + assert str(w) == "String class" + w = image.special(sqc.SO_ARRAY_CLASS) + assert str(w) == "Array class" + w = image.special(sqc.SO_FLOAT_CLASS) + assert str(w) == "Float class" + w = image.special(sqc.SO_METHODCONTEXT_CLASS) + assert str(w) == "MethodContext class" + w = image.special(sqc.SO_BLOCKCONTEXT_CLASS) + assert str(w) == "BlockContext class" + w = image.special(sqc.SO_POINT_CLASS) + assert str(w) == "Point class" + w = image.special(sqc.SO_LARGEPOSITIVEINTEGER_CLASS) + assert str(w) == "LargePositiveInteger class" + + # to be continued + + """SO_SMALLTALK = 8 + SO_DISPLAY_CLASS = 14 + SO_MESSAGE_CLASS = 15 + SO_COMPILEDMETHOD_CLASS = 16 + SO_LOW_SPACE_SEMAPHORE = 17 + SO_SEMAPHORE_CLASS = 18 + SO_CHARACTER_CLASS = 19 + SO_DOES_NOT_UNDERSTAND = 20 + SO_CANNOT_RETURN = 21 + # no clue what 22 is doing + SO_SPECIAL_SELECTORS_ARRAY = 23 + SO_CHARACTER_TABLE_ARRAY = 24 + SO_MUST_BE_BOOLEAN = 25 + SO_BYTEARRAY_CLASS = 26 + SO_PROCESS_CLASS = 27 + SO_COMPACT_CLASSES_ARRAY = 28 + SO_DELAY_SEMAPHORE = 29 + SO_USER_INTERRUPT_SEMAPHORE = 30 + SO_FLOAT_ZERO = 31 + SO_LARGEPOSITIVEINTEGER_ZERO = 32 + SO_A_POINT = 33 + SO_CANNOT_INTERPRET = 34 + SO_A_METHODCONTEXT = 35 + # no clue what 36 is doing + SO_A_BLOCKCONTEXT = 37 + SO_AN_ARRAY = 38 + SO_PSEUDOCONTEXT_CLASS = 39 + SO_TRANSLATEDMETHOD_CLASS = 40 + SO_FINALIZATION_SEMPAHORE = 41 + SO_LARGENEGATIVEINTEGER_CLASS = 42""" + + def test_lookup_abs_in_integer(int=10): image = get_image() From tverwaes at codespeak.net Thu Oct 25 03:16:59 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 03:16:59 +0200 (CEST) Subject: [pypy-svn] r47883 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071025011659.8147D8108@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 03:16:59 2007 New Revision: 47883 Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: better documentation about lookup in the test (difference between mirror-lookup and class-lookup) Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 03:16:59 2007 @@ -167,19 +167,28 @@ def test_lookup_abs_in_integer(int=10): image = get_image() - amethod = None + interp = sqi.Interpreter() + + w_object = sqm.W_SmallInteger(int) + # XXX + # Should get this from w_object w_smallint_class = image.special(sqc.SO_SMALLINTEGER_CLASS) + w_method = w_smallint_class.lookup("abs") - interp = sqi.Interpreter() + # XXX + # currently still using highlevel lookup directly pointing to + # class. Should work using classmirrors when the metaclass of + # SmallInt is correctly set + + # w_classmirror = w_object.getclassmirror() + # w_method = w_classmirror.lookup("abs") - amethod = w_smallint_class.lookup("abs") - # First literal of the abs method is - # a real smalltalk int - w_frame = amethod.createFrame(sqm.W_SmallInteger(int), []) + assert w_method + w_frame = w_method.createFrame(w_object, []) interp.activeContext = w_frame - print amethod + print w_method while True: try: @@ -190,4 +199,6 @@ def test_lookup_neg_abs_in_integer(): py.test.skip("TOFIX methodlookup 'negated' fails in mirror SmallInteger") + # Fails due to same reason because of which + # classmirror-methodlookup fails test_lookup_abs_in_integer(-3) From pedronis at codespeak.net Thu Oct 25 07:55:10 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 25 Oct 2007 07:55:10 +0200 (CEST) Subject: [pypy-svn] r47884 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071025055510.81F5E8119@code0.codespeak.net> Author: pedronis Date: Thu Oct 25 07:55:09 2007 New Revision: 47884 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Log: new task: repair the broken JIT tests Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Thu Oct 25 07:55:09 2007 @@ -66,6 +66,7 @@ JIT +++ + - repair the JIT tests (if it doesn't happen before), the keep alive killing merge broke them - start writing real unit tests for the merging logic From pedronis at codespeak.net Thu Oct 25 08:06:43 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 25 Oct 2007 08:06:43 +0200 (CEST) Subject: [pypy-svn] r47885 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071025060643.3CA9B8119@code0.codespeak.net> Author: pedronis Date: Thu Oct 25 08:06:42 2007 New Revision: 47885 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Log: fix a typo and move to more appropriate place, this list needs a bit of gardening itself Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Thu Oct 25 08:06:42 2007 @@ -39,17 +39,15 @@ can have more than one translator/annotator around (with the timeshifter) - - Kill half concrete wrapper + - kill half concrete wrapper - - Kill opaque hacks on the C backend + - kill opaque hacks on the C backend - decide how to implement constants in rffi - think about/remove orphaned parts - implement callback mechanism for rffi/lltype/ll2ctypes - - - remove orebuilt costate vars (might happen before) Interpreter +++++++++++ @@ -62,6 +60,8 @@ performance-critical - app-level ctypes + + - remove prebuilt costate vars (might happen before) JIT +++ From lukas at codespeak.net Thu Oct 25 09:38:30 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Thu, 25 Oct 2007 09:38:30 +0200 (CEST) Subject: [pypy-svn] r47886 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025073830.B43D78130@code0.codespeak.net> Author: lukas Date: Thu Oct 25 09:38:29 2007 New Revision: 47886 Modified: pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/interpreter.py Log: use ASSOCIATION_VALUE_INDEX instead of magic number Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Thu Oct 25 09:38:29 2007 @@ -11,12 +11,15 @@ CLASS_SUPERCLASS_INDEX = 0 CLASS_METHODDICT_INDEX = 1 -CLASS_FORMAT_INDEX = 2 -CLASS_NAME_INDEX = 6 +CLASS_FORMAT_INDEX = 2 +CLASS_NAME_INDEX = 6 METHODDICT_VALUES_INDEX = 1 METHODDICT_NAMES_INDEX = 2 +ASSOCIATION_KEY_INDEX = 0 +ASSOCIATION_VALUE_INDEX = 1 + # ----- special objects indices ------- SO_NIL = 0 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 09:38:29 2007 @@ -1,5 +1,5 @@ import py -from pypy.lang.smalltalk import model, primitives +from pypy.lang.smalltalk import model, constants, primitives from pypy.lang.smalltalk import fakeimage @@ -76,7 +76,7 @@ # named var (the value). index = self.currentBytecode & 31 association = self.method.getliteral(index) - self.push(association.fetch(1)) + self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) def storeAndPopReceiverVariableBytecode(self, interp): index = self.currentBytecode & 7 @@ -192,7 +192,7 @@ self.push(self.method.getliteral(variableIndex)) elif variableType == 3: association = self.method.getliteral(variableIndex) - self.push(association.fetch(1)) + self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) def extendedStoreBytecode(self, interp): variableType, variableIndex = self.extendedVariableTypeAndIndex() @@ -204,7 +204,7 @@ raise IllegalStoreError elif variableType == 3: association = self.method.getliteral(variableIndex) - association.store(1,self.top()) + association.store(constants.ASSOCIATION_VALUE_INDEX, self.top()) def extendedStoreAndPopBytecode(self, interp): self.extendedStoreBytecode(interp) @@ -239,14 +239,14 @@ elif opType == 4: # pushLiteralVariable association = self.method.getliteral(third) - self.push(association.fetch(1)) + self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) elif opType == 5: self.receiver.store(third, self.top()) elif opType == 6: self.receiver.store(third, self.pop()) elif opType == 7: association = self.method.getliteral(third) - association.store(1,self.top()) + association.store(constants.ASSOCIATION_VALUE_INDEX, self.top()) def singleExtendedSuperBytecode(self, interp): selector, argcount = self.getExtendedSelectorArgcount() From arigo at codespeak.net Thu Oct 25 10:31:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 10:31:43 +0200 (CEST) Subject: [pypy-svn] r47895 - pypy/extradoc/sprintinfo/bern2007 Message-ID: <20071025083143.189F88129@code0.codespeak.net> Author: arigo Date: Thu Oct 25 10:31:42 2007 New Revision: 47895 Modified: pypy/extradoc/sprintinfo/bern2007/planning.txt Log: Planning for today. Modified: pypy/extradoc/sprintinfo/bern2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/bern2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/bern2007/planning.txt Thu Oct 25 10:31:42 2007 @@ -4,42 +4,47 @@ Carl Friedrich Armin Adrian K -Toon -Niko (virtually) +Toon (still sleeping for now) +Niko +Oscar +Lukas +Tudor Task ideas ---------- -- Print the blue book (Adrian K) DONE +- give everybody an account (arigo) -- Solve the svn problem (Armin) DONE +- present coding conventions (cfbolz) -- Complete the simple model (a bit all) DONE +- present the "mirror" ("shadow"?) (cfbolz, arigo) Afterwards: ----------- -- Implement loading the image (Adrian K, Carl Friedrich) IN PROGRESS +- Implement loading the image (Adrian K, Carl Friedrich) + MOSTLY DONE, the mapping between the loaded object and + the ones expected by the rest of the code is incomplete + (cfbolz, akuhn) - interpreter loop and bytecode dispatch (Armin, Adrian L, Lukas, Toon) DONE -- implementing numbered primitives (arithmetic) - (Niko, Tudor, Oscar) IN PROGRESS - - begin implementing the base image, including class objects - - templates for the math operations more or less complete, except for - promoting from small to large integer - - plenty of primitives to go :) (guess that's about it) +- implementing numbered primitives MORE PROGRESS (lukas, + cfbolz, niko) + - still more to go -- implement all the bytecodes (Armin, Toon, Lukas, Adrian L) IN PROGRESS +- implement all the bytecodes (Armin, Toon, Lukas, Adrian L) + MOSTLY DONE - define a common interface for real image and mock image (Niko, Carl - Friedrich, Adrian K) + Friedrich, Adrian K) DONE - try to generate the fib example and fight with RPython :) (Armin, together to teach the smalltalkers) -- write the report for yesterday (Carl Friedrich, Adrian K) +- write the report for yesterday (Carl Friedrich, Adrian K, + ...) okay, done. From cfbolz at codespeak.net Thu Oct 25 10:42:24 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 25 Oct 2007 10:42:24 +0200 (CEST) Subject: [pypy-svn] r47896 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025084224.1209F8129@code0.codespeak.net> Author: cfbolz Date: Thu Oct 25 10:42:23 2007 New Revision: 47896 Modified: pypy/dist/pypy/lang/smalltalk/classtable.py pypy/dist/pypy/lang/smalltalk/mirror.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: reuse the prebuilt mirror instances when loading the image Modified: pypy/dist/pypy/lang/smalltalk/classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/classtable.py Thu Oct 25 10:42:23 2007 @@ -58,17 +58,20 @@ # Other classes def define_cls(cls_nm, supercls_nm, instvarsize=0, format=mirror.POINTERS): + assert cls_nm.startswith("m_") meta_nm = cls_nm + "Class" meta_super_nm = supercls_nm + "Class" m_meta_cls = globals()[meta_nm] = classtable[meta_nm] = \ bootstrap_classmirror(0, # XXX classtable[meta_super_nm], - m_Metaclass) + m_Metaclass, + name=meta_nm[2:]) m_cls = globals()[cls_nm] = classtable[cls_nm] = \ bootstrap_classmirror(instvarsize, classtable[supercls_nm], m_meta_cls, - format=format) + format=format, + name=cls_nm[2:]) define_cls("m_Magnitude", "m_Object") define_cls("m_Character", "m_Magnitude", instvarsize=1) Modified: pypy/dist/pypy/lang/smalltalk/mirror.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/mirror.py (original) +++ pypy/dist/pypy/lang/smalltalk/mirror.py Thu Oct 25 10:42:23 2007 @@ -147,4 +147,8 @@ mirror.check() return mirror + def assign_existing_mirror(self, w_class, m_class): + assert w_class not in self.cache + self.cache[w_class] = m_class + mirrorcache = MirrorCache() Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 10:42:23 2007 @@ -83,6 +83,7 @@ self.init_compactclassesarray() self.init_g_objects() self.init_w_objects() + self.assign_mirrors() self.fillin_w_objects() def read_header(self): @@ -116,6 +117,26 @@ for chunk in self.chunks.itervalues(): chunk.g_object.init_w_object() + def assign_mirrors(self): + # assign the mirrors to the classes already in classtable + from pypy.lang.smalltalk import classtable, constants + for so_index, name in [ + (constants.SO_SMALLINTEGER_CLASS, "m_SmallInteger"), + (constants.SO_STRING_CLASS, "m_String"), + (constants.SO_FLOAT_CLASS, "m_Float"), + #(constants.SO_METHODCONTEXT_CLASS, "m_MethodContext"), + (constants.SO_CHARACTER_CLASS, "m_Character"), + (constants.SO_BYTEARRAY_CLASS, "m_ByteArray"), + (constants.SO_COMPILEDMETHOD_CLASS, "m_CompiledMethod")]: + mirrorcache.assign_existing_mirror( + self.special_object(so_index), + getattr(classtable, name)) + # XXX more missing + + def special_object(self, index): + special = self.chunks[self.specialobjectspointer].g_object.pointers + return special[index].w_object + def fillin_w_objects(self): for chunk in self.chunks.itervalues(): chunk.g_object.fillin_w_object() @@ -212,7 +233,6 @@ self.init_data(chunk) # for pointers self.chunk = chunk # for bytes, words and compiledmethod self.w_object = None - self.init_w_object() def init_class(self, chunk): if chunk.iscompact(): @@ -354,7 +374,7 @@ primitive = primitive) w_compiledmethod.literals = literals - + class ImageChunk(object): Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 10:42:23 2007 @@ -202,3 +202,10 @@ # Fails due to same reason because of which # classmirror-methodlookup fails test_lookup_abs_in_integer(-3) + +def test_map_mirrors_to_classtable(): + from pypy.lang.smalltalk import classtable, mirror + w_compiledmethod_class = image.special(sqc.SO_COMPILEDMETHOD_CLASS) + m_compiledmethod_class = mirror.mirrorcache.getmirror( + w_compiledmethod_class) + assert m_compiledmethod_class is classtable.m_CompiledMethod From niko at codespeak.net Thu Oct 25 10:46:19 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 25 Oct 2007 10:46:19 +0200 (CEST) Subject: [pypy-svn] r47897 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025084619.9F2018129@code0.codespeak.net> Author: niko Date: Thu Oct 25 10:46:19 2007 New Revision: 47897 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: 1. avoid using "import foo as bar" (most of the time) 2. add some stubs for control primitives (I know, I shouldn't mix, but I forgot about #2 till it was too late :) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 10:46:19 2007 @@ -1,7 +1,7 @@ import operator from pypy.lang.smalltalk import model, mirror -import pypy.lang.smalltalk.classtable as ct -import pypy.lang.smalltalk.fakeimage as fimg +from pypy.lang.smalltalk import classtable +from pypy.lang.smalltalk import fakeimage from pypy.rlib import rarithmetic class PrimitiveFailedError(Exception): @@ -19,9 +19,9 @@ if isinstance(w_obj, model.W_PointersObject): return w_obj.getindexedvar(idx) elif isinstance(w_obj, model.W_WordsObject): - return fimg.wrap_int(w_obj.getword(idx)) + return fakeimage.wrap_int(w_obj.getword(idx)) elif isinstance(w_obj, model.W_BytesObject): - return fimg.wrap_int(w_obj.getbyte(idx)) + return fakeimage.wrap_int(w_obj.getbyte(idx)) raise PrimitiveFailedError() def assert_bounds(idx, minb, maxb): @@ -68,7 +68,7 @@ raise PrimitiveFailedError() if value < -1073741824: raise PrimitiveFailedError() - return fimg.wrap_int(value) + return fakeimage.wrap_int(value) ADD = 1 SUBTRACT = 2 @@ -189,7 +189,7 @@ [w_v1, w_v2] = res v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) - w_res = fimg.wrap_float(op(v1, v2)) + w_res = fakeimage.wrap_float(op(v1, v2)) return w_res prim_table[code] = func @@ -242,15 +242,15 @@ def func(stack): w_obj, idx = common_at(stack) byte = w_obj.getbyte(idx) - return fimg.CharacterTable[byte] + return fakeimage.CharacterTable[byte] @primitive(STRING_AT_PUT) @stack(3) def func(stack): w_obj, idx, w_val = common_at_put(stack) - if w_val.getclassmirror() is not ct.m_Character: + if w_val.getclassmirror() is not classtable.m_Character: raise PrimitiveFailedError() - w_obj.setbyte(idx, fimg.ord_w_char(w_val)) + w_obj.setbyte(idx, fakeimage.ord_w_char(w_val)) return w_val # ___________________________________________________________________________ @@ -267,6 +267,7 @@ STORE_STACKP = 76 # Blue Book: primitiveAsObject SOME_INSTANCE = 77 NEXT_INSTANCE = 78 +NEW_METHOD = 79 @primitive(OBJECT_AT) @stack(2) @@ -363,6 +364,73 @@ [w_obj] = stack raise PrimitiveNotYetWrittenError() + at primitive(NEW_METHOD) +def func(frame): + raise PrimitiveNotYetWrittenError() + +# ___________________________________________________________________________ +# Control Primitives + +EQUIVALENT = 110 +CLASS = 111 +BYTES_LEFT = 112 +QUIT = 113 +EXIT_TO_DEBUGGER = 114 +CHANGE_CLASS = 115 # Blue Book: primitiveOopsLeft + + at primitive(EQUIVALENT) + at stack(2) +def func(stack): + [w_arg, w_rcvr] = stack + return w_arg == w_rcvr + + at primitive(EQUIVALENT) + at stack(1) +def func(stack): + [w_obj] = stack + return w_obj.w_class + + at primitive(BYTES_LEFT) +def func(frame): + raise PrimitiveNotYetWrittenError() + + at primitive(QUIT) +def func(frame): + raise PrimitiveNotYetWrittenError() + + at primitive(EXIT_TO_DEBUGGER) +def func(frame): + raise PrimitiveNotYetWrittenError() + + at primitive(CHANGE_CLASS) + at stack(2) +def func(stack): + [w_arg, w_rcvr] = stack + w_arg_class = w_arg.w_class + w_rcvr_class = w_rcvr.w_class + + # We should fail if: + + # 1. Rcvr or arg are SmallIntegers + if (w_arg_class == classtable.w_SmallInteger or + w_rcvr_class == classtable.w_SmallInteger): + raise PrimitiveFailedError() + + # 2. Rcvr is an instance of a compact class and argument isn't + # or vice versa (?) + + # 3. Format of rcvr is different from format of argument + if w_arg_class.format != w_rcvr_class.format: + raise PrimitiveFailedError() + + # Fail when argument class is fixed and rcvr's size differs from the + # size of an instance of the arg + if w_arg_class.instsize() != w_rcvr_class.instsize(): + raise PrimitiveFailedError() + + w_rcvr.w_class = w_arg.w_class + return + # ___________________________________________________________________________ # Boolean Primitives @@ -396,7 +464,7 @@ v1 = unwrap_int(w_v1) v2 = unwrap_int(w_v2) res = op(v1, v2) - w_res = fimg.wrap_bool(res) + w_res = fakeimage.wrap_bool(res) return w_res for (code,op) in bool_ops.items(): @@ -407,7 +475,7 @@ v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) res = op(v1, v2) - w_res = fimg.wrap_bool(res) + w_res = fakeimage.wrap_bool(res) return w_res # ___________________________________________________________________________ @@ -430,13 +498,13 @@ def define_const_primitives(): for (code, const) in [ - (PUSH_TRUE, fimg.w_true), - (PUSH_FALSE, fimg.w_false), - (PUSH_NIL, fimg.w_nil), - (PUSH_MINUS_ONE, fimg.w_mone), - (PUSH_ZERO, fimg.w_zero), - (PUSH_ONE, fimg.w_one), - (PUSH_TWO, fimg.w_two), + (PUSH_TRUE, fakeimage.w_true), + (PUSH_FALSE, fakeimage.w_false), + (PUSH_NIL, fakeimage.w_nil), + (PUSH_MINUS_ONE, fakeimage.w_mone), + (PUSH_ZERO, fakeimage.w_zero), + (PUSH_ONE, fakeimage.w_one), + (PUSH_TWO, fakeimage.w_two), ]: @primitive(code) @stack(1) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 10:46:19 2007 @@ -1,23 +1,25 @@ import py from pypy.lang.smalltalk.primitives import prim_table, PrimitiveFailedError -import pypy.lang.smalltalk.primitives as p from pypy.lang.smalltalk import model, mirror -import pypy.lang.smalltalk.interpreter as interp -import pypy.lang.smalltalk.classtable as ct -import pypy.lang.smalltalk.fakeimage as fimg +from pypy.lang.smalltalk import interpreter +from pypy.lang.smalltalk import classtable +from pypy.lang.smalltalk import fakeimage + +# Violates the guideline, but we use it A LOT to reference the primitive codes: +import pypy.lang.smalltalk.primitives as p -mockclassmirror = ct.bootstrap_classmirror +mockclassmirror = classtable.bootstrap_classmirror -class MockFrame(interp.W_MethodContext): +class MockFrame(interpreter.W_MethodContext): def __init__(self, stack): self.stack = stack def wrap(x): - if isinstance(x, int): return fimg.wrap_int(x) - if isinstance(x, float): return fimg.wrap_float(x) + if isinstance(x, int): return fakeimage.wrap_int(x) + if isinstance(x, float): return fakeimage.wrap_float(x) if isinstance(x, model.W_Object): return x - if isinstance(x, str) and len(x) == 1: return fimg.wrap_char(x) - if isinstance(x, str): return fimg.wrap_string(x) + if isinstance(x, str) and len(x) == 1: return fakeimage.wrap_char(x) + if isinstance(x, str): return fakeimage.wrap_string(x) if isinstance(x, mirror.ClassMirror): return x.w_self raise NotImplementedError @@ -168,11 +170,11 @@ assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i]) def test_object_at(): - w_v = prim(p.OBJECT_AT, ["q", fimg.CHARACTER_VALUE_INDEX]) + w_v = prim(p.OBJECT_AT, ["q", fakeimage.CHARACTER_VALUE_INDEX]) assert w_v.value == ord("q") def test_invalid_object_at(): - prim_fails(p.OBJECT_AT, ["q", fimg.CHARACTER_VALUE_INDEX+1]) + prim_fails(p.OBJECT_AT, ["q", fakeimage.CHARACTER_VALUE_INDEX+1]) def test_object_at_put(): w_obj = mockclassmirror(1).new() @@ -191,24 +193,24 @@ assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i]) def test_new(): - w_res = prim(p.NEW, [ct.m_Object]) - assert w_res.getclassmirror() == ct.m_Object + w_res = prim(p.NEW, [classtable.m_Object]) + assert w_res.getclassmirror() == classtable.m_Object def test_invalid_new(): - prim_fails(p.NEW, [ct.m_ByteString]) + prim_fails(p.NEW, [classtable.m_ByteString]) def test_new_with_arg(): - w_res = prim(p.NEW_WITH_ARG, [ct.m_ByteString, 20]) - assert w_res.getclassmirror() == ct.m_ByteString + w_res = prim(p.NEW_WITH_ARG, [classtable.m_ByteString, 20]) + assert w_res.getclassmirror() == classtable.m_ByteString assert w_res.size() == 20 def test_invalid_new_with_arg(): - prim_fails(p.NEW_WITH_ARG, [ct.m_Object, 20]) + prim_fails(p.NEW_WITH_ARG, [classtable.m_Object, 20]) def test_inst_var_at(): # I am not entirely sure if this is what this primitive is # supposed to do, so the test may be bogus: - w_v = prim(p.INST_VAR_AT, ["q", fimg.CHARACTER_VALUE_INDEX]) + w_v = prim(p.INST_VAR_AT, ["q", fakeimage.CHARACTER_VALUE_INDEX]) assert w_v.value == ord("q") w_v = prim(p.INST_VAR_AT, ["abc", 1]) assert w_v.value == ord("b") @@ -223,30 +225,30 @@ def test_const_primitives(): for (code, const) in [ - (p.PUSH_TRUE, fimg.w_true), - (p.PUSH_FALSE, fimg.w_false), - (p.PUSH_NIL, fimg.w_nil), - (p.PUSH_MINUS_ONE, fimg.w_mone), - (p.PUSH_ZERO, fimg.w_zero), - (p.PUSH_ONE, fimg.w_one), - (p.PUSH_TWO, fimg.w_two), + (p.PUSH_TRUE, fakeimage.w_true), + (p.PUSH_FALSE, fakeimage.w_false), + (p.PUSH_NIL, fakeimage.w_nil), + (p.PUSH_MINUS_ONE, fakeimage.w_mone), + (p.PUSH_ZERO, fakeimage.w_zero), + (p.PUSH_ONE, fakeimage.w_one), + (p.PUSH_TWO, fakeimage.w_two), ]: - assert prim(code, [fimg.w_nil]) is const - assert prim(p.PUSH_SELF, [fimg.w_nil]) is fimg.w_nil + assert prim(code, [fakeimage.w_nil]) is const + assert prim(p.PUSH_SELF, [fakeimage.w_nil]) is fakeimage.w_nil assert prim(p.PUSH_SELF, ["a"]) is wrap("a") def test_boolean(): - assert prim(p.LESSTHAN, [1,2]) == fimg.w_true - assert prim(p.GREATERTHAN, [3,4]) == fimg.w_false - assert prim(p.LESSOREQUAL, [1,2]) == fimg.w_true - assert prim(p.GREATEROREQUAL, [3,4]) == fimg.w_false - assert prim(p.EQUAL, [2,2]) == fimg.w_true - assert prim(p.NOTEQUAL, [2,2]) == fimg.w_false + assert prim(p.LESSTHAN, [1,2]) == fakeimage.w_true + assert prim(p.GREATERTHAN, [3,4]) == fakeimage.w_false + assert prim(p.LESSOREQUAL, [1,2]) == fakeimage.w_true + assert prim(p.GREATEROREQUAL, [3,4]) == fakeimage.w_false + assert prim(p.EQUAL, [2,2]) == fakeimage.w_true + assert prim(p.NOTEQUAL, [2,2]) == fakeimage.w_false def test_float_boolean(): - assert prim(p.FLOAT_LESSTHAN, [1.0,2.0]) == fimg.w_true - assert prim(p.FLOAT_GREATERTHAN, [3.0,4.0]) == fimg.w_false - assert prim(p.FLOAT_LESSOREQUAL, [1.3,2.6]) == fimg.w_true - assert prim(p.FLOAT_GREATEROREQUAL, [3.5,4.9]) == fimg.w_false - assert prim(p.FLOAT_EQUAL, [2.2,2.2]) == fimg.w_true - assert prim(p.FLOAT_NOTEQUAL, [2.2,2.2]) == fimg.w_false + assert prim(p.FLOAT_LESSTHAN, [1.0,2.0]) == fakeimage.w_true + assert prim(p.FLOAT_GREATERTHAN, [3.0,4.0]) == fakeimage.w_false + assert prim(p.FLOAT_LESSOREQUAL, [1.3,2.6]) == fakeimage.w_true + assert prim(p.FLOAT_GREATEROREQUAL, [3.5,4.9]) == fakeimage.w_false + assert prim(p.FLOAT_EQUAL, [2.2,2.2]) == fakeimage.w_true + assert prim(p.FLOAT_NOTEQUAL, [2.2,2.2]) == fakeimage.w_false From akuhn at codespeak.net Thu Oct 25 10:53:48 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Thu, 25 Oct 2007 10:53:48 +0200 (CEST) Subject: [pypy-svn] r47898 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025085348.40417812B@code0.codespeak.net> Author: akuhn Date: Thu Oct 25 10:53:46 2007 New Revision: 47898 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: don't abbr id'fier of import stmts Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 10:53:46 2007 @@ -1,7 +1,7 @@ import sys from pypy.rlib import rrandom from pypy.rlib.rarithmetic import intmask -from pypy.lang.smalltalk import constants as sqc +from pypy.lang.smalltalk import constants class MethodNotFound(Exception): pass @@ -82,9 +82,9 @@ def __str__(self): self.getclassmirror().check() if self.size() >= 9: - return ''.join(self.fetch(sqc.CLASS_NAME_INDEX).bytes) + " class" + return ''.join(self.fetch(constants.CLASS_NAME_INDEX).bytes) + " class" else: - return "a " + ''.join(self.getclass().fetch(sqc.CLASS_NAME_INDEX).bytes) + return "a " + ''.join(self.getclass().fetch(constants.CLASS_NAME_INDEX).bytes) def getclass(self): self.getclassmirror().check() @@ -116,9 +116,9 @@ isinstance(self.vars, list)) def compiledmethodnamed(self, methodname): - w_methoddict = self.fetch(sqc.CLASS_METHODDICT_INDEX).vars - names = w_methoddict[sqc.METHODDICT_NAMES_INDEX:] - values = w_methoddict[sqc.METHODDICT_VALUES_INDEX].vars + w_methoddict = self.fetch(constants.CLASS_METHODDICT_INDEX).vars + names = w_methoddict[constants.METHODDICT_NAMES_INDEX:] + values = w_methoddict[constants.METHODDICT_VALUES_INDEX].vars for var in names: if isinstance(var, W_BytesObject): if str(var) == repr(methodname): @@ -134,7 +134,7 @@ # Current hack because we don't have a ref to the real # nil yet... XXX XXX XXX try: - new_class = in_class.vars[sqc.CLASS_SUPERCLASS_INDEX] + new_class = in_class.vars[constants.CLASS_SUPERCLASS_INDEX] if in_class == new_class: raise Exception else: Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 10:53:46 2007 @@ -2,10 +2,10 @@ # NOT relying on order of methods # using setup_module(module) now import py -from pypy.lang.smalltalk import squeakimage as sq -from pypy.lang.smalltalk import model as sqm -from pypy.lang.smalltalk import constants as sqc -from pypy.lang.smalltalk import interpreter as sqi +from pypy.lang.smalltalk import squeakimage +from pypy.lang.smalltalk import model +from pypy.lang.smalltalk import constants +from pypy.lang.smalltalk import interpreter # lazy initialization of test data, ie ImageReader and Float class @@ -16,11 +16,11 @@ mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') reader = open_miniimage() reader.initialize() - image = sq.SqueakImage() + image = squeakimage.SqueakImage() image.from_reader(get_reader()) def open_miniimage(): - return sq.ImageReader(sq.Stream(mini_image.open())) + return squeakimage.ImageReader(squeakimage.Stream(mini_image.open())) def get_reader(): return reader @@ -30,7 +30,7 @@ def get_float_class(): image = get_image() - return image.special(sqc.SO_FLOAT_CLASS) + return image.special(constants.SO_FLOAT_CLASS) # ------ tests ------------------------------------------ @@ -83,7 +83,7 @@ def test_float_class_name(): w_float_class = get_float_class() w_float_class_name = w_float_class.fetch(6) - assert isinstance(w_float_class_name, sqm.W_BytesObject) + assert isinstance(w_float_class_name, model.W_BytesObject) assert w_float_class_name.bytes == list("Float") def test_str_w_object(): @@ -94,16 +94,16 @@ def test_nil_true_false(): image = get_image() - w = image.special(sqc.SO_NIL) + w = image.special(constants.SO_NIL) assert str(w) == "a UndefinedObject" #yes, with article - w = image.special(sqc.SO_FALSE) + w = image.special(constants.SO_FALSE) assert str(w) == "a False" #yes, with article - w = image.special(sqc.SO_TRUE) + w = image.special(constants.SO_TRUE) assert str(w) == "a True" #yes, with article def test_scheduler(): image = get_image() - w = image.special(sqc.SO_SCHEDULERASSOCIATIONPOINTER) + w = image.special(constants.SO_SCHEDULERASSOCIATIONPOINTER) w0 = w.fetch(0) assert str(w0) == "'Processor'" w0 = w.fetch(1) @@ -111,23 +111,23 @@ def test_special_classes0(): image = get_image() - w = image.special(sqc.SO_BITMAP_CLASS) + w = image.special(constants.SO_BITMAP_CLASS) assert str(w) == "Bitmap class" - w = image.special(sqc.SO_SMALLINTEGER_CLASS) + w = image.special(constants.SO_SMALLINTEGER_CLASS) assert str(w) == "SmallInteger class" - w = image.special(sqc.SO_STRING_CLASS) + w = image.special(constants.SO_STRING_CLASS) assert str(w) == "String class" - w = image.special(sqc.SO_ARRAY_CLASS) + w = image.special(constants.SO_ARRAY_CLASS) assert str(w) == "Array class" - w = image.special(sqc.SO_FLOAT_CLASS) + w = image.special(constants.SO_FLOAT_CLASS) assert str(w) == "Float class" - w = image.special(sqc.SO_METHODCONTEXT_CLASS) + w = image.special(constants.SO_METHODCONTEXT_CLASS) assert str(w) == "MethodContext class" - w = image.special(sqc.SO_BLOCKCONTEXT_CLASS) + w = image.special(constants.SO_BLOCKCONTEXT_CLASS) assert str(w) == "BlockContext class" - w = image.special(sqc.SO_POINT_CLASS) + w = image.special(constants.SO_POINT_CLASS) assert str(w) == "Point class" - w = image.special(sqc.SO_LARGEPOSITIVEINTEGER_CLASS) + w = image.special(constants.SO_LARGEPOSITIVEINTEGER_CLASS) assert str(w) == "LargePositiveInteger class" # to be continued @@ -167,13 +167,13 @@ def test_lookup_abs_in_integer(int=10): image = get_image() - interp = sqi.Interpreter() + interp = interpreter.Interpreter() - w_object = sqm.W_SmallInteger(int) + w_object = model.W_SmallInteger(int) # XXX # Should get this from w_object - w_smallint_class = image.special(sqc.SO_SMALLINTEGER_CLASS) + w_smallint_class = image.special(constants.SO_SMALLINTEGER_CLASS) w_method = w_smallint_class.lookup("abs") # XXX @@ -194,7 +194,7 @@ try: interp.step() print interp.activeContext.stack - except sqi.ReturnFromTopLevel, e: + except interpreter.ReturnFromTopLevel, e: return e.object def test_lookup_neg_abs_in_integer(): @@ -205,7 +205,7 @@ def test_map_mirrors_to_classtable(): from pypy.lang.smalltalk import classtable, mirror - w_compiledmethod_class = image.special(sqc.SO_COMPILEDMETHOD_CLASS) + w_compiledmethod_class = image.special(constants.SO_COMPILEDMETHOD_CLASS) m_compiledmethod_class = mirror.mirrorcache.getmirror( w_compiledmethod_class) assert m_compiledmethod_class is classtable.m_CompiledMethod From lukas at codespeak.net Thu Oct 25 10:54:06 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Thu, 25 Oct 2007 10:54:06 +0200 (CEST) Subject: [pypy-svn] r47899 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025085406.EB2C6812D@code0.codespeak.net> Author: lukas Date: Thu Oct 25 10:54:06 2007 New Revision: 47899 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: added some more primitive skeletons Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 10:54:06 2007 @@ -512,3 +512,66 @@ return const define_const_primitives() +# ___________________________________________________________________________ +# Control Primitives + +PRIMITIVE_BLOCK_COPY = 80 +PRIMITIVE_VALUE = 81 +PRIMITIVE_VALUE_WITH_ARGS = 82 +PRIMITIVE_PERFORM = 83 +PRIMITIVE_PERFORM_WITH_ARGS = 84 +PRIMITIVE_SIGNAL = 85 +PRIMITIVE_WAIT = 86 +PRIMITIVE_RESUME = 87 +PRIMTIIVE_SUSPEND = 88 +PRIMTIIVE_FLUSH_CACHE = 89 + + at primitive(PRIMITIVE_BLOCK_COPY) + at stack(2) +def func(interpreter, receiver, argcount): + raise PrimitiveNotYetWrittenError() + + at primitive(PRIMITIVE_VALUE) + at stack(1) +def func(interpreter, receiver): + raise PrimitiveNotYetWrittenError() + + at primitive(PRIMITIVE_VALUE_WITH_ARGS) + at stack(2) +def func(interpreter, receiver): + raise PrimitiveNotYetWrittenError() + + at primitive(PRIMITIVE_PERFORM) + at stack(2) +def func(interpreter, receiver, selector): + raise PrimitiveNotYetWrittenError() + + at primitive(PRIMITIVE_PERFORM_WITH_ARGS) + at stack(3) +def func(interpreter, receiver, selector, arguments): + raise PrimitiveNotYetWrittenError() + + at primitive(PRIMITIVE_SIGNAL) + at stack(1) +def func(interpreter, receiver): + raise PrimitiveNotYetWrittenError() + + at primitive(PRIMITIVE_WAIT) + at stack(1) +def func(interpreter, receiver): + raise PrimitiveNotYetWrittenError() + + at primitive(PRIMITIVE_RESUME) + at stack(1) +def func(interpreter, receiver): + raise PrimitiveNotYetWrittenError() + + at primitive(PRIMTIIVE_SUSPEND) + at stack(1) +def func(interpreter, receiver): + raise PrimitiveNotYetWrittenError() + + at primitive(PRIMTIIVE_FLUSH_CACHE) + at stack(1) +def func(interpreter, receiver): + raise PrimitiveNotYetWrittenError() From niko at codespeak.net Thu Oct 25 11:33:21 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 25 Oct 2007 11:33:21 +0200 (CEST) Subject: [pypy-svn] r47901 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025093321.328098129@code0.codespeak.net> Author: niko Date: Thu Oct 25 11:33:20 2007 New Revision: 47901 Added: pypy/dist/pypy/lang/smalltalk/objtable.py - copied unchanged from r47900, pypy/dist/pypy/lang/smalltalk/fakeimage.py Removed: pypy/dist/pypy/lang/smalltalk/fakeimage.py Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: rename fakeimage to objtable Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 11:33:20 2007 @@ -1,6 +1,6 @@ import py from pypy.lang.smalltalk import model, constants, primitives -from pypy.lang.smalltalk import fakeimage +from pypy.lang.smalltalk import objtable class MissingBytecode(NotImplementedError): @@ -399,13 +399,13 @@ class Interpreter: - TRUE = fakeimage.w_true - FALSE = fakeimage.w_false - NIL = fakeimage.w_nil - MONE = fakeimage.w_mone - ZERO = fakeimage.w_zero - ONE = fakeimage.w_one - TWO = fakeimage.w_two + TRUE = objtable.w_true + FALSE = objtable.w_false + NIL = objtable.w_nil + MONE = objtable.w_mone + ZERO = objtable.w_zero + ONE = objtable.w_one + TWO = objtable.w_two def __init__(self): self.activeContext = None Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 11:33:20 2007 @@ -1,7 +1,7 @@ import operator from pypy.lang.smalltalk import model, mirror from pypy.lang.smalltalk import classtable -from pypy.lang.smalltalk import fakeimage +from pypy.lang.smalltalk import objtable from pypy.rlib import rarithmetic class PrimitiveFailedError(Exception): @@ -19,9 +19,9 @@ if isinstance(w_obj, model.W_PointersObject): return w_obj.getindexedvar(idx) elif isinstance(w_obj, model.W_WordsObject): - return fakeimage.wrap_int(w_obj.getword(idx)) + return objtable.wrap_int(w_obj.getword(idx)) elif isinstance(w_obj, model.W_BytesObject): - return fakeimage.wrap_int(w_obj.getbyte(idx)) + return objtable.wrap_int(w_obj.getbyte(idx)) raise PrimitiveFailedError() def assert_bounds(idx, minb, maxb): @@ -68,7 +68,7 @@ raise PrimitiveFailedError() if value < -1073741824: raise PrimitiveFailedError() - return fakeimage.wrap_int(value) + return objtable.wrap_int(value) ADD = 1 SUBTRACT = 2 @@ -189,7 +189,7 @@ [w_v1, w_v2] = res v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) - w_res = fakeimage.wrap_float(op(v1, v2)) + w_res = objtable.wrap_float(op(v1, v2)) return w_res prim_table[code] = func @@ -242,7 +242,7 @@ def func(stack): w_obj, idx = common_at(stack) byte = w_obj.getbyte(idx) - return fakeimage.CharacterTable[byte] + return objtable.CharacterTable[byte] @primitive(STRING_AT_PUT) @stack(3) @@ -250,7 +250,7 @@ w_obj, idx, w_val = common_at_put(stack) if w_val.getclassmirror() is not classtable.m_Character: raise PrimitiveFailedError() - w_obj.setbyte(idx, fakeimage.ord_w_char(w_val)) + w_obj.setbyte(idx, objtable.ord_w_char(w_val)) return w_val # ___________________________________________________________________________ @@ -464,7 +464,7 @@ v1 = unwrap_int(w_v1) v2 = unwrap_int(w_v2) res = op(v1, v2) - w_res = fakeimage.wrap_bool(res) + w_res = objtable.wrap_bool(res) return w_res for (code,op) in bool_ops.items(): @@ -475,7 +475,7 @@ v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) res = op(v1, v2) - w_res = fakeimage.wrap_bool(res) + w_res = objtable.wrap_bool(res) return w_res # ___________________________________________________________________________ @@ -498,13 +498,13 @@ def define_const_primitives(): for (code, const) in [ - (PUSH_TRUE, fakeimage.w_true), - (PUSH_FALSE, fakeimage.w_false), - (PUSH_NIL, fakeimage.w_nil), - (PUSH_MINUS_ONE, fakeimage.w_mone), - (PUSH_ZERO, fakeimage.w_zero), - (PUSH_ONE, fakeimage.w_one), - (PUSH_TWO, fakeimage.w_two), + (PUSH_TRUE, objtable.w_true), + (PUSH_FALSE, objtable.w_false), + (PUSH_NIL, objtable.w_nil), + (PUSH_MINUS_ONE, objtable.w_mone), + (PUSH_ZERO, objtable.w_zero), + (PUSH_ONE, objtable.w_one), + (PUSH_TWO, objtable.w_two), ]: @primitive(code) @stack(1) Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 11:33:20 2007 @@ -1,6 +1,6 @@ import py from pypy.lang.smalltalk import model -from pypy.lang.smalltalk import fakeimage +from pypy.lang.smalltalk import objtable from pypy.lang.smalltalk.mirror import mirrorcache from pypy.rlib import objectmodel @@ -222,7 +222,7 @@ self.owner = reader self.value = value self.size = -1 - self.w_object = fakeimage.wrap_int(value) + self.w_object = objtable.wrap_int(value) def initialize(self, chunk, reader): self.owner = reader Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Thu Oct 25 11:33:20 2007 @@ -1,6 +1,6 @@ import py from pypy.lang.smalltalk import model, interpreter, primitives, mirror -from pypy.lang.smalltalk.fakeimage import wrap_int +from pypy.lang.smalltalk.objtable import wrap_int import pypy.lang.smalltalk.classtable as ct mockclassmirror = ct.bootstrap_classmirror Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 11:33:20 2007 @@ -3,7 +3,7 @@ from pypy.lang.smalltalk import model, mirror from pypy.lang.smalltalk import interpreter from pypy.lang.smalltalk import classtable -from pypy.lang.smalltalk import fakeimage +from pypy.lang.smalltalk import objtable # Violates the guideline, but we use it A LOT to reference the primitive codes: import pypy.lang.smalltalk.primitives as p @@ -15,11 +15,11 @@ self.stack = stack def wrap(x): - if isinstance(x, int): return fakeimage.wrap_int(x) - if isinstance(x, float): return fakeimage.wrap_float(x) + if isinstance(x, int): return objtable.wrap_int(x) + if isinstance(x, float): return objtable.wrap_float(x) if isinstance(x, model.W_Object): return x - if isinstance(x, str) and len(x) == 1: return fakeimage.wrap_char(x) - if isinstance(x, str): return fakeimage.wrap_string(x) + if isinstance(x, str) and len(x) == 1: return objtable.wrap_char(x) + if isinstance(x, str): return objtable.wrap_string(x) if isinstance(x, mirror.ClassMirror): return x.w_self raise NotImplementedError @@ -170,11 +170,11 @@ assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i]) def test_object_at(): - w_v = prim(p.OBJECT_AT, ["q", fakeimage.CHARACTER_VALUE_INDEX]) + w_v = prim(p.OBJECT_AT, ["q", objtable.CHARACTER_VALUE_INDEX]) assert w_v.value == ord("q") def test_invalid_object_at(): - prim_fails(p.OBJECT_AT, ["q", fakeimage.CHARACTER_VALUE_INDEX+1]) + prim_fails(p.OBJECT_AT, ["q", objtable.CHARACTER_VALUE_INDEX+1]) def test_object_at_put(): w_obj = mockclassmirror(1).new() @@ -210,7 +210,7 @@ def test_inst_var_at(): # I am not entirely sure if this is what this primitive is # supposed to do, so the test may be bogus: - w_v = prim(p.INST_VAR_AT, ["q", fakeimage.CHARACTER_VALUE_INDEX]) + w_v = prim(p.INST_VAR_AT, ["q", objtable.CHARACTER_VALUE_INDEX]) assert w_v.value == ord("q") w_v = prim(p.INST_VAR_AT, ["abc", 1]) assert w_v.value == ord("b") @@ -225,30 +225,30 @@ def test_const_primitives(): for (code, const) in [ - (p.PUSH_TRUE, fakeimage.w_true), - (p.PUSH_FALSE, fakeimage.w_false), - (p.PUSH_NIL, fakeimage.w_nil), - (p.PUSH_MINUS_ONE, fakeimage.w_mone), - (p.PUSH_ZERO, fakeimage.w_zero), - (p.PUSH_ONE, fakeimage.w_one), - (p.PUSH_TWO, fakeimage.w_two), + (p.PUSH_TRUE, objtable.w_true), + (p.PUSH_FALSE, objtable.w_false), + (p.PUSH_NIL, objtable.w_nil), + (p.PUSH_MINUS_ONE, objtable.w_mone), + (p.PUSH_ZERO, objtable.w_zero), + (p.PUSH_ONE, objtable.w_one), + (p.PUSH_TWO, objtable.w_two), ]: - assert prim(code, [fakeimage.w_nil]) is const - assert prim(p.PUSH_SELF, [fakeimage.w_nil]) is fakeimage.w_nil + assert prim(code, [objtable.w_nil]) is const + assert prim(p.PUSH_SELF, [objtable.w_nil]) is objtable.w_nil assert prim(p.PUSH_SELF, ["a"]) is wrap("a") def test_boolean(): - assert prim(p.LESSTHAN, [1,2]) == fakeimage.w_true - assert prim(p.GREATERTHAN, [3,4]) == fakeimage.w_false - assert prim(p.LESSOREQUAL, [1,2]) == fakeimage.w_true - assert prim(p.GREATEROREQUAL, [3,4]) == fakeimage.w_false - assert prim(p.EQUAL, [2,2]) == fakeimage.w_true - assert prim(p.NOTEQUAL, [2,2]) == fakeimage.w_false + assert prim(p.LESSTHAN, [1,2]) == objtable.w_true + assert prim(p.GREATERTHAN, [3,4]) == objtable.w_false + assert prim(p.LESSOREQUAL, [1,2]) == objtable.w_true + assert prim(p.GREATEROREQUAL, [3,4]) == objtable.w_false + assert prim(p.EQUAL, [2,2]) == objtable.w_true + assert prim(p.NOTEQUAL, [2,2]) == objtable.w_false def test_float_boolean(): - assert prim(p.FLOAT_LESSTHAN, [1.0,2.0]) == fakeimage.w_true - assert prim(p.FLOAT_GREATERTHAN, [3.0,4.0]) == fakeimage.w_false - assert prim(p.FLOAT_LESSOREQUAL, [1.3,2.6]) == fakeimage.w_true - assert prim(p.FLOAT_GREATEROREQUAL, [3.5,4.9]) == fakeimage.w_false - assert prim(p.FLOAT_EQUAL, [2.2,2.2]) == fakeimage.w_true - assert prim(p.FLOAT_NOTEQUAL, [2.2,2.2]) == fakeimage.w_false + assert prim(p.FLOAT_LESSTHAN, [1.0,2.0]) == objtable.w_true + assert prim(p.FLOAT_GREATERTHAN, [3.0,4.0]) == objtable.w_false + assert prim(p.FLOAT_LESSOREQUAL, [1.3,2.6]) == objtable.w_true + assert prim(p.FLOAT_GREATEROREQUAL, [3.5,4.9]) == objtable.w_false + assert prim(p.FLOAT_EQUAL, [2.2,2.2]) == objtable.w_true + assert prim(p.FLOAT_NOTEQUAL, [2.2,2.2]) == objtable.w_false From oscar at codespeak.net Thu Oct 25 11:43:33 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Thu, 25 Oct 2007 11:43:33 +0200 (CEST) Subject: [pypy-svn] r47902 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025094333.0DAAB8129@code0.codespeak.net> Author: oscar Date: Thu Oct 25 11:43:32 2007 New Revision: 47902 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: PRIMTIIVE_ -> PRIMITIVE_ Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 11:43:32 2007 @@ -523,8 +523,8 @@ PRIMITIVE_SIGNAL = 85 PRIMITIVE_WAIT = 86 PRIMITIVE_RESUME = 87 -PRIMTIIVE_SUSPEND = 88 -PRIMTIIVE_FLUSH_CACHE = 89 +PRIMITIVE_SUSPEND = 88 +PRIMITIVE_FLUSH_CACHE = 89 @primitive(PRIMITIVE_BLOCK_COPY) @stack(2) @@ -566,12 +566,12 @@ def func(interpreter, receiver): raise PrimitiveNotYetWrittenError() - at primitive(PRIMTIIVE_SUSPEND) + at primitive(PRIMITIVE_SUSPEND) @stack(1) def func(interpreter, receiver): raise PrimitiveNotYetWrittenError() - at primitive(PRIMTIIVE_FLUSH_CACHE) + at primitive(PRIMITIVE_FLUSH_CACHE) @stack(1) def func(interpreter, receiver): raise PrimitiveNotYetWrittenError() From niko at codespeak.net Thu Oct 25 11:48:26 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 25 Oct 2007 11:48:26 +0200 (CEST) Subject: [pypy-svn] r47903 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025094826.BE0E08129@code0.codespeak.net> Author: niko Date: Thu Oct 25 11:48:26 2007 New Revision: 47903 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: (niko, lukas) inline tuple unpacking into primitive argument signature Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 11:48:26 2007 @@ -93,8 +93,7 @@ for (code,op) in math_ops.items(): @primitive(code) @stack(2) - def func(stack, op=op): # n.b. capture op value - [w_receiver, w_argument] = stack + def func((w_receiver, w_argument), op=op): # n.b. capture op value receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) try: @@ -106,8 +105,7 @@ # #/ -- return the result of a division, only succeed if the division is exact @primitive(DIVIDE) @stack(2) -def func(stack): - [w_receiver, w_argument] = stack +def func((w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) if argument == 0: @@ -119,8 +117,7 @@ # #\\ -- return the remainder of a division @primitive(MOD) @stack(2) -def func(stack): - [w_receiver, w_argument] = stack +def func((w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) if argument == 0: @@ -130,8 +127,7 @@ # #// -- return the result of a division, rounded towards negative zero @primitive(DIV) @stack(2) -def func(stack): - [w_receiver, w_argument] = stack +def func((w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) if argument == 0: @@ -141,8 +137,7 @@ # #// -- return the result of a division, rounded towards negative infinity @primitive(QUO) @stack(2) -def func(stack): - [w_receiver, w_argument] = stack +def func((w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) if argument == 0: @@ -152,8 +147,7 @@ # #bitShift: -- return the shifted value @primitive(BIT_SHIFT) @stack(2) -def func(stack): - [w_receiver, w_argument] = stack +def func((w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) @@ -185,8 +179,7 @@ } for (code,op) in math_ops.items(): @stack(2) - def func(res, op=op): # n.b. capture op value - [w_v1, w_v2] = res + def func((w_v1, w_v2), op=op): # n.b. capture op value v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) w_res = objtable.wrap_float(op(v1, v2)) @@ -202,15 +195,13 @@ STRING_AT = 63 STRING_AT_PUT = 64 -def common_at(stack): - [w_obj, w_idx] = stack +def common_at((w_obj, w_idx)): idx = unwrap_int(w_idx) # XXX should be idx-1, probably assert_valid_index(idx, w_obj) return w_obj, idx -def common_at_put(stack): - [w_obj, w_idx, w_val] = stack +def common_at_put((w_obj, w_idx, w_val)): idx = unwrap_int(w_idx) # XXX should be idx-1, probably assert_valid_index(idx, w_obj) @@ -219,20 +210,19 @@ @primitive(AT) @stack(2) def func(stack): - w_obj, idx = common_at(stack) + [w_obj, idx] = common_at(stack) return w_obj.fetch(idx) @primitive(AT_PUT) @stack(3) def func(stack): - w_obj, idx, w_val = common_at_put(stack) + [w_obj, idx, w_val] = common_at_put(stack) w_obj.store(idx, w_val) return w_val @primitive(SIZE) @stack(1) -def func(stack): - [w_obj] = stack +def func((w_obj,)): if not w_obj.getclassmirror().isvariable(): raise PrimitiveFailedError() return w_obj.size() @@ -271,8 +261,7 @@ @primitive(OBJECT_AT) @stack(2) -def func(stack): - [w_rcvr, w_idx] = stack +def func((w_rcvr, w_idx)): idx = unwrap_int(w_idx) # XXX should be idx-1, probably assert_bounds(idx, 0, w_rcvr.getclassmirror().instance_size) @@ -280,8 +269,7 @@ @primitive(OBJECT_AT_PUT) @stack(3) -def func(stack): - [w_rcvr, w_idx, w_val] = stack +def func((w_rcvr, w_idx, w_val)): idx = unwrap_int(w_idx) # XXX should be idx-1, probably assert_bounds(idx, 0, w_rcvr.getclassmirror().instance_size) @@ -290,8 +278,7 @@ @primitive(NEW) @stack(1) -def func(stack): - [w_cls] = stack +def func((w_cls,)): m_cls = mirror.mirrorcache.getmirror(w_cls) if m_cls.isvariable(): raise PrimitiveFailedError() @@ -299,8 +286,7 @@ @primitive(NEW_WITH_ARG) @stack(2) -def func(stack): - [w_cls, w_size] = stack +def func((w_cls, w_size)): m_cls = mirror.mirrorcache.getmirror(w_cls) if not m_cls.isvariable(): raise PrimitiveFailedError() @@ -313,10 +299,9 @@ @primitive(INST_VAR_AT) @stack(2) -def func(stack): +def func((w_rcvr, w_idx)): # I *think* this is the correct behavior, but I'm not quite sure. # Might be restricted to fixed length fields? - [w_rcvr, w_idx] = stack idx = unwrap_int(w_idx) m_cls = w_rcvr.getclassmirror() if idx < 0: @@ -334,8 +319,7 @@ @primitive(AS_OOP) @stack(1) -def func(stack): - [w_rcvr] = stack +def func((w_rcvr,)): if isinstance(w_rcvr, model.W_SmallInteger): raise PrimitiveFailedError() return w_rcvr.w_hash @@ -349,19 +333,17 @@ @primitive(SOME_INSTANCE) @stack(1) -def func(stack): +def func((w_class,)): # This primitive returns some instance of the class on the stack. # Not sure quite how to do this; maintain a weak list of all # existing instances or something? - [w_class] = stack raise PrimitiveNotYetWrittenError() @primitive(NEXT_INSTANCE) @stack(1) -def func(stack): +def func((w_obj,)): # This primitive is used to iterate through all instances of a class: # it returns the "next" instance after w_obj. - [w_obj] = stack raise PrimitiveNotYetWrittenError() @primitive(NEW_METHOD) @@ -380,14 +362,12 @@ @primitive(EQUIVALENT) @stack(2) -def func(stack): - [w_arg, w_rcvr] = stack +def func((w_arg, w_rcvr)): return w_arg == w_rcvr @primitive(EQUIVALENT) @stack(1) -def func(stack): - [w_obj] = stack +def func((w_obj,)): return w_obj.w_class @primitive(BYTES_LEFT) @@ -404,8 +384,7 @@ @primitive(CHANGE_CLASS) @stack(2) -def func(stack): - [w_arg, w_rcvr] = stack +def func((w_arg, w_rcvr)): w_arg_class = w_arg.w_class w_rcvr_class = w_rcvr.w_class @@ -459,8 +438,7 @@ for (code,op) in bool_ops.items(): @primitive(code) @stack(2) - def func(stack, op=op): # n.b. capture op value - [w_v1, w_v2] = stack + def func((w_v1, w_v2), op=op): # n.b. capture op value v1 = unwrap_int(w_v1) v2 = unwrap_int(w_v2) res = op(v1, v2) @@ -470,8 +448,7 @@ for (code,op) in bool_ops.items(): @primitive(code+_FLOAT_OFFSET) @stack(2) - def func(stack, op=op): # n.b. capture op value - [w_v1, w_v2] = stack + def func((w_v1, w_v2), op=op): # n.b. capture op value v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) res = op(v1, v2) From cfbolz at codespeak.net Thu Oct 25 12:05:31 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 25 Oct 2007 12:05:31 +0200 (CEST) Subject: [pypy-svn] r47904 - pypy/extradoc/sprintinfo/bern2007 Message-ID: <20071025100531.9AEBD8119@code0.codespeak.net> Author: cfbolz Date: Thu Oct 25 12:05:30 2007 New Revision: 47904 Modified: pypy/extradoc/sprintinfo/bern2007/planning.txt Log: some updates Modified: pypy/extradoc/sprintinfo/bern2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/bern2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/bern2007/planning.txt Thu Oct 25 12:05:30 2007 @@ -13,11 +13,11 @@ Task ideas ---------- -- give everybody an account (arigo) +- give everybody an account (arigo) DONE -- present coding conventions (cfbolz) +- present coding conventions (cfbolz) DONE -- present the "mirror" ("shadow"?) (cfbolz, arigo) +- present the "shadow" (cfbolz, arigo) DONE Afterwards: ----------- From fijal at codespeak.net Thu Oct 25 12:10:33 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 25 Oct 2007 12:10:33 +0200 (CEST) Subject: [pypy-svn] r47905 - pypy/dist/pypy/config Message-ID: <20071025101033.94B55811E@code0.codespeak.net> Author: fijal Date: Thu Oct 25 12:10:33 2007 New Revision: 47905 Modified: pypy/dist/pypy/config/pypyoption.py Log: Remove zipimport from allworkingmodules as it has some bootstrapping problems. Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Thu Oct 25 12:10:33 2007 @@ -24,7 +24,7 @@ working_modules.update(dict.fromkeys( ["_socket", "unicodedata", "mmap", "fcntl", "rctime", "select", "crypt", "signal", "dyngram", "readline", "termios", "zlib", - "struct", "md5", "sha", "bz2", 'zipimport' + "struct", "md5", "sha", "bz2" ] )) From niko at codespeak.net Thu Oct 25 12:11:56 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 25 Oct 2007 12:11:56 +0200 (CEST) Subject: [pypy-svn] r47906 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025101156.17AAB811E@code0.codespeak.net> Author: niko Date: Thu Oct 25 12:11:55 2007 New Revision: 47906 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (niko, lukas, oscar) update primitives to take the interpreter instead of the activeContext Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 12:11:55 2007 @@ -139,7 +139,7 @@ if method.primitive: func = primitives.prim_table[method.primitive] try: - w_result = func(self) + w_result = func(interp) except primitives.PrimitiveFailedError: pass # ignore this error and fall back to the Smalltalk version else: @@ -296,7 +296,7 @@ def callPrimitiveAndPush(self, primitive, selector, argcount, interp): try: - self.push(primitives.prim_table[primitive](self)) + self.push(primitives.prim_table[primitive](interp)) except primitives.PrimitiveFailedError: self._sendSelfSelector(selector, argcount, interp) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 12:11:55 2007 @@ -47,9 +47,10 @@ def stack(n): def decorator(wrapped): - def result(frame): + def result(interp): + frame = interp.activeContext items = frame.stack[len(frame.stack)-n:] - res = wrapped(items) + res = wrapped(interp, items) frame.pop_n(n) # only if no exception occurs! return res return result @@ -93,7 +94,7 @@ for (code,op) in math_ops.items(): @primitive(code) @stack(2) - def func((w_receiver, w_argument), op=op): # n.b. capture op value + def func(interp, (w_receiver, w_argument), op=op): # n.b. capture op value receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) try: @@ -105,7 +106,7 @@ # #/ -- return the result of a division, only succeed if the division is exact @primitive(DIVIDE) @stack(2) -def func((w_receiver, w_argument)): +def func(interp, (w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) if argument == 0: @@ -117,7 +118,7 @@ # #\\ -- return the remainder of a division @primitive(MOD) @stack(2) -def func((w_receiver, w_argument)): +def func(interp, (w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) if argument == 0: @@ -127,7 +128,7 @@ # #// -- return the result of a division, rounded towards negative zero @primitive(DIV) @stack(2) -def func((w_receiver, w_argument)): +def func(interp, (w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) if argument == 0: @@ -137,7 +138,7 @@ # #// -- return the result of a division, rounded towards negative infinity @primitive(QUO) @stack(2) -def func((w_receiver, w_argument)): +def func(interp, (w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) if argument == 0: @@ -147,7 +148,7 @@ # #bitShift: -- return the shifted value @primitive(BIT_SHIFT) @stack(2) -def func((w_receiver, w_argument)): +def func(interp, (w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) @@ -179,7 +180,7 @@ } for (code,op) in math_ops.items(): @stack(2) - def func((w_v1, w_v2), op=op): # n.b. capture op value + def func(interp, (w_v1, w_v2), op=op): # n.b. capture op value v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) w_res = objtable.wrap_float(op(v1, v2)) @@ -209,34 +210,34 @@ @primitive(AT) @stack(2) -def func(stack): +def func(interp, stack): [w_obj, idx] = common_at(stack) return w_obj.fetch(idx) @primitive(AT_PUT) @stack(3) -def func(stack): +def func(interp, stack): [w_obj, idx, w_val] = common_at_put(stack) w_obj.store(idx, w_val) return w_val @primitive(SIZE) @stack(1) -def func((w_obj,)): +def func(interp, (w_obj,)): if not w_obj.getclassmirror().isvariable(): raise PrimitiveFailedError() return w_obj.size() @primitive(STRING_AT) @stack(2) -def func(stack): +def func(interp, stack): w_obj, idx = common_at(stack) byte = w_obj.getbyte(idx) return objtable.CharacterTable[byte] @primitive(STRING_AT_PUT) @stack(3) -def func(stack): +def func(interp, stack): w_obj, idx, w_val = common_at_put(stack) if w_val.getclassmirror() is not classtable.m_Character: raise PrimitiveFailedError() @@ -261,7 +262,7 @@ @primitive(OBJECT_AT) @stack(2) -def func((w_rcvr, w_idx)): +def func(interp, (w_rcvr, w_idx)): idx = unwrap_int(w_idx) # XXX should be idx-1, probably assert_bounds(idx, 0, w_rcvr.getclassmirror().instance_size) @@ -269,7 +270,7 @@ @primitive(OBJECT_AT_PUT) @stack(3) -def func((w_rcvr, w_idx, w_val)): +def func(interp, (w_rcvr, w_idx, w_val)): idx = unwrap_int(w_idx) # XXX should be idx-1, probably assert_bounds(idx, 0, w_rcvr.getclassmirror().instance_size) @@ -278,7 +279,7 @@ @primitive(NEW) @stack(1) -def func((w_cls,)): +def func(interp, (w_cls,)): m_cls = mirror.mirrorcache.getmirror(w_cls) if m_cls.isvariable(): raise PrimitiveFailedError() @@ -286,7 +287,7 @@ @primitive(NEW_WITH_ARG) @stack(2) -def func((w_cls, w_size)): +def func(interp, (w_cls, w_size)): m_cls = mirror.mirrorcache.getmirror(w_cls) if not m_cls.isvariable(): raise PrimitiveFailedError() @@ -294,12 +295,12 @@ return m_cls.new(size) @primitive(ARRAY_BECOME_ONE_WAY) -def func(frame): +def func(interp): raise PrimitiveNotYetWrittenError @primitive(INST_VAR_AT) @stack(2) -def func((w_rcvr, w_idx)): +def func(interp, (w_rcvr, w_idx)): # I *think* this is the correct behavior, but I'm not quite sure. # Might be restricted to fixed length fields? idx = unwrap_int(w_idx) @@ -314,26 +315,26 @@ raise PrimitiveFailedError() @primitive(INST_VAR_AT_PUT) -def func(frame): +def func(interp): raise PrimitiveNotYetWrittenError() @primitive(AS_OOP) @stack(1) -def func((w_rcvr,)): +def func(interp, (w_rcvr,)): if isinstance(w_rcvr, model.W_SmallInteger): raise PrimitiveFailedError() return w_rcvr.w_hash @primitive(STORE_STACKP) @stack(2) -def func(stack): +def func(interp, stack): # This primitive seems to resize the stack. I don't think this is # really relevant in our implementation. raise PrimitiveNotYetWrittenError() @primitive(SOME_INSTANCE) @stack(1) -def func((w_class,)): +def func(interp, (w_class,)): # This primitive returns some instance of the class on the stack. # Not sure quite how to do this; maintain a weak list of all # existing instances or something? @@ -341,13 +342,13 @@ @primitive(NEXT_INSTANCE) @stack(1) -def func((w_obj,)): +def func(interp, (w_obj,)): # This primitive is used to iterate through all instances of a class: # it returns the "next" instance after w_obj. raise PrimitiveNotYetWrittenError() @primitive(NEW_METHOD) -def func(frame): +def func(interp): raise PrimitiveNotYetWrittenError() # ___________________________________________________________________________ @@ -362,29 +363,29 @@ @primitive(EQUIVALENT) @stack(2) -def func((w_arg, w_rcvr)): +def func(interp, (w_arg, w_rcvr)): return w_arg == w_rcvr @primitive(EQUIVALENT) @stack(1) -def func((w_obj,)): +def func(interp, (w_obj,)): return w_obj.w_class @primitive(BYTES_LEFT) -def func(frame): +def func(interp): raise PrimitiveNotYetWrittenError() @primitive(QUIT) -def func(frame): +def func(interp): raise PrimitiveNotYetWrittenError() @primitive(EXIT_TO_DEBUGGER) -def func(frame): +def func(interp): raise PrimitiveNotYetWrittenError() @primitive(CHANGE_CLASS) @stack(2) -def func((w_arg, w_rcvr)): +def func(interp, (w_arg, w_rcvr)): w_arg_class = w_arg.w_class w_rcvr_class = w_rcvr.w_class @@ -438,7 +439,7 @@ for (code,op) in bool_ops.items(): @primitive(code) @stack(2) - def func((w_v1, w_v2), op=op): # n.b. capture op value + def func(interp, (w_v1, w_v2), op=op): # n.b. capture op value v1 = unwrap_int(w_v1) v2 = unwrap_int(w_v2) res = op(v1, v2) @@ -448,7 +449,7 @@ for (code,op) in bool_ops.items(): @primitive(code+_FLOAT_OFFSET) @stack(2) - def func((w_v1, w_v2), op=op): # n.b. capture op value + def func(interp, (w_v1, w_v2), op=op): # n.b. capture op value v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) res = op(v1, v2) @@ -469,7 +470,7 @@ @primitive(PUSH_SELF) @stack(1) -def func(stack): +def func(interp, stack): [w_self] = stack return w_self @@ -485,7 +486,7 @@ ]: @primitive(code) @stack(1) - def func(stack, const=const): # n.b.: capture const + def func(interp, stack, const=const): # n.b.: capture const return const define_const_primitives() @@ -505,50 +506,50 @@ @primitive(PRIMITIVE_BLOCK_COPY) @stack(2) -def func(interpreter, receiver, argcount): +def func(interp, (w_rcvr, w_argcnt)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_VALUE) @stack(1) -def func(interpreter, receiver): +def func(interp, (w_rcvr,)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_VALUE_WITH_ARGS) @stack(2) -def func(interpreter, receiver): +def func(interp, (w_rcvr, w_args)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_PERFORM) @stack(2) -def func(interpreter, receiver, selector): +def func(interp, (w_rcvr, w_sel)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_PERFORM_WITH_ARGS) @stack(3) -def func(interpreter, receiver, selector, arguments): +def func(interp, (w_rcvr, w_sel, w_args)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_SIGNAL) @stack(1) -def func(interpreter, receiver): +def func(interp, (w_rcvr,)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_WAIT) @stack(1) -def func(interpreter, receiver): +def func(interp, (w_rcvr,)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_RESUME) @stack(1) -def func(interpreter, receiver): +def func(interp, (w_rcvr,)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_SUSPEND) @stack(1) -def func(interpreter, receiver): +def func(interp, (w_rcvr,)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_FLUSH_CACHE) @stack(1) -def func(interpreter, receiver): +def func(interp, (w_rcvr,)): raise PrimitiveNotYetWrittenError() Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 12:11:55 2007 @@ -25,22 +25,25 @@ def mock(stack): mapped_stack = [wrap(x) for x in stack] - return MockFrame(mapped_stack) + frame = MockFrame(mapped_stack) + interp = interpreter.Interpreter() + interp.activeContext = frame + return interp def prim(code, stack): - stack_w = mock(stack) - res = prim_table[code](stack_w) - assert not len(stack_w.stack) # only pass as many arguments as it uses + interp = mock(stack) + res = prim_table[code](interp) + assert not len(interp.activeContext.stack) # check that args are consumed return res def prim_fails(code, stack): - stack_w = mock(stack) - orig_stack = list(stack_w.stack) + interp = mock(stack) + orig_stack = list(interp.activeContext.stack) try: - prim_table[code](stack_w) + prim_table[code](interp) py.test.fail("Expected PrimitiveFailedError") except PrimitiveFailedError: - assert stack_w.stack == orig_stack + assert interp.activeContext.stack == orig_stack # smallinteger tests def test_small_int_add(): From niko at codespeak.net Thu Oct 25 12:43:44 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 25 Oct 2007 12:43:44 +0200 (CEST) Subject: [pypy-svn] r47907 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025104344.552BD8120@code0.codespeak.net> Author: niko Date: Thu Oct 25 12:43:44 2007 New Revision: 47907 Modified: pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (niko, lukas) Implemented BLOCK_COPY primitive, and modified primitives to take an Args instance Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Thu Oct 25 12:43:44 2007 @@ -1,6 +1,8 @@ # ___________________________________________________________________________ # Slot Names +# +# n.b. Off by one from SQUEAK! CHARACTER_VALUE_INDEX = 0 # Page 630 of the blue book @@ -20,6 +22,11 @@ ASSOCIATION_KEY_INDEX = 0 ASSOCIATION_VALUE_INDEX = 1 +BLOCK_CONTEXT_INSTRUCTION_POINTER_INDEX = 0 +BLOCK_CONTEXT_BLOCK_ARGUMENT_COUNT_INDEX = 2 +BLOCK_CONTEXT_INITIAL_IP_INDEX = 3 +BLOCK_CONTEXT_HOME_INDEX = 4 + # ----- special objects indices ------- SO_NIL = 0 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 12:43:44 2007 @@ -139,7 +139,7 @@ if method.primitive: func = primitives.prim_table[method.primitive] try: - w_result = func(interp) + w_result = func(primitives.Args(interp, argcount)) except primitives.PrimitiveFailedError: pass # ignore this error and fall back to the Smalltalk version else: @@ -296,7 +296,8 @@ def callPrimitiveAndPush(self, primitive, selector, argcount, interp): try: - self.push(primitives.prim_table[primitive](interp)) + args = primitives.Args(interp, argcount) + self.push(primitives.prim_table[primitive](args)) except primitives.PrimitiveFailedError: self._sendSelfSelector(selector, argcount, interp) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 12:43:44 2007 @@ -33,12 +33,17 @@ # ___________________________________________________________________________ # Primitive table: it is filled in at initialization time with the -# primitive functions. Each primitive function takes a single argument, -# the frame (a W_ContextFrame object); the function either completes, and -# returns a result, or throws a PrimitiveFailedError. +# primitive functions. Each primitive function takes a single +# argument, an instance of the Args class below; the function either +# completes, and returns a result, or throws a PrimitiveFailedError. prim_table = [None] * 576 # Squeak has primitives all the way up to 575 +class Args: + def __init__(self, interp, argument_count): + self.interp = interp + self.argument_count = argument_count + def primitive(code): def decorator(func): prim_table[code] = func @@ -47,10 +52,10 @@ def stack(n): def decorator(wrapped): - def result(interp): - frame = interp.activeContext + def result(args): + frame = args.interp.activeContext items = frame.stack[len(frame.stack)-n:] - res = wrapped(interp, items) + res = wrapped(args, items) frame.pop_n(n) # only if no exception occurs! return res return result @@ -94,7 +99,7 @@ for (code,op) in math_ops.items(): @primitive(code) @stack(2) - def func(interp, (w_receiver, w_argument), op=op): # n.b. capture op value + def func(args, (w_receiver, w_argument), op=op): # n.b. capture op value receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) try: @@ -106,7 +111,7 @@ # #/ -- return the result of a division, only succeed if the division is exact @primitive(DIVIDE) @stack(2) -def func(interp, (w_receiver, w_argument)): +def func(args, (w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) if argument == 0: @@ -118,7 +123,7 @@ # #\\ -- return the remainder of a division @primitive(MOD) @stack(2) -def func(interp, (w_receiver, w_argument)): +def func(args, (w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) if argument == 0: @@ -128,7 +133,7 @@ # #// -- return the result of a division, rounded towards negative zero @primitive(DIV) @stack(2) -def func(interp, (w_receiver, w_argument)): +def func(args, (w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) if argument == 0: @@ -138,7 +143,7 @@ # #// -- return the result of a division, rounded towards negative infinity @primitive(QUO) @stack(2) -def func(interp, (w_receiver, w_argument)): +def func(args, (w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) if argument == 0: @@ -148,7 +153,7 @@ # #bitShift: -- return the shifted value @primitive(BIT_SHIFT) @stack(2) -def func(interp, (w_receiver, w_argument)): +def func(args, (w_receiver, w_argument)): receiver = unwrap_int(w_receiver) argument = unwrap_int(w_argument) @@ -180,7 +185,7 @@ } for (code,op) in math_ops.items(): @stack(2) - def func(interp, (w_v1, w_v2), op=op): # n.b. capture op value + def func(args, (w_v1, w_v2), op=op): # n.b. capture op value v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) w_res = objtable.wrap_float(op(v1, v2)) @@ -210,34 +215,34 @@ @primitive(AT) @stack(2) -def func(interp, stack): +def func(args, stack): [w_obj, idx] = common_at(stack) return w_obj.fetch(idx) @primitive(AT_PUT) @stack(3) -def func(interp, stack): +def func(args, stack): [w_obj, idx, w_val] = common_at_put(stack) w_obj.store(idx, w_val) return w_val @primitive(SIZE) @stack(1) -def func(interp, (w_obj,)): +def func(args, (w_obj,)): if not w_obj.getclassmirror().isvariable(): raise PrimitiveFailedError() return w_obj.size() @primitive(STRING_AT) @stack(2) -def func(interp, stack): +def func(args, stack): w_obj, idx = common_at(stack) byte = w_obj.getbyte(idx) return objtable.CharacterTable[byte] @primitive(STRING_AT_PUT) @stack(3) -def func(interp, stack): +def func(args, stack): w_obj, idx, w_val = common_at_put(stack) if w_val.getclassmirror() is not classtable.m_Character: raise PrimitiveFailedError() @@ -262,7 +267,7 @@ @primitive(OBJECT_AT) @stack(2) -def func(interp, (w_rcvr, w_idx)): +def func(args, (w_rcvr, w_idx)): idx = unwrap_int(w_idx) # XXX should be idx-1, probably assert_bounds(idx, 0, w_rcvr.getclassmirror().instance_size) @@ -270,7 +275,7 @@ @primitive(OBJECT_AT_PUT) @stack(3) -def func(interp, (w_rcvr, w_idx, w_val)): +def func(args, (w_rcvr, w_idx, w_val)): idx = unwrap_int(w_idx) # XXX should be idx-1, probably assert_bounds(idx, 0, w_rcvr.getclassmirror().instance_size) @@ -279,7 +284,7 @@ @primitive(NEW) @stack(1) -def func(interp, (w_cls,)): +def func(args, (w_cls,)): m_cls = mirror.mirrorcache.getmirror(w_cls) if m_cls.isvariable(): raise PrimitiveFailedError() @@ -287,7 +292,7 @@ @primitive(NEW_WITH_ARG) @stack(2) -def func(interp, (w_cls, w_size)): +def func(args, (w_cls, w_size)): m_cls = mirror.mirrorcache.getmirror(w_cls) if not m_cls.isvariable(): raise PrimitiveFailedError() @@ -295,12 +300,12 @@ return m_cls.new(size) @primitive(ARRAY_BECOME_ONE_WAY) -def func(interp): +def func(args): raise PrimitiveNotYetWrittenError @primitive(INST_VAR_AT) @stack(2) -def func(interp, (w_rcvr, w_idx)): +def func(args, (w_rcvr, w_idx)): # I *think* this is the correct behavior, but I'm not quite sure. # Might be restricted to fixed length fields? idx = unwrap_int(w_idx) @@ -315,26 +320,26 @@ raise PrimitiveFailedError() @primitive(INST_VAR_AT_PUT) -def func(interp): +def func(args): raise PrimitiveNotYetWrittenError() @primitive(AS_OOP) @stack(1) -def func(interp, (w_rcvr,)): +def func(args, (w_rcvr,)): if isinstance(w_rcvr, model.W_SmallInteger): raise PrimitiveFailedError() return w_rcvr.w_hash @primitive(STORE_STACKP) @stack(2) -def func(interp, stack): +def func(args, stack): # This primitive seems to resize the stack. I don't think this is # really relevant in our implementation. raise PrimitiveNotYetWrittenError() @primitive(SOME_INSTANCE) @stack(1) -def func(interp, (w_class,)): +def func(args, (w_class,)): # This primitive returns some instance of the class on the stack. # Not sure quite how to do this; maintain a weak list of all # existing instances or something? @@ -342,13 +347,13 @@ @primitive(NEXT_INSTANCE) @stack(1) -def func(interp, (w_obj,)): +def func(args, (w_obj,)): # This primitive is used to iterate through all instances of a class: # it returns the "next" instance after w_obj. raise PrimitiveNotYetWrittenError() @primitive(NEW_METHOD) -def func(interp): +def func(args): raise PrimitiveNotYetWrittenError() # ___________________________________________________________________________ @@ -363,29 +368,29 @@ @primitive(EQUIVALENT) @stack(2) -def func(interp, (w_arg, w_rcvr)): +def func(args, (w_arg, w_rcvr)): return w_arg == w_rcvr @primitive(EQUIVALENT) @stack(1) -def func(interp, (w_obj,)): +def func(args, (w_obj,)): return w_obj.w_class @primitive(BYTES_LEFT) -def func(interp): +def func(args): raise PrimitiveNotYetWrittenError() @primitive(QUIT) -def func(interp): +def func(args): raise PrimitiveNotYetWrittenError() @primitive(EXIT_TO_DEBUGGER) -def func(interp): +def func(args): raise PrimitiveNotYetWrittenError() @primitive(CHANGE_CLASS) @stack(2) -def func(interp, (w_arg, w_rcvr)): +def func(args, (w_arg, w_rcvr)): w_arg_class = w_arg.w_class w_rcvr_class = w_rcvr.w_class @@ -439,7 +444,7 @@ for (code,op) in bool_ops.items(): @primitive(code) @stack(2) - def func(interp, (w_v1, w_v2), op=op): # n.b. capture op value + def func(args, (w_v1, w_v2), op=op): # n.b. capture op value v1 = unwrap_int(w_v1) v2 = unwrap_int(w_v2) res = op(v1, v2) @@ -449,7 +454,7 @@ for (code,op) in bool_ops.items(): @primitive(code+_FLOAT_OFFSET) @stack(2) - def func(interp, (w_v1, w_v2), op=op): # n.b. capture op value + def func(args, (w_v1, w_v2), op=op): # n.b. capture op value v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) res = op(v1, v2) @@ -470,7 +475,7 @@ @primitive(PUSH_SELF) @stack(1) -def func(interp, stack): +def func(args, stack): [w_self] = stack return w_self @@ -486,7 +491,7 @@ ]: @primitive(code) @stack(1) - def func(interp, stack, const=const): # n.b.: capture const + def func(args, stack, const=const): # n.b.: capture const return const define_const_primitives() @@ -506,50 +511,71 @@ @primitive(PRIMITIVE_BLOCK_COPY) @stack(2) -def func(interp, (w_rcvr, w_argcnt)): - raise PrimitiveNotYetWrittenError() +def func(args, (w_argcnt, w_context)): + frame = args.interp.activeContext + + # From B.B.: If receiver is a MethodContext, then it becomes + # the new BlockContext's home context. Otherwise, the home + # context of the receiver is used for the new BlockContext. + if isinstance(w_context, classtable.w_BlockContext): + w_method_context = w_context.fetch(BLOCK_CONTEXT_HOME_INDEX) + else: + w_method_context = w_context + + # The block bytecodes are stored inline: so we skip past the + # byteodes to invoke this primitive to find them (hence +3) + w_new_context = classtable.w_BlockContext.new() + initialip = frame.pc + 3 + + # Initialize various fields. + w_new_context.store(BLOCK_CONTEXT_INITIAL_IP_INDEX, initialip) + w_new_context.store(BLOCK_CONTEXT_INSTRUCTION_POINTER_INDEX, initialip) + w_new_context.store(BLOCK_CONTEXT_BLOCK_ARGUMENT_COUNT_INDEX, w_argcnt) + w_new_context.store(BLOCK_CONTEXT_HOME_INDEX, w_method_context) + return w_new_context @primitive(PRIMITIVE_VALUE) - at stack(1) -def func(interp, (w_rcvr,)): +def func(args): + #w_rcvr = args.interp.activeContext.peek(0) + #w_argcnt = w_rcvr.fetch(BLOCK_CONTEXT_BLOCK_ARGUMENT_COUNT_INDEX) raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_VALUE_WITH_ARGS) @stack(2) -def func(interp, (w_rcvr, w_args)): +def func(args, (w_rcvr, w_args)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_PERFORM) @stack(2) -def func(interp, (w_rcvr, w_sel)): +def func(args, (w_rcvr, w_sel)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_PERFORM_WITH_ARGS) @stack(3) -def func(interp, (w_rcvr, w_sel, w_args)): +def func(args, (w_rcvr, w_sel, w_args)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_SIGNAL) @stack(1) -def func(interp, (w_rcvr,)): +def func(args, (w_rcvr,)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_WAIT) @stack(1) -def func(interp, (w_rcvr,)): +def func(args, (w_rcvr,)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_RESUME) @stack(1) -def func(interp, (w_rcvr,)): +def func(args, (w_rcvr,)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_SUSPEND) @stack(1) -def func(interp, (w_rcvr,)): +def func(args, (w_rcvr,)): raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_FLUSH_CACHE) @stack(1) -def func(interp, (w_rcvr,)): +def func(args, (w_rcvr,)): raise PrimitiveNotYetWrittenError() Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 12:43:44 2007 @@ -28,22 +28,22 @@ frame = MockFrame(mapped_stack) interp = interpreter.Interpreter() interp.activeContext = frame - return interp + return p.Args(interp, len(stack)) def prim(code, stack): - interp = mock(stack) - res = prim_table[code](interp) - assert not len(interp.activeContext.stack) # check that args are consumed + args = mock(stack) + res = prim_table[code](args) + assert not len(args.interp.activeContext.stack) # check args are consumed return res def prim_fails(code, stack): - interp = mock(stack) - orig_stack = list(interp.activeContext.stack) + args = mock(stack) + orig_stack = list(args.interp.activeContext.stack) try: - prim_table[code](interp) + prim_table[code](args) py.test.fail("Expected PrimitiveFailedError") except PrimitiveFailedError: - assert interp.activeContext.stack == orig_stack + assert args.interp.activeContext.stack == orig_stack # smallinteger tests def test_small_int_add(): @@ -255,3 +255,4 @@ assert prim(p.FLOAT_GREATEROREQUAL, [3.5,4.9]) == objtable.w_false assert prim(p.FLOAT_EQUAL, [2.2,2.2]) == objtable.w_true assert prim(p.FLOAT_NOTEQUAL, [2.2,2.2]) == objtable.w_false + From cfbolz at codespeak.net Thu Oct 25 13:36:11 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 25 Oct 2007 13:36:11 +0200 (CEST) Subject: [pypy-svn] r47911 - pypy/extradoc/sprintinfo/bern2007 Message-ID: <20071025113611.7C69C8123@code0.codespeak.net> Author: cfbolz Date: Thu Oct 25 13:36:11 2007 New Revision: 47911 Modified: pypy/extradoc/sprintinfo/bern2007/planning.txt Log: toon appeared, looking tired Modified: pypy/extradoc/sprintinfo/bern2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/bern2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/bern2007/planning.txt Thu Oct 25 13:36:11 2007 @@ -4,7 +4,7 @@ Carl Friedrich Armin Adrian K -Toon (still sleeping for now) +Toon Niko Oscar Lukas From arigo at codespeak.net Thu Oct 25 13:38:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 13:38:20 +0200 (CEST) Subject: [pypy-svn] r47912 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025113820.D430F8129@code0.codespeak.net> Author: arigo Date: Thu Oct 25 13:38:20 2007 New Revision: 47912 Added: pypy/dist/pypy/lang/smalltalk/shadow.py - copied, changed from r47897, pypy/dist/pypy/lang/smalltalk/mirror.py Removed: pypy/dist/pypy/lang/smalltalk/mirror.py Modified: pypy/dist/pypy/lang/smalltalk/classtable.py pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/objtable.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_classtable.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_model.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (cfbolz, arigo) Rename the "mirror" to "shadow". Make it a bit more general. Now we use it a bit more like a cache, and go via the w_class in general instead of directly to the w_class' shadow. Modified: pypy/dist/pypy/lang/smalltalk/classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/classtable.py Thu Oct 25 13:38:20 2007 @@ -1,92 +1,96 @@ -from pypy.lang.smalltalk import mirror +from pypy.lang.smalltalk import shadow -def bootstrap_classmirror(instsize, m_superclass=None, m_metaclass=None, - name='?', format=mirror.POINTERS, varsized=False): +def bootstrap_class(instsize, w_superclass=None, w_metaclass=None, + name='?', format=shadow.POINTERS, varsized=False): from pypy.lang.smalltalk import model - w_class = model.W_Object() # a dummy placeholder for testing - m = mirror.ClassMirror(w_class) - m.methoddict = {} - m.m_superclass = m_superclass - m.m_metaclass = m_metaclass - m.name = name - m.instance_size = instsize - m.instance_kind = format - m.instance_varsized = varsized or format != mirror.POINTERS - m.invalid = False - mirror.mirrorcache.cache[w_class] = m - return m + w_class = model.W_PointersObject(None, 0) # a dummy placeholder for testing + s = shadow.ClassShadow(w_class) + s.methoddict = {} + if w_superclass is not None: + s.s_superclass = w_superclass.as_class_get_shadow() + if w_metaclass is not None: + s.s_metaclass = w_metaclass.as_class_get_shadow() + s.name = name + s.instance_size = instsize + s.instance_kind = format + s.instance_varsized = varsized or format != shadow.POINTERS + s.invalid = False + w_class._shadow = s + return w_class # ___________________________________________________________________________ # Core Bootstrapping Objects classtable = {} def create_classtable(): - def define_core_cls(name, m_superclass, m_metaclass): - assert name.startswith('m_') - mirror = bootstrap_classmirror(instsize=0, # XXX - m_superclass=m_superclass, - m_metaclass=m_metaclass, - name=name[2:]) - classtable[name] = mirror - globals()[name] = mirror - return mirror + def define_core_cls(name, w_superclass, w_metaclass): + assert name.startswith('w_') + shadow = bootstrap_class(instsize=0, # XXX + w_superclass=w_superclass, + w_metaclass=w_metaclass, + name=name[2:]) + classtable[name] = shadow + globals()[name] = shadow + return shadow # Class Name Super class name cls_nm_tbl = [ - ["m_Object", "m_ProtoObject"], - ["m_Behavior", "m_Object"], - ["m_ClassDescription", "m_Behavior"], - ["m_Class", "m_ClassDescription"], - ["m_Metaclass", "m_ClassDescription"], + ["w_Object", "w_ProtoObject"], + ["w_Behavior", "w_Object"], + ["w_ClassDescription", "w_Behavior"], + ["w_Class", "w_ClassDescription"], + ["w_Metaclass", "w_ClassDescription"], ] - define_core_cls("m_ProtoObjectClass", None, None) - define_core_cls("m_ProtoObject", None, m_ProtoObjectClass) + define_core_cls("w_ProtoObjectClass", None, None) + define_core_cls("w_ProtoObject", None, w_ProtoObjectClass) for (cls_nm, super_cls_nm) in cls_nm_tbl: meta_nm = cls_nm + "Class" meta_super_nm = super_cls_nm + "Class" - m_metacls = define_core_cls(meta_nm, classtable[meta_super_nm], None) - define_core_cls(cls_nm, classtable[super_cls_nm], m_metacls) - m_ProtoObjectClass.m_superclass = m_Class - # at this point, all classes that still lack a m_metaclass are themselves + w_metacls = define_core_cls(meta_nm, classtable[meta_super_nm], None) + define_core_cls(cls_nm, classtable[super_cls_nm], w_metacls) + w_ProtoObjectClass.as_class_get_shadow().s_superclass = \ + w_Class.as_class_get_shadow() + # at this point, all classes that still lack a w_metaclass are themselves # metaclasses - for nm, m_cls_obj in classtable.items(): - if m_cls_obj.m_metaclass is None: - m_cls_obj.m_metaclass = m_Metaclass + for nm, w_cls_obj in classtable.items(): + s = w_cls_obj.as_class_get_shadow() + if s.s_metaclass is None: + s.s_metaclass = w_Metaclass.as_class_get_shadow() create_classtable() # ___________________________________________________________________________ # Other classes -def define_cls(cls_nm, supercls_nm, instvarsize=0, format=mirror.POINTERS): - assert cls_nm.startswith("m_") +def define_cls(cls_nm, supercls_nm, instvarsize=0, format=shadow.POINTERS): + assert cls_nm.startswith("w_") meta_nm = cls_nm + "Class" meta_super_nm = supercls_nm + "Class" - m_meta_cls = globals()[meta_nm] = classtable[meta_nm] = \ - bootstrap_classmirror(0, # XXX - classtable[meta_super_nm], - m_Metaclass, - name=meta_nm[2:]) - m_cls = globals()[cls_nm] = classtable[cls_nm] = \ - bootstrap_classmirror(instvarsize, - classtable[supercls_nm], - m_meta_cls, - format=format, - name=cls_nm[2:]) - -define_cls("m_Magnitude", "m_Object") -define_cls("m_Character", "m_Magnitude", instvarsize=1) -define_cls("m_Number", "m_Magnitude") -define_cls("m_Integer", "m_Number") -define_cls("m_SmallInteger", "m_Integer") -define_cls("m_Float", "m_Number", format=mirror.BYTES) -define_cls("m_Collection", "m_Object") -define_cls("m_SequencableCollection", "m_Collection") -define_cls("m_ArrayedCollection", "m_SequencableCollection") -define_cls("m_String", "m_ArrayedCollection") -define_cls("m_ByteString", "m_String", format=mirror.BYTES) -define_cls("m_UndefinedObject", "m_Object") -define_cls("m_Boolean", "m_Object") -define_cls("m_True", "m_Boolean") -define_cls("m_False", "m_Boolean") -define_cls("m_ByteArray", "m_ArrayedCollection", format=mirror.BYTES) -define_cls("m_CompiledMethod", "m_ByteArray", format=mirror.COMPILED_METHOD) + w_meta_cls = globals()[meta_nm] = classtable[meta_nm] = \ + bootstrap_class(0, # XXX + classtable[meta_super_nm], + w_Metaclass, + name=meta_nm[2:]) + w_cls = globals()[cls_nm] = classtable[cls_nm] = \ + bootstrap_class(instvarsize, + classtable[supercls_nm], + w_meta_cls, + format=format, + name=cls_nm[2:]) + +define_cls("w_Magnitude", "w_Object") +define_cls("w_Character", "w_Magnitude", instvarsize=1) +define_cls("w_Number", "w_Magnitude") +define_cls("w_Integer", "w_Number") +define_cls("w_SmallInteger", "w_Integer") +define_cls("w_Float", "w_Number", format=shadow.BYTES) +define_cls("w_Collection", "w_Object") +define_cls("w_SequencableCollection", "w_Collection") +define_cls("w_ArrayedCollection", "w_SequencableCollection") +define_cls("w_String", "w_ArrayedCollection") +define_cls("w_ByteString", "w_String", format=shadow.BYTES) +define_cls("w_UndefinedObject", "w_Object") +define_cls("w_Boolean", "w_Object") +define_cls("w_True", "w_Boolean") +define_cls("w_False", "w_Boolean") +define_cls("w_ByteArray", "w_ArrayedCollection", format=shadow.BYTES) +define_cls("w_CompiledMethod", "w_ByteArray", format=shadow.COMPILED_METHOD) Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 13:38:20 2007 @@ -19,9 +19,9 @@ self.temps = arguments + [None] * method.tempsize self.pc = 0 - def getclassmirror(self): - from pypy.lang.smalltalk.classtable import m_MethodContext # XXX do me - return m_MethodContext + def getclass(self): + from pypy.lang.smalltalk.classtable import w_MethodContext # XXX do me + return w_MethodContext def gethash(self): return 44 # XXX @@ -126,15 +126,16 @@ def _sendSelfSelector(self, selector, argcount, interp): receiver = self.peek(argcount) self._sendSelector(selector, argcount, interp, - receiver, receiver.getclassmirror()) + receiver, receiver.shadow_of_my_class()) def _sendSuperSelector(self, selector, argcount, interp): + s_compiledin = self.method.w_compiledin.as_class_get_shadow() self._sendSelector(selector, argcount, interp, self.receiver, - self.method.m_compiledin.m_superclass) + s_compiledin.s_superclass) def _sendSelector(self, selector, argcount, interp, - receiver, m_receiverclass): - method = m_receiverclass.lookup(selector) + receiver, receiverclassshadow): + method = receiverclassshadow.lookup(selector) assert method if method.primitive: func = primitives.prim_table[method.primitive] Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 13:38:20 2007 @@ -11,7 +11,7 @@ def size(self): return 0 - def getclassmirror(self): + def getclass(self): raise NotImplementedError def gethash(self): @@ -20,13 +20,16 @@ def invariant(self): return True + def shadow_of_my_class(self): + return self.getclass().as_class_get_shadow() + class W_SmallInteger(W_Object): def __init__(self, value): self.value = value - def getclassmirror(self): - from pypy.lang.smalltalk.classtable import m_SmallInteger - return m_SmallInteger + def getclass(self): + from pypy.lang.smalltalk.classtable import w_SmallInteger + return w_SmallInteger def gethash(self): return self.value @@ -41,9 +44,9 @@ def __init__(self, value): self.value = value - def getclassmirror(self): - from pypy.lang.smalltalk.classtable import m_Float - return m_Float + def getclass(self): + from pypy.lang.smalltalk.classtable import w_Float + return w_Float def gethash(self): return XXX # check this @@ -71,54 +74,64 @@ return isinstance(self.hash, int) class W_AbstractObjectWithClassReference(W_AbstractObjectWithIdentityHash): - """ The base class of objects that store 'm_class' explicitly. """ + """ The base class of objects that store 'w_class' explicitly. """ - def __init__(self, m_class): - self.m_class = m_class + def __init__(self, w_class): + if w_class is not None: # it's None only for testing + assert isinstance(w_class, W_PointersObject) + self.w_class = w_class - def getclassmirror(self): - return self.m_class + def getclass(self): + return self.w_class def __str__(self): - self.getclassmirror().check() + # XXX use the shadow of my class if self.size() >= 9: return ''.join(self.fetch(constants.CLASS_NAME_INDEX).bytes) + " class" else: return "a " + ''.join(self.getclass().fetch(constants.CLASS_NAME_INDEX).bytes) - - def getclass(self): - self.getclassmirror().check() - return self.getclassmirror().w_self - + def invariant(self): - from pypy.lang.smalltalk.mirror import ClassMirror return (W_AbstractObjectWithIdentityHash.invariant(self) and - isinstance(self.m_class, ClassMirror)) + isinstance(self.w_class, W_PointersObject)) class W_PointersObject(W_AbstractObjectWithClassReference): """ The normal object """ - def __init__(self, m_class, size): - W_AbstractObjectWithClassReference.__init__(self, m_class) - self.vars = [None] * size + def __init__(self, w_class, size): + W_AbstractObjectWithClassReference.__init__(self, w_class) + self._vars = [None] * size + self._shadow = None def fetch(self, index): - return self.vars[index] + return self._vars[index] def store(self, index, w_value): - self.vars[index] = w_value + if self._shadow is not None: + self._shadow.invalidate() + self._vars[index] = w_value def size(self): - return len(self.vars) + return len(self._vars) def invariant(self): return (W_AbstractObjectWithClassReference.invariant(self) and - isinstance(self.vars, list)) + isinstance(self._vars, list)) + + def as_class_get_shadow(self): + from pypy.lang.smalltalk.shadow import ClassShadow + shadow = self._shadow + if shadow is None: + self._shadow = shadow = ClassShadow(self) + assert isinstance(shadow, ClassShadow) # for now, the only kind + shadow.check_for_updates() + return shadow def compiledmethodnamed(self, methodname): - w_methoddict = self.fetch(constants.CLASS_METHODDICT_INDEX).vars + # XXX kill me. Temporary, for testing + w_methoddict = self.fetch(constants.CLASS_METHODDICT_INDEX)._vars names = w_methoddict[constants.METHODDICT_NAMES_INDEX:] - values = w_methoddict[constants.METHODDICT_VALUES_INDEX].vars + values = w_methoddict[constants.METHODDICT_VALUES_INDEX]._vars for var in names: if isinstance(var, W_BytesObject): if str(var) == repr(methodname): @@ -126,6 +139,7 @@ raise MethodNotFound def lookup(self, methodname): + # XXX kill me. Temporary, for testing in_class = self while in_class != None: try: @@ -143,8 +157,8 @@ return self.lookup("doesNotUnderstand") class W_BytesObject(W_AbstractObjectWithClassReference): - def __init__(self, m_class, size): - W_AbstractObjectWithClassReference.__init__(self, m_class) + def __init__(self, w_class, size): + W_AbstractObjectWithClassReference.__init__(self, w_class) self.bytes = ['\x00'] * size def getbyte(self, n): @@ -168,8 +182,8 @@ return True class W_WordsObject(W_AbstractObjectWithClassReference): - def __init__(self, m_class, size): - W_AbstractObjectWithClassReference.__init__(self, m_class) + def __init__(self, w_class, size): + W_AbstractObjectWithClassReference.__init__(self, w_class) self.words = [0] * size def getword(self, n): @@ -208,18 +222,18 @@ The trailer has two variant formats. In the first variant, the last byte is at least 252 and the last four bytes represent a source pointer into one of the sources files (see #sourcePointer). In the second variant, the last byte is less than 252, and the last several bytes are a compressed version of the names of the method's temporary variables. The number of bytes used for this purpose is the value of the last byte in the method. """ def __init__(self, literalsize, bytes, argsize=0, - tempsize=0, primitive=0, m_compiledin=None): + tempsize=0, primitive=0, w_compiledin=None): # self.literals = [None] * size self.literalsize = literalsize - self.m_compiledin = m_compiledin + self.w_compiledin = w_compiledin self.bytes = bytes self.argsize = argsize self.tempsize = tempsize self.primitive = primitive - def getclassmirror(self): - from pypy.lang.smalltalk.classtable import m_CompiledMethod - return m_CompiledMethod + def getclass(self): + from pypy.lang.smalltalk.classtable import w_CompiledMethod + return w_CompiledMethod def gethash(self): return 43 # XXX Modified: pypy/dist/pypy/lang/smalltalk/objtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/objtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/objtable.py Thu Oct 25 13:38:20 2007 @@ -14,7 +14,7 @@ return model.W_Float(i) def wrap_string(str): - w_inst = ct.m_ByteString.new(len(str)) + w_inst = ct.w_ByteString.as_class_get_shadow().new(len(str)) for i in range(len(str)): w_inst.setbyte(i, ord(str[i])) return w_inst @@ -23,9 +23,9 @@ return CharacterTable[ord(c)] def ord_w_char(w_c): - assert w_c.getclassmirror() is ct.m_Character + assert w_c.getclass() is ct.w_Character w_ord = w_c.fetch(CHARACTER_VALUE_INDEX) - assert w_ord.getclassmirror() is ct.m_SmallInteger + assert w_ord.getclass() is ct.w_SmallInteger assert isinstance(w_ord, model.W_SmallInteger) return w_ord.value @@ -41,15 +41,15 @@ def wrap_char_table(): global CharacterTable def bld_char(i): - w_cinst = ct.m_Character.new() + w_cinst = ct.w_Character.as_class_get_shadow().new() w_cinst.store(CHARACTER_VALUE_INDEX, wrap_int(i)) return w_cinst CharacterTable = [bld_char(i) for i in range(256)] wrap_char_table() -w_true = ct.m_True.new() -w_false = ct.m_False.new() -w_nil = ct.m_UndefinedObject.new() +w_true = ct.w_True.as_class_get_shadow().new() +w_false = ct.w_False.as_class_get_shadow().new() +w_nil = ct.w_UndefinedObject.as_class_get_shadow().new() w_mone = wrap_int(-1) w_zero = wrap_int(0) w_one = wrap_int(1) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 13:38:20 2007 @@ -1,5 +1,5 @@ import operator -from pypy.lang.smalltalk import model, mirror +from pypy.lang.smalltalk import model, shadow from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import objtable from pypy.rlib import rarithmetic @@ -229,7 +229,7 @@ @primitive(SIZE) @stack(1) def func(args, (w_obj,)): - if not w_obj.getclassmirror().isvariable(): + if not w_obj.shadow_of_my_class().isvariable(): raise PrimitiveFailedError() return w_obj.size() @@ -244,7 +244,7 @@ @stack(3) def func(args, stack): w_obj, idx, w_val = common_at_put(stack) - if w_val.getclassmirror() is not classtable.m_Character: + if w_val.getclass() is not classtable.w_Character: raise PrimitiveFailedError() w_obj.setbyte(idx, objtable.ord_w_char(w_val)) return w_val @@ -270,7 +270,7 @@ def func(args, (w_rcvr, w_idx)): idx = unwrap_int(w_idx) # XXX should be idx-1, probably - assert_bounds(idx, 0, w_rcvr.getclassmirror().instance_size) + assert_bounds(idx, 0, w_rcvr.shadow_of_my_class().instance_size) return w_rcvr.fetch(idx) @primitive(OBJECT_AT_PUT) @@ -278,26 +278,26 @@ def func(args, (w_rcvr, w_idx, w_val)): idx = unwrap_int(w_idx) # XXX should be idx-1, probably - assert_bounds(idx, 0, w_rcvr.getclassmirror().instance_size) + assert_bounds(idx, 0, w_rcvr.shadow_of_my_class().instance_size) w_rcvr.store(idx, w_val) return w_val @primitive(NEW) @stack(1) def func(args, (w_cls,)): - m_cls = mirror.mirrorcache.getmirror(w_cls) - if m_cls.isvariable(): + shadow = w_cls.as_class_get_shadow() + if shadow.isvariable(): raise PrimitiveFailedError() - return m_cls.new() + return shadow.new() @primitive(NEW_WITH_ARG) @stack(2) def func(args, (w_cls, w_size)): - m_cls = mirror.mirrorcache.getmirror(w_cls) - if not m_cls.isvariable(): + shadow = w_cls.as_class_get_shadow() + if not shadow.isvariable(): raise PrimitiveFailedError() size = unwrap_int(w_size) - return m_cls.new(size) + return shadow.new(size) @primitive(ARRAY_BECOME_ONE_WAY) def func(args): @@ -309,12 +309,12 @@ # I *think* this is the correct behavior, but I'm not quite sure. # Might be restricted to fixed length fields? idx = unwrap_int(w_idx) - m_cls = w_rcvr.getclassmirror() + shadow = w_rcvr.shadow_of_my_class() if idx < 0: raise PrimitiveFailedError() - if idx < m_cls.instsize(): + if idx < shadow.instsize(): return w_rcvr.fetch(idx) - idx -= m_cls.instsize() + idx -= shadow.instsize() if idx < w_rcvr.size(): return subscript(idx, w_rcvr) raise PrimitiveFailedError() Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 13:38:20 2007 @@ -1,7 +1,6 @@ import py from pypy.lang.smalltalk import model from pypy.lang.smalltalk import objtable -from pypy.lang.smalltalk.mirror import mirrorcache from pypy.rlib import objectmodel def int2str(integer): @@ -120,6 +119,7 @@ def assign_mirrors(self): # assign the mirrors to the classes already in classtable from pypy.lang.smalltalk import classtable, constants + import py; py.test.skip("FIX ME") for so_index, name in [ (constants.SO_SMALLINTEGER_CLASS, "m_SmallInteger"), (constants.SO_STRING_CLASS, "m_String"), Modified: pypy/dist/pypy/lang/smalltalk/test/test_classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_classtable.py Thu Oct 25 13:38:20 2007 @@ -1,23 +1,20 @@ -from pypy.lang.smalltalk.classtable import classtable -import pypy.lang.smalltalk.classtable as ct - -def inherits_from(m_cls, m_superclass): - m_p = m_cls - while m_p and m_p != m_superclass: - m_p = m_p.m_superclass - return m_p == m_superclass +from pypy.lang.smalltalk import classtable def test_every_class_is_an_instance_of_a_metaclass(): - for (nm, m_cls) in classtable.items(): - assert (m_cls.m_metaclass is ct.m_Metaclass or - m_cls.m_metaclass.m_metaclass is ct.m_Metaclass) - + for (nm, w_cls) in classtable.classtable.items(): + shadow = w_cls.as_class_get_shadow() + assert shadow.ismetaclass() or shadow.s_metaclass.ismetaclass() + def test_every_metaclass_inherits_from_class_and_behavior(): - for (nm, m_cls) in classtable.items(): - if m_cls.m_metaclass is ct.m_Metaclass: - assert inherits_from(m_cls, ct.m_Class) - assert inherits_from(ct.m_Class, ct.m_Behavior) + s_Class = classtable.w_Class.as_class_get_shadow() + s_Behavior = classtable.w_Behavior.as_class_get_shadow() + for (nm, w_cls) in classtable.classtable.items(): + shadow = w_cls.as_class_get_shadow() + if shadow.ismetaclass(): + assert shadow.inherits_from(s_Class) + assert s_Class.inherits_from(s_Behavior) def test_metaclass_of_metaclass_is_an_instance_of_metaclass(): - assert ct.m_Metaclass.m_metaclass.m_metaclass is ct.m_Metaclass + s_Metaclass = classtable.w_Metaclass.as_class_get_shadow() + assert s_Metaclass.s_metaclass.s_metaclass is s_Metaclass Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Thu Oct 25 13:38:20 2007 @@ -1,9 +1,9 @@ import py -from pypy.lang.smalltalk import model, interpreter, primitives, mirror +from pypy.lang.smalltalk import model, interpreter, primitives, shadow from pypy.lang.smalltalk.objtable import wrap_int import pypy.lang.smalltalk.classtable as ct -mockclassmirror = ct.bootstrap_classmirror +mockclass = ct.bootstrap_class # expose the bytecode's values as global constants. # Bytecodes that have a whole range are exposed as global functions: @@ -79,7 +79,7 @@ def test_pushReceiverVariableBytecode(bytecode = (pushReceiverVariableBytecode(0) + pushReceiverVariableBytecode(1) + pushReceiverVariableBytecode(2))): - w_demo = mockclassmirror(3).new() + w_demo = mockclass(3).as_class_get_shadow().new() w_demo.store(0, "egg") w_demo.store(1, "bar") w_demo.store(2, "baz") @@ -110,7 +110,7 @@ assert interp.activeContext.stack == ["a", "b", "c"] def test_pushLiteralVariableBytecode(bytecode=pushLiteralVariableBytecode(0)): - w_association = mockclassmirror(2).new() + w_association = mockclass(2).as_class_get_shadow().new() w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(bytecode) @@ -120,9 +120,9 @@ def test_storeAndPopReceiverVariableBytecode(bytecode=storeAndPopReceiverVariableBytecode, popped=True): - m_class = mockclassmirror(8) + shadow = mockclass(8).as_class_get_shadow() for index in range(8): - w_object = m_class.new() + w_object = shadow.new() interp = new_interpreter(pushConstantTrueBytecode + bytecode(index)) interp.activeContext.receiver = w_object interp.step() @@ -198,18 +198,19 @@ interp.step() assert interp.activeContext.stack == [interp.ZERO, interp.ZERO] -# m_class - the class from which the method is going to be called +# w_class - the class from which the method is going to be called # (and on which it is going to be installed) # w_object - the actual object we will be sending the method to # bytecodes - the bytecode to be executed -def sendBytecodesTest(m_class, w_object, bytecodes): +def sendBytecodesTest(w_class, w_object, bytecodes): for bytecode, result in [ (returnReceiver, w_object), (returnTrue, interpreter.Interpreter.TRUE), (returnFalse, interpreter.Interpreter.FALSE), (returnNil, interpreter.Interpreter.NIL), (returnTopFromMethod, interpreter.Interpreter.ONE) ]: - m_class.installmethod("foo", - model.W_CompiledMethod(0, pushConstantOneBytecode + bytecode)) + shadow = w_class.as_class_get_shadow() + shadow.installmethod("foo", + model.W_CompiledMethod(0, pushConstantOneBytecode + bytecode)) interp = new_interpreter(bytecodes) interp.activeContext.method.literals = fakeliterals("foo") interp.activeContext.push(w_object) @@ -218,7 +219,7 @@ assert interp.activeContext.sender == callerContext assert interp.activeContext.stack == [] assert interp.activeContext.receiver == w_object - assert interp.activeContext.method == m_class.methoddict["foo"] + assert interp.activeContext.method == shadow.methoddict["foo"] assert callerContext.stack == [] interp.step() interp.step() @@ -226,17 +227,17 @@ assert interp.activeContext.stack == [result] def test_sendLiteralSelectorBytecode(): - m_class = mockclassmirror(0) - w_object = m_class.new() - sendBytecodesTest(m_class, w_object, sendLiteralSelectorBytecode(0)) + w_class = mockclass(0) + w_object = w_class.as_class_get_shadow().new() + sendBytecodesTest(w_class, w_object, sendLiteralSelectorBytecode(0)) def test_fibWithArgument(): bytecode = ''.join(map(chr, [ 16, 119, 178, 154, 118, 164, 11, 112, 16, 118, 177, 224, 112, 16, 119, 177, 224, 176, 124 ])) - m_class = mockclassmirror(0) + shadow = mockclass(0).as_class_get_shadow() method = model.W_CompiledMethod(1, bytecode, 1) method.literals = fakeliterals("fib:") - m_class.installmethod("fib:", method) - w_object = m_class.new() + shadow.installmethod("fib:", method) + w_object = shadow.new() interp = new_interpreter(sendLiteralSelectorBytecode(16) + returnTopFromMethod) interp.activeContext.method.literals = fakeliterals("fib:") interp.activeContext.push(w_object) @@ -245,10 +246,10 @@ assert primitives.unwrap_int(result) == 34 def test_send_to_primitive(): - m_smallintclass = ct.m_SmallInteger + s_smallintclass = ct.w_SmallInteger.as_class_get_shadow() prim_meth = model.W_CompiledMethod(0, "", argsize=1, primitive=primitives.SUBTRACT) - m_smallintclass.installmethod("sub", prim_meth) + s_smallintclass.installmethod("sub", prim_meth) try: interp = new_interpreter(sendLiteralSelectorBytecode(1 + 16)) interp.activeContext.method.literals = fakeliterals("foo", "sub") @@ -261,7 +262,7 @@ w_result = interp.activeContext.pop() assert primitives.unwrap_int(w_result) == 42 finally: - del m_smallintclass.methoddict['sub'] # clean up after you + del s_smallintclass.methoddict['sub'] # clean up after you def test_longJumpIfTrue(): interp = new_interpreter(longJumpIfTrue(0) + chr(15) + longJumpIfTrue(0) + chr(15)) @@ -334,7 +335,7 @@ test_pushLiteralVariableBytecode(extendedPushBytecode + chr((3<<6) + 0)) def storeAssociation(bytecode): - w_association = mockclassmirror(2).new() + w_association = mockclass(2).as_class_get_shadow().new() w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(pushConstantOneBytecode + bytecode) @@ -356,13 +357,13 @@ def test_callPrimitiveAndPush_fallback(): interp = new_interpreter(bytecodePrimAdd) - m_class = mockclassmirror(0) - m_class.installmethod("+", model.W_CompiledMethod(1, "", 1)) - w_object = m_class.new() + shadow = mockclass(0).as_class_get_shadow() + shadow.installmethod("+", model.W_CompiledMethod(1, "", 1)) + w_object = shadow.new() interp.activeContext.push(w_object) interp.activeContext.push(interp.ONE) interp.step() - assert interp.activeContext.method == m_class.methoddict["+"] + assert interp.activeContext.method == shadow.methoddict["+"] assert interp.activeContext.receiver is w_object assert interp.activeContext.gettemp(0) == interp.ONE assert interp.activeContext.stack == [] @@ -383,49 +384,50 @@ interp.FALSE, interp.TRUE] def test_singleExtendedSendBytecode(): - m_class = mockclassmirror(0) - w_object = m_class.new() - sendBytecodesTest(m_class, w_object, singleExtendedSendBytecode + chr((0<<5)+0)) + w_class = mockclass(0) + w_object = w_class.as_class_get_shadow().new() + sendBytecodesTest(w_class, w_object, singleExtendedSendBytecode + chr((0<<5)+0)) def test_singleExtendedSuperBytecode(bytecode=singleExtendedSuperBytecode + chr((0<<5) + 0)): - m_supersuper = mockclassmirror(0) - m_super = mockclassmirror(0, m_superclass=m_supersuper) - m_class = mockclassmirror(0, m_superclass=m_super) - w_object = m_class.new() - # first call method installed in m_class + w_supersuper = mockclass(0) + w_super = mockclass(0, w_superclass=w_supersuper) + w_class = mockclass(0, w_superclass=w_super) + w_object = w_class.as_class_get_shadow().new() + # first call method installed in w_class bytecodes = singleExtendedSendBytecode + chr(0) # which does a call to its super - m_class.installmethod("foo", - model.W_CompiledMethod(0, bytecode)) + meth1 = model.W_CompiledMethod(0, bytecode) + w_class.as_class_get_shadow().installmethod("foo", meth1) # and that one again to its super - m_super.installmethod("foo", - model.W_CompiledMethod(0, bytecode)) - m_supersuper.installmethod("foo", - model.W_CompiledMethod(0, "")) - m_class.methoddict["foo"].literals = fakeliterals("foo") - m_super.methoddict["foo"].literals = fakeliterals("foo") + meth2 = model.W_CompiledMethod(0, bytecode) + w_super.as_class_get_shadow().installmethod("foo", meth2) + meth3 = model.W_CompiledMethod(0, "") + w_supersuper.as_class_get_shadow().installmethod("foo", meth3) + meth1.literals = fakeliterals("foo") + meth2.literals = fakeliterals("foo") interp = new_interpreter(bytecodes) interp.activeContext.method.literals = fakeliterals("foo") interp.activeContext.push(w_object) - for m_specificclass in [m_class, m_super, m_supersuper]: + for w_specificclass in [w_class, w_super, w_supersuper]: callerContext = interp.activeContext interp.step() assert interp.activeContext.sender == callerContext assert interp.activeContext.stack == [] assert interp.activeContext.receiver == w_object - assert interp.activeContext.method == m_specificclass.methoddict["foo"] + meth = w_specificclass.as_class_get_shadow().methoddict["foo"] + assert interp.activeContext.method == meth assert callerContext.stack == [] def test_secondExtendedSendBytecode(): - m_class = mockclassmirror(0) - w_object = m_class.new() - sendBytecodesTest(m_class, w_object, secondExtendedSendBytecode + chr(0)) + w_class = mockclass(0) + w_object = w_class.as_class_get_shadow().new() + sendBytecodesTest(w_class, w_object, secondExtendedSendBytecode + chr(0)) def test_doubleExtendedDoAnythinBytecode(): - m_class = mockclassmirror(0) - w_object = m_class.new() + w_class = mockclass(0) + w_object = w_class.as_class_get_shadow().new() - sendBytecodesTest(m_class, w_object, doubleExtendedDoAnythingBytecode + chr((0<<5) + 0) + chr(0)) + sendBytecodesTest(w_class, w_object, doubleExtendedDoAnythingBytecode + chr((0<<5) + 0) + chr(0)) test_singleExtendedSuperBytecode(doubleExtendedDoAnythingBytecode + (chr((1<<5) + 0) + chr(0))) Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Thu Oct 25 13:38:20 2007 @@ -1,31 +1,32 @@ import py -from pypy.lang.smalltalk import model, mirror +from pypy.lang.smalltalk import model, shadow import pypy.lang.smalltalk.classtable as ct -mockclassmirror = ct.bootstrap_classmirror +mockclass = ct.bootstrap_class def test_new(): - m_mycls = mockclassmirror(0) - w_myinstance = m_mycls.new() + w_mycls = mockclass(0) + w_myinstance = w_mycls.as_class_get_shadow().new() assert isinstance(w_myinstance, model.W_PointersObject) - assert w_myinstance.getclassmirror() is m_mycls + assert w_myinstance.getclass() is w_mycls + assert w_myinstance.shadow_of_my_class() is w_mycls.as_class_get_shadow() def test_new_namedvars(): - m_mycls = mockclassmirror(3) - w_myinstance = m_mycls.new() + w_mycls = mockclass(3) + w_myinstance = w_mycls.as_class_get_shadow().new() assert isinstance(w_myinstance, model.W_PointersObject) - assert w_myinstance.getclassmirror() is m_mycls + assert w_myinstance.getclass() is w_mycls assert w_myinstance.fetch(0) is None py.test.raises(IndexError, lambda: w_myinstance.fetch(3)) w_myinstance.store(1, w_myinstance) assert w_myinstance.fetch(1) is w_myinstance def test_bytes_object(): - m_class = mockclassmirror(0, format=mirror.BYTES) - w_bytes = m_class.new(20) - assert w_bytes.getclassmirror() is m_class + w_class = mockclass(0, format=shadow.BYTES) + w_bytes = w_class.as_class_get_shadow().new(20) + assert w_bytes.getclass() is w_class assert w_bytes.size() == 20 - assert m_class.instsize() == 0 + assert w_class.as_class_get_shadow().instsize() == 0 assert w_bytes.getbyte(3) == 00 w_bytes.setbyte(3, 0xAA) assert w_bytes.getbyte(3) == 0xAA @@ -33,11 +34,11 @@ py.test.raises(IndexError, lambda: w_bytes.getbyte(20)) def test_word_object(): - m_class = mockclassmirror(0, format=mirror.WORDS) - w_bytes = m_class.new(20) - assert w_bytes.getclassmirror() is m_class + w_class = mockclass(0, format=shadow.WORDS) + w_bytes = w_class.as_class_get_shadow().new(20) + assert w_bytes.getclass() is w_class assert w_bytes.size() == 20 - assert m_class.instsize() == 0 + assert w_class.as_class_get_shadow().instsize() == 0 assert w_bytes.getword(3) == 0 w_bytes.setword(3, 42) assert w_bytes.getword(3) == 42 @@ -45,29 +46,34 @@ py.test.raises(IndexError, lambda: w_bytes.getword(20)) def test_method_lookup(): - m_class = mockclassmirror(0) - m_class.methoddict["foo"] = 1 - m_class.methoddict["bar"] = 2 - m_subclass = mockclassmirror(0, m_superclass=m_class) - m_subclass.methoddict["foo"] = 3 - assert m_class.lookup("foo") == 1 - assert m_class.lookup("bar") == 2 - assert m_class.lookup("zork") == None - assert m_subclass.lookup("foo") == 3 - assert m_subclass.lookup("bar") == 2 - assert m_subclass.lookup("zork") == None - -def test_m_compiledin(): - m_super = mockclassmirror(0) - m_class = mockclassmirror(0, m_superclass=m_super) - m_super.installmethod("foo", model.W_CompiledMethod(0, "")) - assert m_class.lookup("foo").m_compiledin is m_super + w_class = mockclass(0) + shadow = w_class.as_class_get_shadow() + shadow.methoddict["foo"] = 1 + shadow.methoddict["bar"] = 2 + w_subclass = mockclass(0, w_superclass=w_class) + subshadow = w_subclass.as_class_get_shadow() + assert subshadow.s_superclass is shadow + subshadow.methoddict["foo"] = 3 + assert shadow.lookup("foo") == 1 + assert shadow.lookup("bar") == 2 + assert shadow.lookup("zork") == None + assert subshadow.lookup("foo") == 3 + assert subshadow.lookup("bar") == 2 + assert subshadow.lookup("zork") == None + +def test_w_compiledin(): + w_super = mockclass(0) + w_class = mockclass(0, w_superclass=w_super) + supershadow = w_super.as_class_get_shadow() + supershadow.installmethod("foo", model.W_CompiledMethod(0, "")) + classshadow = w_class.as_class_get_shadow() + assert classshadow.lookup("foo").w_compiledin is w_super def test_hashes(): w_five = model.W_SmallInteger(5) assert w_five.gethash() == 5 - m_class = mockclassmirror(0) - w_inst = m_class.new() + w_class = mockclass(0) + w_inst = w_class.as_class_get_shadow().new() assert w_inst.hash == w_inst.UNASSIGNED_HASH h1 = w_inst.gethash() h2 = w_inst.gethash() Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 13:38:20 2007 @@ -1,6 +1,6 @@ import py from pypy.lang.smalltalk.primitives import prim_table, PrimitiveFailedError -from pypy.lang.smalltalk import model, mirror +from pypy.lang.smalltalk import model, shadow from pypy.lang.smalltalk import interpreter from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import objtable @@ -8,7 +8,7 @@ # Violates the guideline, but we use it A LOT to reference the primitive codes: import pypy.lang.smalltalk.primitives as p -mockclassmirror = classtable.bootstrap_classmirror +mockclass = classtable.bootstrap_class class MockFrame(interpreter.W_MethodContext): def __init__(self, stack): @@ -20,7 +20,6 @@ if isinstance(x, model.W_Object): return x if isinstance(x, str) and len(x) == 1: return objtable.wrap_char(x) if isinstance(x, str): return objtable.wrap_string(x) - if isinstance(x, mirror.ClassMirror): return x.w_self raise NotImplementedError def mock(stack): @@ -145,21 +144,21 @@ assert prim(p.FLOAT_ADD, [3,4.5]).value == 7.5 def test_at(): - w_obj = mockclassmirror(0, varsized=True).new(1) + w_obj = mockclass(0, varsized=True).as_class_get_shadow().new(1) w_obj.store(0, "foo") assert prim(p.AT, [w_obj, 0]) == "foo" def test_invalid_at(): - w_obj = mockclassmirror(0).new() + w_obj = mockclass(0).as_class_get_shadow().new() prim_fails(p.AT, [w_obj, 0]) def test_at_put(): - w_obj = mockclassmirror(0, varsized=1).new(1) + w_obj = mockclass(0, varsized=1).as_class_get_shadow().new(1) assert prim(p.AT_PUT, [w_obj, 0, 22]).value == 22 assert prim(p.AT, [w_obj, 0]).value == 22 def test_invalid_at_put(): - w_obj = mockclassmirror(0).new() + w_obj = mockclass(0).as_class_get_shadow().new() prim_fails(p.AT_PUT, [w_obj, 0, 22]) def test_string_at(): @@ -180,12 +179,12 @@ prim_fails(p.OBJECT_AT, ["q", objtable.CHARACTER_VALUE_INDEX+1]) def test_object_at_put(): - w_obj = mockclassmirror(1).new() + w_obj = mockclass(1).as_class_get_shadow().new() assert prim(p.OBJECT_AT_PUT, [w_obj, 0, "q"]) is wrap("q") assert prim(p.OBJECT_AT, [w_obj, 0]) is wrap("q") def test_invalid_object_at_put(): - w_obj = mockclassmirror(1).new() + w_obj = mockclass(1).as_class_get_shadow().new() prim_fails(p.OBJECT_AT, [w_obj, 1, 1]) def test_string_at_put(): @@ -196,19 +195,19 @@ assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i]) def test_new(): - w_res = prim(p.NEW, [classtable.m_Object]) - assert w_res.getclassmirror() == classtable.m_Object + w_res = prim(p.NEW, [classtable.w_Object]) + assert w_res.getclass() == classtable.w_Object def test_invalid_new(): - prim_fails(p.NEW, [classtable.m_ByteString]) + prim_fails(p.NEW, [classtable.w_ByteString]) def test_new_with_arg(): - w_res = prim(p.NEW_WITH_ARG, [classtable.m_ByteString, 20]) - assert w_res.getclassmirror() == classtable.m_ByteString + w_res = prim(p.NEW_WITH_ARG, [classtable.w_ByteString, 20]) + assert w_res.getclass() == classtable.w_ByteString assert w_res.size() == 20 def test_invalid_new_with_arg(): - prim_fails(p.NEW_WITH_ARG, [classtable.m_Object, 20]) + prim_fails(p.NEW_WITH_ARG, [classtable.w_Object, 20]) def test_inst_var_at(): # I am not entirely sure if this is what this primitive is @@ -219,7 +218,7 @@ assert w_v.value == ord("b") def test_as_oop(): - w_obj = mockclassmirror(0).new() + w_obj = mockclass(0).as_class_get_shadow().new() w_obj.w_hash = wrap(22) assert prim(p.AS_OOP, [w_obj]).value == 22 From jacob at codespeak.net Thu Oct 25 13:46:04 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Thu, 25 Oct 2007 13:46:04 +0200 (CEST) Subject: [pypy-svn] r47913 - pypy/branch/dist-future-fixing Message-ID: <20071025114604.552B48129@code0.codespeak.net> Author: jacob Date: Thu Oct 25 13:46:03 2007 New Revision: 47913 Added: pypy/branch/dist-future-fixing/ - copied from r47912, pypy/dist/ Log: Creating a branch for working on the parser and fixing the __future__ mess. From arigo at codespeak.net Thu Oct 25 13:50:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 13:50:20 +0200 (CEST) Subject: [pypy-svn] r47914 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025115020.4D9FA812B@code0.codespeak.net> Author: arigo Date: Thu Oct 25 13:50:19 2007 New Revision: 47914 Modified: pypy/dist/pypy/lang/smalltalk/classtable.py pypy/dist/pypy/lang/smalltalk/interpreter.py Log: Define w_MethodContext. Modified: pypy/dist/pypy/lang/smalltalk/classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/classtable.py Thu Oct 25 13:50:19 2007 @@ -94,3 +94,4 @@ define_cls("w_False", "w_Boolean") define_cls("w_ByteArray", "w_ArrayedCollection", format=shadow.BYTES) define_cls("w_CompiledMethod", "w_ByteArray", format=shadow.COMPILED_METHOD) +define_cls("w_MethodContext", "w_Object") Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 13:50:19 2007 @@ -20,7 +20,7 @@ self.pc = 0 def getclass(self): - from pypy.lang.smalltalk.classtable import w_MethodContext # XXX do me + from pypy.lang.smalltalk.classtable import w_MethodContext return w_MethodContext def gethash(self): From niko at codespeak.net Thu Oct 25 14:09:28 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 25 Oct 2007 14:09:28 +0200 (CEST) Subject: [pypy-svn] r47915 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025120928.25385812D@code0.codespeak.net> Author: niko Date: Thu Oct 25 14:09:26 2007 New Revision: 47915 Modified: pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (niko, tudor, lukas) implement value primitive and sketch of a test Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Thu Oct 25 14:09:26 2007 @@ -1,8 +1,6 @@ # ___________________________________________________________________________ # Slot Names -# -# n.b. Off by one from SQUEAK! CHARACTER_VALUE_INDEX = 0 # Page 630 of the blue book @@ -22,10 +20,13 @@ ASSOCIATION_KEY_INDEX = 0 ASSOCIATION_VALUE_INDEX = 1 -BLOCK_CONTEXT_INSTRUCTION_POINTER_INDEX = 0 -BLOCK_CONTEXT_BLOCK_ARGUMENT_COUNT_INDEX = 2 -BLOCK_CONTEXT_INITIAL_IP_INDEX = 3 -BLOCK_CONTEXT_HOME_INDEX = 4 +BLOCK_CONTEXT_CALLER_INDEX = 0 +BLOCK_CONTEXT_INSTRUCTION_POINTER_INDEX = 1 +BLOCK_CONTEXT_STACK_POINTER_INDEX = 2 +BLOCK_CONTEXT_BLOCK_ARGUMENT_COUNT_INDEX = 3 +BLOCK_CONTEXT_INITIAL_IP_INDEX = 4 +BLOCK_CONTEXT_HOME_INDEX = 5 +BLOCK_CONTEXT_TEMP_FRAME_START = 6 # ----- special objects indices ------- Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 14:09:26 2007 @@ -148,14 +148,14 @@ self.push(w_result) return arguments = self.stack[len(self.stack)-argcount:] - interp.activeContext = method.createFrame(receiver, arguments, self) + interp.w_active_context = method.createFrame(receiver, arguments, self) self.pop_n(argcount + 1) def _return(self, object, interp): if self.sender is None: # for tests, when returning from the top-level context raise ReturnFromTopLevel(object) self.sender.push(object) - interp.activeContext = self.sender + interp.w_active_context = self.sender def returnReceiver(self, interp): self._return(self.receiver, interp) @@ -410,7 +410,7 @@ TWO = objtable.w_two def __init__(self): - self.activeContext = None + self.w_active_context = None def interpret(self): @@ -421,9 +421,9 @@ return e.object def step(self): - next = self.activeContext.getNextBytecode() + next = self.w_active_context.getNextBytecode() bytecodeimpl = BYTECODE_TABLE[next] - bytecodeimpl(self.activeContext, self) + bytecodeimpl(self.w_active_context, self) class ReturnFromTopLevel(Exception): def __init__(self, object): Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 14:09:26 2007 @@ -53,7 +53,7 @@ def stack(n): def decorator(wrapped): def result(args): - frame = args.interp.activeContext + frame = args.interp.w_active_context items = frame.stack[len(frame.stack)-n:] res = wrapped(args, items) frame.pop_n(n) # only if no exception occurs! @@ -512,7 +512,7 @@ @primitive(PRIMITIVE_BLOCK_COPY) @stack(2) def func(args, (w_argcnt, w_context)): - frame = args.interp.activeContext + frame = args.interp.w_active_context # From B.B.: If receiver is a MethodContext, then it becomes # the new BlockContext's home context. Otherwise, the home @@ -524,7 +524,7 @@ # The block bytecodes are stored inline: so we skip past the # byteodes to invoke this primitive to find them (hence +3) - w_new_context = classtable.w_BlockContext.new() + w_new_context = classtable.w_BlockContext.new(unwrap_int(w_argcnt)) initialip = frame.pc + 3 # Initialize various fields. @@ -536,9 +536,32 @@ @primitive(PRIMITIVE_VALUE) def func(args): - #w_rcvr = args.interp.activeContext.peek(0) - #w_argcnt = w_rcvr.fetch(BLOCK_CONTEXT_BLOCK_ARGUMENT_COUNT_INDEX) - raise PrimitiveNotYetWrittenError() + + # If nargs == 4, stack looks like: + # 3 2 1 0 + # Rcvr | Arg 0 | Arg1 | Arg 2 + # + + w_block_ctx = args.interp.w_active_context.peek(args.argument_count-1) + + w_exp_arg_cnt = w_block_ctx.fetch(BLOCK_CONTEXT_BLOCK_ARGUMENT_COUNT_INDEX) + exp_arg_cnt = unwrap_int(w_exp_arg_cnt) + if args.argument_count != exp_arg_cnt: + raise PrimitiveFailedError() + + # Copy the values from the stack such that the most recently pushed + # item (index 0) ends up in slot BLOCK_CONTEXT_TEMP_FRAME_START + nargs - 1 + for idx in range(exp_arg_cnt - 1): + w_block_ctx.store( + BLOCK_CONTEXT_TEMP_FRAME_START+idx, + w_block_ctx.fetch(exp_arg_cnt - idx - 1)) + + # Set some fields + w_initial_ip = w_block_ctx.fetch(BLOCK_CONTEXT_INITIAL_IP_INDEX) + w_block_ctx.store(BLOCK_CONTEXT_INSTRUCTION_POINTER_INDEX, w_initial_ip) + w_block_ctx.store(BLOCK_CONTEXT_STACK_POINTER_INDEX, w_exp_arg_cnt) + w_block_ctx.store(BLOCK_CONTEXT_CALLER_INDEX, args.interp.w_active_context) + args.interp.w_active_context = w_block_ctx @primitive(PRIMITIVE_VALUE_WITH_ARGS) @stack(2) Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Thu Oct 25 14:09:26 2007 @@ -34,7 +34,7 @@ argsize=2, tempsize=1) w_frame = w_method.createFrame(receiver, ["foo", "bar"]) interp = interpreter.Interpreter() - interp.activeContext = w_frame + interp.w_active_context = w_frame return interp def test_create_frame(): @@ -53,7 +53,7 @@ def test_push_pop(): interp = new_interpreter("") - frame = interp.activeContext + frame = interp.w_active_context frame.push(12) frame.push(34) frame.push(56) @@ -74,7 +74,7 @@ def test_pushReceiverBytecode(): interp = new_interpreter(pushReceiverBytecode) interp.step() - assert interp.activeContext.top() == interp.activeContext.receiver + assert interp.w_active_context.top() == interp.w_active_context.receiver def test_pushReceiverVariableBytecode(bytecode = (pushReceiverVariableBytecode(0) + pushReceiverVariableBytecode(1) + @@ -87,36 +87,36 @@ interp.step() interp.step() interp.step() - assert interp.activeContext.stack == ["egg", "bar", "baz"] + assert interp.w_active_context.stack == ["egg", "bar", "baz"] def test_pushTemporaryVariableBytecode(bytecode=(pushTemporaryVariableBytecode(0) + pushTemporaryVariableBytecode(1) + pushTemporaryVariableBytecode(2))): interp = new_interpreter(bytecode) - interp.activeContext.settemp(2, "temp") + interp.w_active_context.settemp(2, "temp") interp.step() interp.step() interp.step() - assert interp.activeContext.stack == ["foo", "bar", "temp"] + assert interp.w_active_context.stack == ["foo", "bar", "temp"] def test_pushLiteralConstantBytecode(bytecode=pushLiteralConstantBytecode(0) + pushLiteralConstantBytecode(1) + pushLiteralConstantBytecode(2)): interp = new_interpreter(bytecode) - interp.activeContext.method.literals = fakeliterals("a", "b", "c") + interp.w_active_context.method.literals = fakeliterals("a", "b", "c") interp.step() interp.step() interp.step() - assert interp.activeContext.stack == ["a", "b", "c"] + assert interp.w_active_context.stack == ["a", "b", "c"] def test_pushLiteralVariableBytecode(bytecode=pushLiteralVariableBytecode(0)): w_association = mockclass(2).as_class_get_shadow().new() w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(bytecode) - interp.activeContext.method.literals = fakeliterals(w_association) + interp.w_active_context.method.literals = fakeliterals(w_association) interp.step() - assert interp.activeContext.stack == ["myvalue"] + assert interp.w_active_context.stack == ["myvalue"] def test_storeAndPopReceiverVariableBytecode(bytecode=storeAndPopReceiverVariableBytecode, popped=True): @@ -124,13 +124,13 @@ for index in range(8): w_object = shadow.new() interp = new_interpreter(pushConstantTrueBytecode + bytecode(index)) - interp.activeContext.receiver = w_object + interp.w_active_context.receiver = w_object interp.step() interp.step() if popped: - assert interp.activeContext.stack == [] + assert interp.w_active_context.stack == [] else: - assert interp.activeContext.stack == [interp.TRUE] + assert interp.w_active_context.stack == [interp.TRUE] for test_index in range(8): if test_index == index: @@ -141,62 +141,62 @@ def test_storeAndPopTemporaryVariableBytecode(bytecode=storeAndPopTemporaryVariableBytecode): for index in range(8): interp = new_interpreter(pushConstantTrueBytecode + bytecode(index)) - interp.activeContext.temps = [None] * 8 + interp.w_active_context.temps = [None] * 8 interp.step() interp.step() - assert interp.activeContext.stack == [] + assert interp.w_active_context.stack == [] for test_index in range(8): if test_index == index: - assert interp.activeContext.temps[test_index] == interp.TRUE + assert interp.w_active_context.temps[test_index] == interp.TRUE else: - assert interp.activeContext.temps[test_index] == None + assert interp.w_active_context.temps[test_index] == None def test_pushConstantTrueBytecode(): interp = new_interpreter(pushConstantTrueBytecode) interp.step() - assert interp.activeContext.top() == interp.TRUE + assert interp.w_active_context.top() == interp.TRUE def test_pushConstantFalseBytecode(): interp = new_interpreter(pushConstantFalseBytecode) interp.step() - assert interp.activeContext.top() == interp.FALSE + assert interp.w_active_context.top() == interp.FALSE def test_pushConstantNilBytecode(): interp = new_interpreter(pushConstantNilBytecode) interp.step() - assert interp.activeContext.top() == interp.NIL + assert interp.w_active_context.top() == interp.NIL def test_pushConstantMinusOneBytecode(): interp = new_interpreter(pushConstantMinusOneBytecode) interp.step() - assert interp.activeContext.top() == interp.MONE + assert interp.w_active_context.top() == interp.MONE def test_pushConstantZeroBytecode(): interp = new_interpreter(pushConstantZeroBytecode) interp.step() - assert interp.activeContext.top() == interp.ZERO + assert interp.w_active_context.top() == interp.ZERO def test_pushConstantOneBytecode(): interp = new_interpreter(pushConstantOneBytecode) interp.step() - assert interp.activeContext.top() == interp.ONE + assert interp.w_active_context.top() == interp.ONE def test_pushConstantTwoBytecode(): interp = new_interpreter(pushConstantTwoBytecode) interp.step() - assert interp.activeContext.top() + assert interp.w_active_context.top() def test_pushActiveContextBytecode(): interp = new_interpreter(pushActiveContextBytecode) interp.step() - assert interp.activeContext.top() == interp.activeContext + assert interp.w_active_context.top() == interp.w_active_context def test_duplicateTopBytecode(): interp = new_interpreter(pushConstantZeroBytecode + duplicateTopBytecode) interp.step() interp.step() - assert interp.activeContext.stack == [interp.ZERO, interp.ZERO] + assert interp.w_active_context.stack == [interp.ZERO, interp.ZERO] # w_class - the class from which the method is going to be called # (and on which it is going to be installed) @@ -212,19 +212,19 @@ shadow.installmethod("foo", model.W_CompiledMethod(0, pushConstantOneBytecode + bytecode)) interp = new_interpreter(bytecodes) - interp.activeContext.method.literals = fakeliterals("foo") - interp.activeContext.push(w_object) - callerContext = interp.activeContext + interp.w_active_context.method.literals = fakeliterals("foo") + interp.w_active_context.push(w_object) + callerContext = interp.w_active_context interp.step() - assert interp.activeContext.sender == callerContext - assert interp.activeContext.stack == [] - assert interp.activeContext.receiver == w_object - assert interp.activeContext.method == shadow.methoddict["foo"] + assert interp.w_active_context.sender == callerContext + assert interp.w_active_context.stack == [] + assert interp.w_active_context.receiver == w_object + assert interp.w_active_context.method == shadow.methoddict["foo"] assert callerContext.stack == [] interp.step() interp.step() - assert interp.activeContext == callerContext - assert interp.activeContext.stack == [result] + assert interp.w_active_context == callerContext + assert interp.w_active_context.stack == [result] def test_sendLiteralSelectorBytecode(): w_class = mockclass(0) @@ -239,9 +239,9 @@ shadow.installmethod("fib:", method) w_object = shadow.new() interp = new_interpreter(sendLiteralSelectorBytecode(16) + returnTopFromMethod) - interp.activeContext.method.literals = fakeliterals("fib:") - interp.activeContext.push(w_object) - interp.activeContext.push(wrap_int(8)) + interp.w_active_context.method.literals = fakeliterals("fib:") + interp.w_active_context.push(w_object) + interp.w_active_context.push(wrap_int(8)) result = interp.interpret() assert primitives.unwrap_int(result) == 34 @@ -252,72 +252,72 @@ s_smallintclass.installmethod("sub", prim_meth) try: interp = new_interpreter(sendLiteralSelectorBytecode(1 + 16)) - interp.activeContext.method.literals = fakeliterals("foo", "sub") - interp.activeContext.push(wrap_int(50)) - interp.activeContext.push(wrap_int(8)) - callerContext = interp.activeContext + interp.w_active_context.method.literals = fakeliterals("foo", "sub") + interp.w_active_context.push(wrap_int(50)) + interp.w_active_context.push(wrap_int(8)) + callerContext = interp.w_active_context interp.step() - assert interp.activeContext is callerContext - assert len(interp.activeContext.stack) == 1 - w_result = interp.activeContext.pop() + assert interp.w_active_context is callerContext + assert len(interp.w_active_context.stack) == 1 + w_result = interp.w_active_context.pop() assert primitives.unwrap_int(w_result) == 42 finally: del s_smallintclass.methoddict['sub'] # clean up after you def test_longJumpIfTrue(): interp = new_interpreter(longJumpIfTrue(0) + chr(15) + longJumpIfTrue(0) + chr(15)) - interp.activeContext.push(interp.FALSE) - pc = interp.activeContext.pc + 2 + interp.w_active_context.push(interp.FALSE) + pc = interp.w_active_context.pc + 2 interp.step() - assert interp.activeContext.pc == pc - interp.activeContext.push(interp.TRUE) - pc = interp.activeContext.pc + 2 + assert interp.w_active_context.pc == pc + interp.w_active_context.push(interp.TRUE) + pc = interp.w_active_context.pc + 2 interp.step() - assert interp.activeContext.pc == pc + 15 + assert interp.w_active_context.pc == pc + 15 def test_longJumpIfFalse(): interp = new_interpreter(pushConstantTrueBytecode + longJumpIfFalse(0) + chr(15) + pushConstantFalseBytecode + longJumpIfFalse(0) + chr(15)) interp.step() - pc = interp.activeContext.pc + 2 + pc = interp.w_active_context.pc + 2 interp.step() - assert interp.activeContext.pc == pc + assert interp.w_active_context.pc == pc interp.step() - pc = interp.activeContext.pc + 2 + pc = interp.w_active_context.pc + 2 interp.step() - assert interp.activeContext.pc == pc + 15 + assert interp.w_active_context.pc == pc + 15 def test_longUnconditionalJump(): interp = new_interpreter(longUnconditionalJump(0) + chr(15)) - pc = interp.activeContext.pc + 2 + pc = interp.w_active_context.pc + 2 interp.step() - assert interp.activeContext.pc == pc + 15 + assert interp.w_active_context.pc == pc + 15 def test_shortUnconditionalJump(): interp = new_interpreter(chr(145)) - pc = interp.activeContext.pc + 1 + pc = interp.w_active_context.pc + 1 interp.step() - assert interp.activeContext.pc == pc + 2 + assert interp.w_active_context.pc == pc + 2 def test_shortConditionalJump(): interp = new_interpreter(pushConstantTrueBytecode + shortConditionalJump(3) + pushConstantFalseBytecode + shortConditionalJump(3)) interp.step() - pc = interp.activeContext.pc + 1 + pc = interp.w_active_context.pc + 1 interp.step() - assert interp.activeContext.pc == pc + assert interp.w_active_context.pc == pc interp.step() - pc = interp.activeContext.pc + 1 + pc = interp.w_active_context.pc + 1 interp.step() - assert interp.activeContext.pc == pc + 4 + assert interp.w_active_context.pc == pc + 4 def test_popStackBytecode(): interp = new_interpreter(pushConstantTrueBytecode + popStackBytecode) interp.step() - assert interp.activeContext.stack == [interp.TRUE] + assert interp.w_active_context.stack == [interp.TRUE] interp.step() - assert interp.activeContext.stack == [] + assert interp.w_active_context.stack == [] def test_extendedPushBytecode(): test_pushReceiverVariableBytecode(extendedPushBytecode + chr((0<<6) + 0) + @@ -339,7 +339,7 @@ w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(pushConstantOneBytecode + bytecode) - interp.activeContext.method.literals = fakeliterals(w_association) + interp.w_active_context.method.literals = fakeliterals(w_association) interp.step() interp.step() assert w_association.fetch(1) == interp.ONE @@ -360,13 +360,13 @@ shadow = mockclass(0).as_class_get_shadow() shadow.installmethod("+", model.W_CompiledMethod(1, "", 1)) w_object = shadow.new() - interp.activeContext.push(w_object) - interp.activeContext.push(interp.ONE) + interp.w_active_context.push(w_object) + interp.w_active_context.push(interp.ONE) interp.step() - assert interp.activeContext.method == shadow.methoddict["+"] - assert interp.activeContext.receiver is w_object - assert interp.activeContext.gettemp(0) == interp.ONE - assert interp.activeContext.stack == [] + assert interp.w_active_context.method == shadow.methoddict["+"] + assert interp.w_active_context.receiver is w_object + assert interp.w_active_context.gettemp(0) == interp.ONE + assert interp.w_active_context.stack == [] def test_bytecodePrimBool(): interp = new_interpreter(bytecodePrimLessThan + @@ -376,10 +376,10 @@ bytecodePrimEqual + bytecodePrimNotEqual) for i in range(6): - interp.activeContext.push(interp.ONE) - interp.activeContext.push(interp.TWO) + interp.w_active_context.push(interp.ONE) + interp.w_active_context.push(interp.TWO) interp.step() - assert interp.activeContext.stack == [interp.TRUE, interp.FALSE, + assert interp.w_active_context.stack == [interp.TRUE, interp.FALSE, interp.TRUE, interp.FALSE, interp.FALSE, interp.TRUE] @@ -406,16 +406,16 @@ meth1.literals = fakeliterals("foo") meth2.literals = fakeliterals("foo") interp = new_interpreter(bytecodes) - interp.activeContext.method.literals = fakeliterals("foo") - interp.activeContext.push(w_object) + interp.w_active_context.method.literals = fakeliterals("foo") + interp.w_active_context.push(w_object) for w_specificclass in [w_class, w_super, w_supersuper]: - callerContext = interp.activeContext + callerContext = interp.w_active_context interp.step() - assert interp.activeContext.sender == callerContext - assert interp.activeContext.stack == [] - assert interp.activeContext.receiver == w_object + assert interp.w_active_context.sender == callerContext + assert interp.w_active_context.stack == [] + assert interp.w_active_context.receiver == w_object meth = w_specificclass.as_class_get_shadow().methoddict["foo"] - assert interp.activeContext.method == meth + assert interp.w_active_context.method == meth assert callerContext.stack == [] def test_secondExtendedSendBytecode(): @@ -446,3 +446,17 @@ test_storeAndPopReceiverVariableBytecode(lambda index: doubleExtendedDoAnythingBytecode + chr(6<<5) + chr(index)) storeAssociation(doubleExtendedDoAnythingBytecode + chr(7<<5) + chr(0)) + +def test_block_copy_and_value(): + + py.test.skip("waiting for bytecode") + + bc_3_plus_4 = None + bc_x_plus_x_plus_1 = None + bc_x_plus_y = None + + for bcode in [ bc_3_plus_4, bc_x_plus_x_plus_1, bc_x_plus_y ]: + interp = new_interpreter(bcode) + res = interp.interpret() + assert res == wrap_int(7) + Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 14:09:26 2007 @@ -186,14 +186,14 @@ assert w_method w_frame = w_method.createFrame(w_object, []) - interp.activeContext = w_frame + interp.w_active_context = w_frame print w_method while True: try: interp.step() - print interp.activeContext.stack + print interp.w_active_context.stack except interpreter.ReturnFromTopLevel, e: return e.object Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 14:09:26 2007 @@ -26,23 +26,23 @@ mapped_stack = [wrap(x) for x in stack] frame = MockFrame(mapped_stack) interp = interpreter.Interpreter() - interp.activeContext = frame + interp.w_active_context = frame return p.Args(interp, len(stack)) def prim(code, stack): args = mock(stack) res = prim_table[code](args) - assert not len(args.interp.activeContext.stack) # check args are consumed + assert not len(args.interp.w_active_context.stack) # check args are consumed return res def prim_fails(code, stack): args = mock(stack) - orig_stack = list(args.interp.activeContext.stack) + orig_stack = list(args.interp.w_active_context.stack) try: prim_table[code](args) py.test.fail("Expected PrimitiveFailedError") except PrimitiveFailedError: - assert args.interp.activeContext.stack == orig_stack + assert args.interp.w_active_context.stack == orig_stack # smallinteger tests def test_small_int_add(): @@ -255,3 +255,5 @@ assert prim(p.FLOAT_EQUAL, [2.2,2.2]) == objtable.w_true assert prim(p.FLOAT_NOTEQUAL, [2.2,2.2]) == objtable.w_false +def test_block_copy_and_value(): + # see test_interpreter for tests of these opcodes From lukas at codespeak.net Thu Oct 25 14:12:12 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Thu, 25 Oct 2007 14:12:12 +0200 (CEST) Subject: [pypy-svn] r47916 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071025121212.D123E8125@code0.codespeak.net> Author: lukas Date: Thu Oct 25 14:12:12 2007 New Revision: 47916 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: added bytecodes for block copy and value primtives Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Thu Oct 25 14:12:12 2007 @@ -451,9 +451,9 @@ py.test.skip("waiting for bytecode") - bc_3_plus_4 = None - bc_x_plus_x_plus_1 = None - bc_x_plus_y = None + bc_3_plus_4 = [ 137, 117, 200, 164, 4, 32, 33, 176, 125, 201, 124] + bc_x_plus_x_plus_1 = [ 137, 118, 200, 164, 7, 104, 16, 16, 176, 118, 176, 125, 32, 202, 124 ] + bc_x_plus_y = [ 137, 119, 200, 164, 6, 105, 104, 16, 17, 176, 125, 33, 34, 240, 124 ] for bcode in [ bc_3_plus_4, bc_x_plus_x_plus_1, bc_x_plus_y ]: interp = new_interpreter(bcode) From niko at codespeak.net Thu Oct 25 14:15:10 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 25 Oct 2007 14:15:10 +0200 (CEST) Subject: [pypy-svn] r47917 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071025121510.97C21812B@code0.codespeak.net> Author: niko Date: Thu Oct 25 14:15:10 2007 New Revision: 47917 Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: fix syntax error Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 14:15:10 2007 @@ -257,3 +257,4 @@ def test_block_copy_and_value(): # see test_interpreter for tests of these opcodes + return From cfbolz at codespeak.net Thu Oct 25 14:15:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 25 Oct 2007 14:15:40 +0200 (CEST) Subject: [pypy-svn] r47918 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025121540.06389812E@code0.codespeak.net> Author: cfbolz Date: Thu Oct 25 14:15:40 2007 New Revision: 47918 Modified: pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: (cfbolz, arigo, tverwaes): fix image loading: use prebuilt objects when filling the w_objects. Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Thu Oct 25 14:15:40 2007 @@ -73,3 +73,20 @@ SO_TRANSLATEDMETHOD_CLASS = 40 SO_FINALIZATION_SEMPAHORE = 41 SO_LARGENEGATIVEINTEGER_CLASS = 42 + +# XXX more missing? +classes_in_special_object_table = { + "SmallInteger": SO_SMALLINTEGER_CLASS, + "String": SO_STRING_CLASS, + "Float": SO_FLOAT_CLASS, + "MethodContext": SO_METHODCONTEXT_CLASS, + "Character": SO_CHARACTER_CLASS, + "ByteArray": SO_BYTEARRAY_CLASS, + "CompiledMethod": SO_COMPILEDMETHOD_CLASS, +} + +objects_in_special_object_table = { + "nil": SO_NIL, + "true": SO_TRUE, + "false": SO_FALSE, +} Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 14:15:40 2007 @@ -140,6 +140,7 @@ def lookup(self, methodname): # XXX kill me. Temporary, for testing + from pypy.lang.smalltalk import objtable in_class = self while in_class != None: try: @@ -148,12 +149,12 @@ # Current hack because we don't have a ref to the real # nil yet... XXX XXX XXX try: - new_class = in_class.vars[constants.CLASS_SUPERCLASS_INDEX] - if in_class == new_class: - raise Exception + new_class = in_class._vars[constants.CLASS_SUPERCLASS_INDEX] + if new_class is objtable.w_nil: + raise IndexError else: in_class = new_class - except: + except IndexError: return self.lookup("doesNotUnderstand") class W_BytesObject(W_AbstractObjectWithClassReference): Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 14:15:40 2007 @@ -82,7 +82,6 @@ self.init_compactclassesarray() self.init_g_objects() self.init_w_objects() - self.assign_mirrors() self.fillin_w_objects() def read_header(self): @@ -113,29 +112,24 @@ chunk.as_g_object(self) # initialized g_object def init_w_objects(self): + self.assign_prebuilt_constants() for chunk in self.chunks.itervalues(): chunk.g_object.init_w_object() - def assign_mirrors(self): - # assign the mirrors to the classes already in classtable - from pypy.lang.smalltalk import classtable, constants - import py; py.test.skip("FIX ME") - for so_index, name in [ - (constants.SO_SMALLINTEGER_CLASS, "m_SmallInteger"), - (constants.SO_STRING_CLASS, "m_String"), - (constants.SO_FLOAT_CLASS, "m_Float"), - #(constants.SO_METHODCONTEXT_CLASS, "m_MethodContext"), - (constants.SO_CHARACTER_CLASS, "m_Character"), - (constants.SO_BYTEARRAY_CLASS, "m_ByteArray"), - (constants.SO_COMPILEDMETHOD_CLASS, "m_CompiledMethod")]: - mirrorcache.assign_existing_mirror( - self.special_object(so_index), - getattr(classtable, name)) - # XXX more missing + def assign_prebuilt_constants(self): + from pypy.lang.smalltalk import classtable, constants, objtable + # assign w_objects for objects that are already in classtable + for name, so_index in constants.classes_in_special_object_table.items(): + w_object = getattr(classtable, "w_" + name) + self.special_object(so_index).w_object = w_object + # assign w_objects for objects that are already in objtable + for name, so_index in constants.objects_in_special_object_table.items(): + w_object = getattr(objtable, "w_" + name) + self.special_object(so_index).w_object = w_object def special_object(self, index): special = self.chunks[self.specialobjectspointer].g_object.pointers - return special[index].w_object + return special[index] def fillin_w_objects(self): for chunk in self.chunks.itervalues(): @@ -316,13 +310,13 @@ def fillin_pointersobject(self, w_pointersobject): assert self.pointers is not None - w_pointersobject.vars = [g_object.w_object for g_object in self.pointers] - w_pointersobject.m_class = mirrorcache.get_or_build(self.g_class.w_object) + w_pointersobject._vars = [g_object.w_object for g_object in self.pointers] + w_pointersobject.w_class = self.g_class.w_object w_pointersobject.hash = self.chunk.hash12 def fillin_wordsobject(self, w_wordsobject): w_wordsobject.words = self.chunk.data - w_wordsobject.m_class = mirrorcache.get_or_build(self.g_class.w_object) + w_wordsobject.w_class = self.g_class.w_object w_wordsobject.hash = self.chunk.hash12 # XXX check this def fillin_bytesobject(self, w_bytesobject): @@ -332,7 +326,7 @@ bbytes.append(chr((each >> 16) & 0xff)) bbytes.append(chr((each >> 8) & 0xff)) bbytes.append(chr((each >> 0) & 0xff)) - w_bytesobject.m_class = mirrorcache.get_or_build(self.g_class.w_object) + w_bytesobject.w_class = self.g_class.w_object #strange, for example range(4)[:0] returns [] instead of [0,1,2,3]! #hence what we have to write list[:-odd] as list[:len(list)-odd] instead :( w_bytesobject.bytes = bbytes[:len(bbytes)-(self.format & 3)] # omit odd bytes Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 14:15:40 2007 @@ -181,8 +181,8 @@ # class. Should work using classmirrors when the metaclass of # SmallInt is correctly set - # w_classmirror = w_object.getclassmirror() - # w_method = w_classmirror.lookup("abs") + # s_class = w_object.shadow_of_my_class() + # w_method = s_class.lookup("abs") assert w_method w_frame = w_method.createFrame(w_object, []) @@ -198,14 +198,18 @@ return e.object def test_lookup_neg_abs_in_integer(): - py.test.skip("TOFIX methodlookup 'negated' fails in mirror SmallInteger") + py.test.skip("TOFIX methodlookup 'negated' fails in shadow SmallInteger") # Fails due to same reason because of which # classmirror-methodlookup fails test_lookup_abs_in_integer(-3) def test_map_mirrors_to_classtable(): - from pypy.lang.smalltalk import classtable, mirror + from pypy.lang.smalltalk import classtable, shadow, objtable w_compiledmethod_class = image.special(constants.SO_COMPILEDMETHOD_CLASS) - m_compiledmethod_class = mirror.mirrorcache.getmirror( - w_compiledmethod_class) - assert m_compiledmethod_class is classtable.m_CompiledMethod + assert w_compiledmethod_class is classtable.w_CompiledMethod + w_nil = image.special(constants.SO_NIL) + assert w_nil is objtable.w_nil + w_true = image.special(constants.SO_TRUE) + assert w_true is objtable.w_true + w_false = image.special(constants.SO_FALSE) + assert w_false is objtable.w_false From akuhn at codespeak.net Thu Oct 25 14:17:26 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Thu, 25 Oct 2007 14:17:26 +0200 (CEST) Subject: [pypy-svn] r47919 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025121726.3D14E8132@code0.codespeak.net> Author: akuhn Date: Thu Oct 25 14:17:25 2007 New Revision: 47919 Modified: pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (akuhn, oscar) implemented three missing float primitives Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Thu Oct 25 14:17:25 2007 @@ -52,7 +52,9 @@ SO_CHARACTER_CLASS = 19 SO_DOES_NOT_UNDERSTAND = 20 SO_CANNOT_RETURN = 21 -# no clue what 22 is doing + +# XXX no clue what 22 is doing, lookup in Squeak: ObjectMemory >> initializeSpecialObjectIndices + SO_SPECIAL_SELECTORS_ARRAY = 23 SO_CHARACTER_TABLE_ARRAY = 24 SO_MUST_BE_BOOLEAN = 25 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 14:17:25 2007 @@ -179,18 +179,32 @@ _FLOAT_OFFSET = 40 FLOAT_ADD = 41 FLOAT_SUBTRACT = 42 +FLOAT_MULTIPLY = 49 +FLOAT_DIVIDE = 50 +FLOAT_TRUNCATED = 51 + math_ops = { FLOAT_ADD: operator.add, - FLOAT_SUBTRACT: operator.sub + FLOAT_SUBTRACT: operator.sub, + FLOAT_MULTIPLY: operator.mul, + FLOAT_DIVIDE: operator.div, } for (code,op) in math_ops.items(): + @primitive(code) @stack(2) def func(args, (w_v1, w_v2), op=op): # n.b. capture op value v1 = unwrap_float(w_v1) v2 = unwrap_float(w_v2) w_res = objtable.wrap_float(op(v1, v2)) return w_res - prim_table[code] = func + + at primitive(FLOAT_TRUNCATED) + at stack(1) +def func(args, (w_float,)): + f = unwrap_float(w_float) + w_res = objtable.wrap_int(int(f)) + return w_res + # ___________________________________________________________________________ # Subscript and Stream Primitives @@ -571,6 +585,7 @@ @primitive(PRIMITIVE_PERFORM) @stack(2) def func(args, (w_rcvr, w_sel)): + # XXX we can implement this when lookup on shadow class is done raise PrimitiveNotYetWrittenError() @primitive(PRIMITIVE_PERFORM_WITH_ARGS) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 14:17:25 2007 @@ -139,9 +139,30 @@ prim_fails(p.BIT_SHIFT, [4, 29]) prim_fails(p.BIT_SHIFT, [4, 28]) -def test_float(): +def test_float_add(): assert prim(p.FLOAT_ADD, [1.0,2.0]).value == 3.0 - assert prim(p.FLOAT_ADD, [3,4.5]).value == 7.5 + assert prim(p.FLOAT_ADD, [3.0,4.5]).value == 7.5 + +def test_float_subtract(): + assert prim(p.FLOAT_SUBTRACT, [1.0,2.0]).value == -1.0 + assert prim(p.FLOAT_SUBTRACT, [15.0,4.5]).value == 10.5 + +def test_float_multiply(): + assert prim(p.FLOAT_MULTIPLY, [10.0,2.0]).value == 20.0 + assert prim(p.FLOAT_MULTIPLY, [3.0,4.5]).value == 13.5 + +def test_float_divide(): + assert prim(p.FLOAT_DIVIDE, [1.0,2.0]).value == 0.5 + assert prim(p.FLOAT_DIVIDE, [3.5,4.0]).value == 0.875 + +def test_float_truncate(): + assert prim(p.FLOAT_TRUNCATED, [-4.6]).value == -4 + assert prim(p.FLOAT_TRUNCATED, [-4.5]).value == -4 + assert prim(p.FLOAT_TRUNCATED, [-4.4]).value == -4 + assert prim(p.FLOAT_TRUNCATED, [4.4]).value == 4 + assert prim(p.FLOAT_TRUNCATED, [4.5]).value == 4 + assert prim(p.FLOAT_TRUNCATED, [4.6]).value == 4 + def test_at(): w_obj = mockclass(0, varsized=True).as_class_get_shadow().new(1) From arigo at codespeak.net Thu Oct 25 15:07:28 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 15:07:28 +0200 (CEST) Subject: [pypy-svn] r47920 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025130728.B8CEA8134@code0.codespeak.net> Author: arigo Date: Thu Oct 25 15:07:27 2007 New Revision: 47920 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (arigo, all watching) Some fixes found by trying to translate it. Not there yet. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 15:07:27 2007 @@ -327,7 +327,7 @@ self.callPrimitiveAndPush(primitives.NOTEQUAL, "~=", 1, interp) def bytecodePrimMultiply(self, interp): - self.callPrimitiveAndPush(primitives.MUL, "*", 1, interp) + self.callPrimitiveAndPush(primitives.MULTIPLY, "*", 1, interp) def bytecodePrimDivide(self, interp): self.callPrimitiveAndPush(primitives.DIVIDE, "/", 1, interp) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 15:07:27 2007 @@ -224,8 +224,7 @@ """ def __init__(self, literalsize, bytes, argsize=0, tempsize=0, primitive=0, w_compiledin=None): - # self.literals = [None] * size - self.literalsize = literalsize + self.literals = [None] * literalsize self.w_compiledin = w_compiledin self.bytes = bytes self.argsize = argsize Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 15:07:27 2007 @@ -97,16 +97,18 @@ BIT_XOR: operator.xor } for (code,op) in math_ops.items(): - @primitive(code) - @stack(2) - def func(args, (w_receiver, w_argument), op=op): # n.b. capture op value - receiver = unwrap_int(w_receiver) - argument = unwrap_int(w_argument) - try: - res = rarithmetic.ovfcheck(op(receiver, argument)) - except OverflowError: - raise PrimitiveFailedError() - return wrap_int(res) + def make_func(op): + @primitive(code) + @stack(2) + def func(args, (w_receiver, w_argument)): + receiver = unwrap_int(w_receiver) + argument = unwrap_int(w_argument) + try: + res = rarithmetic.ovfcheck(op(receiver, argument)) + except OverflowError: + raise PrimitiveFailedError() + return wrap_int(res) + make_func(op) # #/ -- return the result of a division, only succeed if the division is exact @primitive(DIVIDE) @@ -190,13 +192,15 @@ FLOAT_DIVIDE: operator.div, } for (code,op) in math_ops.items(): - @primitive(code) - @stack(2) - def func(args, (w_v1, w_v2), op=op): # n.b. capture op value - v1 = unwrap_float(w_v1) - v2 = unwrap_float(w_v2) - w_res = objtable.wrap_float(op(v1, v2)) - return w_res + def make_func(op): + @primitive(code) + @stack(2) + def func(args, (w_v1, w_v2)): + v1 = unwrap_float(w_v1) + v2 = unwrap_float(w_v2) + w_res = objtable.wrap_float(op(v1, v2)) + return w_res + make_func(op) @primitive(FLOAT_TRUNCATED) @stack(1) @@ -456,24 +460,28 @@ NOTEQUAL: operator.ne } for (code,op) in bool_ops.items(): - @primitive(code) - @stack(2) - def func(args, (w_v1, w_v2), op=op): # n.b. capture op value - v1 = unwrap_int(w_v1) - v2 = unwrap_int(w_v2) - res = op(v1, v2) - w_res = objtable.wrap_bool(res) - return w_res + def make_func(op): + @primitive(code) + @stack(2) + def func(args, (w_v1, w_v2)): + v1 = unwrap_int(w_v1) + v2 = unwrap_int(w_v2) + res = op(v1, v2) + w_res = objtable.wrap_bool(res) + return w_res + make_func(op) for (code,op) in bool_ops.items(): - @primitive(code+_FLOAT_OFFSET) - @stack(2) - def func(args, (w_v1, w_v2), op=op): # n.b. capture op value - v1 = unwrap_float(w_v1) - v2 = unwrap_float(w_v2) - res = op(v1, v2) - w_res = objtable.wrap_bool(res) - return w_res + def make_func(op): + @primitive(code+_FLOAT_OFFSET) + @stack(2) + def func(args, (w_v1, w_v2)): + v1 = unwrap_float(w_v1) + v2 = unwrap_float(w_v2) + res = op(v1, v2) + w_res = objtable.wrap_bool(res) + return w_res + make_func(op) # ___________________________________________________________________________ # Quick Push Const Primitives @@ -526,6 +534,7 @@ @primitive(PRIMITIVE_BLOCK_COPY) @stack(2) def func(args, (w_argcnt, w_context)): + raise PrimitiveNotYetWrittenError() frame = args.interp.w_active_context # From B.B.: If receiver is a MethodContext, then it becomes @@ -550,6 +559,7 @@ @primitive(PRIMITIVE_VALUE) def func(args): + raise PrimitiveNotYetWrittenError() # If nargs == 4, stack looks like: # 3 2 1 0 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Thu Oct 25 15:07:27 2007 @@ -1,5 +1,6 @@ import py from pypy.lang.smalltalk import model, interpreter, primitives, shadow +from pypy.lang.smalltalk import objtable from pypy.lang.smalltalk.objtable import wrap_int import pypy.lang.smalltalk.classtable as ct @@ -28,7 +29,7 @@ def fakeliterals(*literals): return ["methodheader"] + list(literals) -def new_interpreter(bytes, receiver="receiver"): +def new_interpreter(bytes, receiver=objtable.w_nil): assert isinstance(bytes, str) w_method = model.W_CompiledMethod(0, bytes=bytes, argsize=2, tempsize=1) From arigo at codespeak.net Thu Oct 25 15:09:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 15:09:24 +0200 (CEST) Subject: [pypy-svn] r47921 - pypy/dist/pypy/translator/goal Message-ID: <20071025130924.612418135@code0.codespeak.net> Author: arigo Date: Thu Oct 25 15:09:23 2007 New Revision: 47921 Added: pypy/dist/pypy/translator/goal/targetfibsmalltalk.py (contents, props changed) Log: (arigo, all watching) The target for running Fibonacci in the Smalltalk interpreter. Added: pypy/dist/pypy/translator/goal/targetfibsmalltalk.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/goal/targetfibsmalltalk.py Thu Oct 25 15:09:23 2007 @@ -0,0 +1,42 @@ +from pypy.lang.smalltalk import model, interpreter, primitives, shadow +from pypy.lang.smalltalk import objtable +from pypy.lang.smalltalk.objtable import wrap_int +from pypy.lang.smalltalk import classtable +from pypy.lang.smalltalk.test.test_interpreter import * + +mockclass = classtable.bootstrap_class + +def new_interpreter(bytes): + assert isinstance(bytes, str) + w_method = model.W_CompiledMethod(0, bytes=bytes) + w_frame = w_method.createFrame(objtable.w_nil, []) + interp = interpreter.Interpreter() + interp.w_active_context = w_frame + return interp + + +bytecode = ''.join(map(chr, [ 16, 119, 178, 154, 118, 164, 11, 112, 16, 118, 177, 224, 112, 16, 119, 177, 224, 176, 124 ])) +shadow = mockclass(0).as_class_get_shadow() +method = model.W_CompiledMethod(1, bytecode, 1) +method.literals = fakeliterals("fib:") +shadow.installmethod("fib:", method) +w_object = shadow.new() +interp = new_interpreter(sendLiteralSelectorBytecode(16) + returnTopFromMethod) +interp.w_active_context.method.literals = fakeliterals("fib:") + + +def entry_point(argv): + if len(argv) > 1: + n = int(argv[1]) + else: + n = 8 + interp.w_active_context.push(w_object) + interp.w_active_context.push(wrap_int(n)) + result = interp.interpret() + print primitives.unwrap_int(result) + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None From cfbolz at codespeak.net Thu Oct 25 15:13:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 25 Oct 2007 15:13:52 +0200 (CEST) Subject: [pypy-svn] r47922 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025131352.4D3748137@code0.codespeak.net> Author: cfbolz Date: Thu Oct 25 15:13:52 2007 New Revision: 47922 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py Log: method contexts just use identity hashing Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 15:13:52 2007 @@ -10,7 +10,7 @@ """Illegal Store.""" -class W_MethodContext(model.W_Object): +class W_MethodContext(model.W_AbstractObjectWithIdentityHash): def __init__(self, method, receiver, arguments, sender = None): self.method = method self.receiver = receiver @@ -23,9 +23,6 @@ from pypy.lang.smalltalk.classtable import w_MethodContext return w_MethodContext - def gethash(self): - return 44 # XXX - def pop(self): return self.stack.pop() From cfbolz at codespeak.net Thu Oct 25 15:16:41 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 25 Oct 2007 15:16:41 +0200 (CEST) Subject: [pypy-svn] r47923 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025131641.E0E8C813A@code0.codespeak.net> Author: cfbolz Date: Thu Oct 25 15:16:40 2007 New Revision: 47923 Modified: pypy/dist/pypy/lang/smalltalk/model.py Log: W_CompiledMethod gets the correct hash from W_AbstractObjectIdentityHash Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 15:16:40 2007 @@ -235,9 +235,6 @@ from pypy.lang.smalltalk.classtable import w_CompiledMethod return w_CompiledMethod - def gethash(self): - return 43 # XXX - def getliteral(self, index): return self.literals[index + 1] # header of compiledmethod at index 0 From niko at codespeak.net Thu Oct 25 15:17:09 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 25 Oct 2007 15:17:09 +0200 (CEST) Subject: [pypy-svn] r47924 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025131709.1B17F8137@code0.codespeak.net> Author: niko Date: Thu Oct 25 15:17:08 2007 New Revision: 47924 Modified: pypy/dist/pypy/lang/smalltalk/classtable.py pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (niko, lukas, tudor) commit some intermediate work toward getting the block_copy_and_value to work Modified: pypy/dist/pypy/lang/smalltalk/classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/classtable.py Thu Oct 25 15:17:08 2007 @@ -1,4 +1,5 @@ from pypy.lang.smalltalk import shadow +from pypy.lang.smalltalk import constants def bootstrap_class(instsize, w_superclass=None, w_metaclass=None, name='?', format=shadow.POINTERS, varsized=False): @@ -94,4 +95,7 @@ define_cls("w_False", "w_Boolean") define_cls("w_ByteArray", "w_ArrayedCollection", format=shadow.BYTES) define_cls("w_CompiledMethod", "w_ByteArray", format=shadow.COMPILED_METHOD) -define_cls("w_MethodContext", "w_Object") +define_cls("w_ContextPart", "w_Object") +define_cls("w_MethodContext", "w_ContextPart") +define_cls("w_BlockContext", "w_ContextPart", + instvarsize=constants.BLKCTX_TEMP_FRAME_START) Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Thu Oct 25 15:17:08 2007 @@ -20,13 +20,13 @@ ASSOCIATION_KEY_INDEX = 0 ASSOCIATION_VALUE_INDEX = 1 -BLOCK_CONTEXT_CALLER_INDEX = 0 -BLOCK_CONTEXT_INSTRUCTION_POINTER_INDEX = 1 -BLOCK_CONTEXT_STACK_POINTER_INDEX = 2 -BLOCK_CONTEXT_BLOCK_ARGUMENT_COUNT_INDEX = 3 -BLOCK_CONTEXT_INITIAL_IP_INDEX = 4 -BLOCK_CONTEXT_HOME_INDEX = 5 -BLOCK_CONTEXT_TEMP_FRAME_START = 6 +BLKCTX_CALLER_INDEX = 0 +BLKCTX_INSTRUCTION_POINTER_INDEX = 1 +BLKCTX_STACK_POINTER_INDEX = 2 +BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX = 3 +BLKCTX_INITIAL_IP_INDEX = 4 +BLKCTX_HOME_INDEX = 5 +BLKCTX_TEMP_FRAME_START = 6 # ----- special objects indices ------- Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 15:17:08 2007 @@ -372,10 +372,12 @@ raise MissingBytecode def bytecodePrimBlockCopy(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush( + primitives.PRIMITIVE_BLOCK_COPY, "blockCopy:", 1, interp) def bytecodePrimValue(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush( + primitives.PRIMITIVE_VALUE, "value", 0, interp) def bytecodePrimValueWithArg(self, interp): raise MissingBytecode Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 15:17:08 2007 @@ -4,6 +4,15 @@ from pypy.lang.smalltalk import objtable from pypy.rlib import rarithmetic +from pypy.lang.smalltalk.constants import \ + BLKCTX_CALLER_INDEX, \ + BLKCTX_INSTRUCTION_POINTER_INDEX, \ + BLKCTX_STACK_POINTER_INDEX, \ + BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX, \ + BLKCTX_INITIAL_IP_INDEX, \ + BLKCTX_HOME_INDEX, \ + BLKCTX_TEMP_FRAME_START + class PrimitiveFailedError(Exception): pass @@ -533,28 +542,29 @@ @primitive(PRIMITIVE_BLOCK_COPY) @stack(2) -def func(args, (w_argcnt, w_context)): +def func(args, (w_context, w_argcnt)): raise PrimitiveNotYetWrittenError() frame = args.interp.w_active_context # From B.B.: If receiver is a MethodContext, then it becomes # the new BlockContext's home context. Otherwise, the home # context of the receiver is used for the new BlockContext. - if isinstance(w_context, classtable.w_BlockContext): - w_method_context = w_context.fetch(BLOCK_CONTEXT_HOME_INDEX) + if w_context.getclass() == classtable.w_BlockContext: + w_method_context = w_context.fetch(BLKCTX_HOME_INDEX) else: w_method_context = w_context # The block bytecodes are stored inline: so we skip past the # byteodes to invoke this primitive to find them (hence +3) - w_new_context = classtable.w_BlockContext.new(unwrap_int(w_argcnt)) + w_new_context = classtable.w_BlockContext.as_class_get_shadow().new( + unwrap_int(w_argcnt)) initialip = frame.pc + 3 # Initialize various fields. - w_new_context.store(BLOCK_CONTEXT_INITIAL_IP_INDEX, initialip) - w_new_context.store(BLOCK_CONTEXT_INSTRUCTION_POINTER_INDEX, initialip) - w_new_context.store(BLOCK_CONTEXT_BLOCK_ARGUMENT_COUNT_INDEX, w_argcnt) - w_new_context.store(BLOCK_CONTEXT_HOME_INDEX, w_method_context) + w_new_context.store(BLKCTX_INITIAL_IP_INDEX, initialip) + w_new_context.store(BLKCTX_INSTRUCTION_POINTER_INDEX, initialip) + w_new_context.store(BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX, w_argcnt) + w_new_context.store(BLKCTX_HOME_INDEX, w_method_context) return w_new_context @primitive(PRIMITIVE_VALUE) @@ -568,23 +578,23 @@ w_block_ctx = args.interp.w_active_context.peek(args.argument_count-1) - w_exp_arg_cnt = w_block_ctx.fetch(BLOCK_CONTEXT_BLOCK_ARGUMENT_COUNT_INDEX) + w_exp_arg_cnt = w_block_ctx.fetch(BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX) exp_arg_cnt = unwrap_int(w_exp_arg_cnt) if args.argument_count != exp_arg_cnt: raise PrimitiveFailedError() # Copy the values from the stack such that the most recently pushed - # item (index 0) ends up in slot BLOCK_CONTEXT_TEMP_FRAME_START + nargs - 1 + # item (index 0) ends up in slot BLKCTX_TEMP_FRAME_START + nargs - 1 for idx in range(exp_arg_cnt - 1): w_block_ctx.store( - BLOCK_CONTEXT_TEMP_FRAME_START+idx, + BLKCTX_TEMP_FRAME_START+idx, w_block_ctx.fetch(exp_arg_cnt - idx - 1)) # Set some fields - w_initial_ip = w_block_ctx.fetch(BLOCK_CONTEXT_INITIAL_IP_INDEX) - w_block_ctx.store(BLOCK_CONTEXT_INSTRUCTION_POINTER_INDEX, w_initial_ip) - w_block_ctx.store(BLOCK_CONTEXT_STACK_POINTER_INDEX, w_exp_arg_cnt) - w_block_ctx.store(BLOCK_CONTEXT_CALLER_INDEX, args.interp.w_active_context) + w_initial_ip = w_block_ctx.fetch(BLKCTX_INITIAL_IP_INDEX) + w_block_ctx.store(BLKCTX_INSTRUCTION_POINTER_INDEX, w_initial_ip) + w_block_ctx.store(BLKCTX_STACK_POINTER_INDEX, w_exp_arg_cnt) + w_block_ctx.store(BLKCTX_CALLER_INDEX, args.interp.w_active_context) args.interp.w_active_context = w_block_ctx @primitive(PRIMITIVE_VALUE_WITH_ARGS) Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Thu Oct 25 15:17:08 2007 @@ -450,13 +450,14 @@ def test_block_copy_and_value(): - py.test.skip("waiting for bytecode") - + py.test.skip("block_copy_and_value not working yet") + bc_3_plus_4 = [ 137, 117, 200, 164, 4, 32, 33, 176, 125, 201, 124] bc_x_plus_x_plus_1 = [ 137, 118, 200, 164, 7, 104, 16, 16, 176, 118, 176, 125, 32, 202, 124 ] bc_x_plus_y = [ 137, 119, 200, 164, 6, 105, 104, 16, 17, 176, 125, 33, 34, 240, 124 ] - - for bcode in [ bc_3_plus_4, bc_x_plus_x_plus_1, bc_x_plus_y ]: + + for bcodes in [ bc_3_plus_4, bc_x_plus_x_plus_1, bc_x_plus_y ]: + bcode = "".join([chr(x) for x in bcodes]) interp = new_interpreter(bcode) res = interp.interpret() assert res == wrap_int(7) From akuhn at codespeak.net Thu Oct 25 15:41:54 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Thu, 25 Oct 2007 15:41:54 +0200 (CEST) Subject: [pypy-svn] r47925 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025134154.958488137@code0.codespeak.net> Author: akuhn Date: Thu Oct 25 15:41:52 2007 New Revision: 47925 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: (akuhn, tverwaes, arigo) added test to test for class of #doesNotUnderstand: in mini.image (which is, for some strange reason, an Array) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 15:41:52 2007 @@ -134,7 +134,7 @@ values = w_methoddict[constants.METHODDICT_VALUES_INDEX]._vars for var in names: if isinstance(var, W_BytesObject): - if str(var) == repr(methodname): + if str(var) == str(methodname): return values[names.index(var)] raise MethodNotFound @@ -155,7 +155,7 @@ else: in_class = new_class except IndexError: - return self.lookup("doesNotUnderstand") + return self.lookup("doesNotUnderstand:") class W_BytesObject(W_AbstractObjectWithClassReference): def __init__(self, w_class, size): @@ -172,7 +172,11 @@ return len(self.bytes) def __str__(self): - return repr("".join(self.bytes)) + return "".join(self.bytes) + + def __repr__(self): + return "" % ("".join(self.bytes),) + def invariant(self): if not W_AbstractObjectWithClassReference.invariant(self): Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 15:41:52 2007 @@ -105,7 +105,7 @@ image = get_image() w = image.special(constants.SO_SCHEDULERASSOCIATIONPOINTER) w0 = w.fetch(0) - assert str(w0) == "'Processor'" + assert str(w0) == "Processor" w0 = w.fetch(1) assert str(w0) == "a ProcessorScheduler" @@ -138,8 +138,16 @@ SO_COMPILEDMETHOD_CLASS = 16 SO_LOW_SPACE_SEMAPHORE = 17 SO_SEMAPHORE_CLASS = 18 - SO_CHARACTER_CLASS = 19 - SO_DOES_NOT_UNDERSTAND = 20 + SO_CHARACTER_CLASS = 19""" + +def test_special_classes0(): + image = get_image() + w = image.special(constants.SO_DOES_NOT_UNDERSTAND) + assert str(w) == "doesNotUnderstand:" + assert str(w.getclass()) == "Array class" # for some strange reason not a symbol + + + """SO_DOES_NOT_UNDERSTAND = 20 SO_CANNOT_RETURN = 21 # no clue what 22 is doing SO_SPECIAL_SELECTORS_ARRAY = 23 From oscar at codespeak.net Thu Oct 25 15:42:01 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Thu, 25 Oct 2007 15:42:01 +0200 (CEST) Subject: [pypy-svn] r47926 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025134201.78FD2813D@code0.codespeak.net> Author: oscar Date: Thu Oct 25 15:42:01 2007 New Revision: 47926 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (cfbolz, oscar) implemented squareroot and default primitive failing Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 15:42:01 2007 @@ -1,4 +1,5 @@ import operator +import math from pypy.lang.smalltalk import model, shadow from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import objtable @@ -46,7 +47,12 @@ # argument, an instance of the Args class below; the function either # completes, and returns a result, or throws a PrimitiveFailedError. -prim_table = [None] * 576 # Squeak has primitives all the way up to 575 +def raise_failing_default(args): + raise PrimitiveFailedError + +# Squeak has primitives all the way up to 575 +# So all optional primitives will default to the bytecode implementation +prim_table = [raise_failing_default] * 576 class Args: def __init__(self, interp, argument_count): @@ -193,6 +199,7 @@ FLOAT_MULTIPLY = 49 FLOAT_DIVIDE = 50 FLOAT_TRUNCATED = 51 +FLOAT_SQUARE_ROOT = 55 math_ops = { FLOAT_ADD: operator.add, @@ -218,6 +225,12 @@ w_res = objtable.wrap_int(int(f)) return w_res + at primitive(FLOAT_SQUARE_ROOT) + at stack(1) +def func(args, (w_float,)): + f = unwrap_float(w_float) + w_res = objtable.wrap_float(math.sqrt(f)) + return w_res # ___________________________________________________________________________ # Subscript and Stream Primitives Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 15:42:01 2007 @@ -279,3 +279,9 @@ def test_block_copy_and_value(): # see test_interpreter for tests of these opcodes return + +ROUNDING_DIGITS = 8 +def test_primitive_square_root(): + assert prim(p.FLOAT_SQUARE_ROOT, [4.0]).value == 2.0 + assert round(prim(p.FLOAT_SQUARE_ROOT, [2.0]).value,ROUNDING_DIGITS) == round(1.414213562373095,ROUNDING_DIGITS) + From oscar at codespeak.net Thu Oct 25 15:47:06 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Thu, 25 Oct 2007 15:47:06 +0200 (CEST) Subject: [pypy-svn] r47927 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025134706.42CCB8135@code0.codespeak.net> Author: oscar Date: Thu Oct 25 15:47:05 2007 New Revision: 47927 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (cbolz, oscar) failing test for square root and fix Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 15:47:05 2007 @@ -229,6 +229,8 @@ @stack(1) def func(args, (w_float,)): f = unwrap_float(w_float) + if f < 0.0: + raise PrimitiveFailedError w_res = objtable.wrap_float(math.sqrt(f)) return w_res Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 15:47:05 2007 @@ -284,4 +284,4 @@ def test_primitive_square_root(): assert prim(p.FLOAT_SQUARE_ROOT, [4.0]).value == 2.0 assert round(prim(p.FLOAT_SQUARE_ROOT, [2.0]).value,ROUNDING_DIGITS) == round(1.414213562373095,ROUNDING_DIGITS) - + prim_fails(p.FLOAT_SQUARE_ROOT, [-2.0]) From akuhn at codespeak.net Thu Oct 25 15:51:04 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Thu, 25 Oct 2007 15:51:04 +0200 (CEST) Subject: [pypy-svn] r47928 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071025135104.8DFA480BC@code0.codespeak.net> Author: akuhn Date: Thu Oct 25 15:51:03 2007 New Revision: 47928 Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: (akuhn, tverwaes, arigo) added test to test for class of #doesNotUnderstand: in mini.image (which is, for some strange reason, an Array) Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 15:51:03 2007 @@ -144,7 +144,7 @@ image = get_image() w = image.special(constants.SO_DOES_NOT_UNDERSTAND) assert str(w) == "doesNotUnderstand:" - assert str(w.getclass()) == "Array class" # for some strange reason not a symbol + assert str(w.getclass()) == "Symbol class" # for some strange reason not a symbol """SO_DOES_NOT_UNDERSTAND = 20 From akuhn at codespeak.net Thu Oct 25 15:55:08 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Thu, 25 Oct 2007 15:55:08 +0200 (CEST) Subject: [pypy-svn] r47929 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025135508.A55B88134@code0.codespeak.net> Author: akuhn Date: Thu Oct 25 15:55:05 2007 New Revision: 47929 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: (akuhn, with help by lr) fixed yet another OFF-BY-ONE bug due to the 0-based 1-based mismatch Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 15:55:05 2007 @@ -230,7 +230,8 @@ def init_class(self, chunk): if chunk.iscompact(): - self.g_class = self.owner.compactclasses[chunk.classid].g_object + self.g_class = self.owner.compactclasses[chunk.classid + - 1].g_object # Smalltalk is 1-based indexed else: self.g_class = self.owner.chunks[chunk.classid].g_object From oscar at codespeak.net Thu Oct 25 15:55:25 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Thu, 25 Oct 2007 15:55:25 +0200 (CEST) Subject: [pypy-svn] r47930 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025135525.33FEB813B@code0.codespeak.net> Author: oscar Date: Thu Oct 25 15:55:24 2007 New Revision: 47930 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (cbolz, oscar) float sin primitive Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 15:55:24 2007 @@ -200,6 +200,7 @@ FLOAT_DIVIDE = 50 FLOAT_TRUNCATED = 51 FLOAT_SQUARE_ROOT = 55 +FLOAT_SIN = 56 math_ops = { FLOAT_ADD: operator.add, @@ -234,6 +235,13 @@ w_res = objtable.wrap_float(math.sqrt(f)) return w_res + at primitive(FLOAT_SIN) + at stack(1) +def func(args, (w_float,)): + f = unwrap_float(w_float) + w_res = objtable.wrap_float(math.sin(f)) + return w_res + # ___________________________________________________________________________ # Subscript and Stream Primitives Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 15:55:24 2007 @@ -1,4 +1,5 @@ import py +import math from pypy.lang.smalltalk.primitives import prim_table, PrimitiveFailedError from pypy.lang.smalltalk import model, shadow from pypy.lang.smalltalk import interpreter @@ -281,7 +282,16 @@ return ROUNDING_DIGITS = 8 + +def float_equals(w_f,f): + return round(w_f.value,ROUNDING_DIGITS) == round(f,ROUNDING_DIGITS) + def test_primitive_square_root(): - assert prim(p.FLOAT_SQUARE_ROOT, [4.0]).value == 2.0 - assert round(prim(p.FLOAT_SQUARE_ROOT, [2.0]).value,ROUNDING_DIGITS) == round(1.414213562373095,ROUNDING_DIGITS) - prim_fails(p.FLOAT_SQUARE_ROOT, [-2.0]) + assert prim(p.FLOAT_SQUARE_ROOT, [4.0]).value == 2.0 + assert float_equals(prim(p.FLOAT_SQUARE_ROOT, [2.0]), 1.414213562373095) + prim_fails(p.FLOAT_SQUARE_ROOT, [-2.0]) + +def test_primitive_sin(): + assert prim(p.FLOAT_SIN, [0.0]).value == 0.0 + assert float_equals(prim(p.FLOAT_SIN, [math.pi]), 0.0) + assert float_equals(prim(p.FLOAT_SIN, [math.pi/2]), 1.0) From tverwaes at codespeak.net Thu Oct 25 15:55:52 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 15:55:52 +0200 (CEST) Subject: [pypy-svn] r47931 - in pypy/dist/pypy/lang/smalltalk: . tool Message-ID: <20071025135552.521398134@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 15:55:52 2007 New Revision: 47931 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: Fixed shadows of loaded classes Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 15:55:52 2007 @@ -98,10 +98,12 @@ class W_PointersObject(W_AbstractObjectWithClassReference): """ The normal object """ + + _shadow = None # Default value + def __init__(self, w_class, size): W_AbstractObjectWithClassReference.__init__(self, w_class) self._vars = [None] * size - self._shadow = None def fetch(self, index): return self._vars[index] Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Thu Oct 25 15:55:52 2007 @@ -47,8 +47,13 @@ except sqi.ReturnFromTopLevel, e: return e.object +def testSelector(): + image = create_squeakimage() + w_doesnot = image.special(sqc.SO_DOES_NOT_UNDERSTAND) + print str(w_doesnot.shadow_of_my_class()) + def test_do(): - testCompiledMethods() + testSelector() #printStringsInImage() if __name__ == '__main__': From oscar at codespeak.net Thu Oct 25 15:59:24 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Thu, 25 Oct 2007 15:59:24 +0200 (CEST) Subject: [pypy-svn] r47932 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025135924.542388134@code0.codespeak.net> Author: oscar Date: Thu Oct 25 15:59:23 2007 New Revision: 47932 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (cbolz, oscar) arctan primitive Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 15:59:23 2007 @@ -201,6 +201,7 @@ FLOAT_TRUNCATED = 51 FLOAT_SQUARE_ROOT = 55 FLOAT_SIN = 56 +FLOAT_ARCTAN = 57 math_ops = { FLOAT_ADD: operator.add, @@ -242,6 +243,13 @@ w_res = objtable.wrap_float(math.sin(f)) return w_res + at primitive(FLOAT_ARCTAN) + at stack(1) +def func(args, (w_float,)): + f = unwrap_float(w_float) + w_res = objtable.wrap_float(math.atan(f)) + return w_res + # ___________________________________________________________________________ # Subscript and Stream Primitives Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 15:59:23 2007 @@ -295,3 +295,8 @@ assert prim(p.FLOAT_SIN, [0.0]).value == 0.0 assert float_equals(prim(p.FLOAT_SIN, [math.pi]), 0.0) assert float_equals(prim(p.FLOAT_SIN, [math.pi/2]), 1.0) + +def test_primitive_arctan(): + assert prim(p.FLOAT_ARCTAN, [0.0]).value == 0.0 + assert float_equals(prim(p.FLOAT_ARCTAN, [1]), math.pi/4) + assert float_equals(prim(p.FLOAT_ARCTAN, [1e99]), math.pi/2) From arigo at codespeak.net Thu Oct 25 15:59:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 15:59:44 +0200 (CEST) Subject: [pypy-svn] r47933 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025135944.083608134@code0.codespeak.net> Author: arigo Date: Thu Oct 25 15:59:44 2007 New Revision: 47933 Modified: pypy/dist/pypy/lang/smalltalk/classtable.py pypy/dist/pypy/lang/smalltalk/objtable.py pypy/dist/pypy/lang/smalltalk/shadow.py pypy/dist/pypy/lang/smalltalk/test/test_classtable.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: Put as classtable.w_Xxx globals only the classes that are known to the VM. The other classes should not be used directly, because they will become invalid after an image is loaded. Modified: pypy/dist/pypy/lang/smalltalk/classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/classtable.py Thu Oct 25 15:59:44 2007 @@ -1,4 +1,4 @@ -from pypy.lang.smalltalk import shadow +from pypy.lang.smalltalk import shadow, constants from pypy.lang.smalltalk import constants def bootstrap_class(instsize, w_superclass=None, w_metaclass=None, @@ -31,7 +31,6 @@ w_metaclass=w_metaclass, name=name[2:]) classtable[name] = shadow - globals()[name] = shadow return shadow # Class Name Super class name @@ -43,12 +42,15 @@ ["w_Metaclass", "w_ClassDescription"], ] define_core_cls("w_ProtoObjectClass", None, None) + w_ProtoObjectClass = classtable["w_ProtoObjectClass"] define_core_cls("w_ProtoObject", None, w_ProtoObjectClass) for (cls_nm, super_cls_nm) in cls_nm_tbl: meta_nm = cls_nm + "Class" meta_super_nm = super_cls_nm + "Class" w_metacls = define_core_cls(meta_nm, classtable[meta_super_nm], None) define_core_cls(cls_nm, classtable[super_cls_nm], w_metacls) + w_Class = classtable["w_Class"] + w_Metaclass = classtable["w_Metaclass"] w_ProtoObjectClass.as_class_get_shadow().s_superclass = \ w_Class.as_class_get_shadow() # at this point, all classes that still lack a w_metaclass are themselves @@ -59,6 +61,11 @@ s.s_metaclass = w_Metaclass.as_class_get_shadow() create_classtable() +def copy_in_globals_classes_known_to_the_vm(): + for name in constants.classes_in_special_object_table: + name = 'w_' + name + globals()[name] = classtable[name] + # ___________________________________________________________________________ # Other classes @@ -66,12 +73,13 @@ assert cls_nm.startswith("w_") meta_nm = cls_nm + "Class" meta_super_nm = supercls_nm + "Class" - w_meta_cls = globals()[meta_nm] = classtable[meta_nm] = \ + w_Metaclass = classtable["w_Metaclass"] + w_meta_cls = classtable[meta_nm] = \ bootstrap_class(0, # XXX classtable[meta_super_nm], w_Metaclass, name=meta_nm[2:]) - w_cls = globals()[cls_nm] = classtable[cls_nm] = \ + w_cls = classtable[cls_nm] = \ bootstrap_class(instvarsize, classtable[supercls_nm], w_meta_cls, @@ -87,15 +95,17 @@ define_cls("w_Collection", "w_Object") define_cls("w_SequencableCollection", "w_Collection") define_cls("w_ArrayedCollection", "w_SequencableCollection") -define_cls("w_String", "w_ArrayedCollection") -define_cls("w_ByteString", "w_String", format=shadow.BYTES) +define_cls("w_String", "w_ArrayedCollection", format=shadow.BYTES) define_cls("w_UndefinedObject", "w_Object") define_cls("w_Boolean", "w_Object") define_cls("w_True", "w_Boolean") define_cls("w_False", "w_Boolean") define_cls("w_ByteArray", "w_ArrayedCollection", format=shadow.BYTES) define_cls("w_CompiledMethod", "w_ByteArray", format=shadow.COMPILED_METHOD) +define_cls("w_MethodContext", "w_Object") define_cls("w_ContextPart", "w_Object") define_cls("w_MethodContext", "w_ContextPart") define_cls("w_BlockContext", "w_ContextPart", instvarsize=constants.BLKCTX_TEMP_FRAME_START) + +copy_in_globals_classes_known_to_the_vm() Modified: pypy/dist/pypy/lang/smalltalk/objtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/objtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/objtable.py Thu Oct 25 15:59:44 2007 @@ -14,7 +14,7 @@ return model.W_Float(i) def wrap_string(str): - w_inst = ct.w_ByteString.as_class_get_shadow().new(len(str)) + w_inst = ct.w_String.as_class_get_shadow().new(len(str)) for i in range(len(str)): w_inst.setbyte(i, ord(str[i])) return w_inst @@ -47,9 +47,9 @@ CharacterTable = [bld_char(i) for i in range(256)] wrap_char_table() -w_true = ct.w_True.as_class_get_shadow().new() -w_false = ct.w_False.as_class_get_shadow().new() -w_nil = ct.w_UndefinedObject.as_class_get_shadow().new() +w_true = ct.classtable['w_True'].as_class_get_shadow().new() +w_false = ct.classtable['w_False'].as_class_get_shadow().new() +w_nil = ct.classtable['w_UndefinedObject'].as_class_get_shadow().new() w_mone = wrap_int(-1) w_zero = wrap_int(0) w_one = wrap_int(1) Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Thu Oct 25 15:59:44 2007 @@ -130,11 +130,6 @@ " Number of named instance variables for each instance of this class " return self.instance_size - def ismetaclass(self): - "Heuristic to detect if this is the shadow of a metaclass." - from pypy.lang.smalltalk import classtable - return self.s_metaclass.w_self is classtable.w_Metaclass - def inherits_from(self, s_superclass): classshadow = self while classshadow is not None: Modified: pypy/dist/pypy/lang/smalltalk/test/test_classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_classtable.py Thu Oct 25 15:59:44 2007 @@ -1,20 +1,26 @@ from pypy.lang.smalltalk import classtable +def ismetaclass(shadow): + # Heuristic to detect if this is the shadow of a metaclass. Don't use + # apart from in this test file, because classtable['w_Metaclass'] is + # bogus after loading an image. + return shadow.s_metaclass.w_self is classtable.classtable['w_Metaclass'] + def test_every_class_is_an_instance_of_a_metaclass(): for (nm, w_cls) in classtable.classtable.items(): shadow = w_cls.as_class_get_shadow() - assert shadow.ismetaclass() or shadow.s_metaclass.ismetaclass() + assert ismetaclass(shadow) or ismetaclass(shadow.s_metaclass) def test_every_metaclass_inherits_from_class_and_behavior(): - s_Class = classtable.w_Class.as_class_get_shadow() - s_Behavior = classtable.w_Behavior.as_class_get_shadow() + s_Class = classtable.classtable['w_Class'].as_class_get_shadow() + s_Behavior = classtable.classtable['w_Behavior'].as_class_get_shadow() for (nm, w_cls) in classtable.classtable.items(): shadow = w_cls.as_class_get_shadow() - if shadow.ismetaclass(): + if ismetaclass(shadow): assert shadow.inherits_from(s_Class) assert s_Class.inherits_from(s_Behavior) def test_metaclass_of_metaclass_is_an_instance_of_metaclass(): - s_Metaclass = classtable.w_Metaclass.as_class_get_shadow() + s_Metaclass = classtable.classtable['w_Metaclass'].as_class_get_shadow() assert s_Metaclass.s_metaclass.s_metaclass is s_Metaclass Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 15:59:44 2007 @@ -217,19 +217,21 @@ assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i]) def test_new(): - w_res = prim(p.NEW, [classtable.w_Object]) - assert w_res.getclass() == classtable.w_Object + w_Object = classtable.classtable['w_Object'] + w_res = prim(p.NEW, [w_Object]) + assert w_res.getclass() is w_Object def test_invalid_new(): - prim_fails(p.NEW, [classtable.w_ByteString]) + prim_fails(p.NEW, [classtable.w_String]) def test_new_with_arg(): - w_res = prim(p.NEW_WITH_ARG, [classtable.w_ByteString, 20]) - assert w_res.getclass() == classtable.w_ByteString + w_res = prim(p.NEW_WITH_ARG, [classtable.w_String, 20]) + assert w_res.getclass() == classtable.w_String assert w_res.size() == 20 def test_invalid_new_with_arg(): - prim_fails(p.NEW_WITH_ARG, [classtable.w_Object, 20]) + w_Object = classtable.classtable['w_Object'] + prim_fails(p.NEW_WITH_ARG, [w_Object, 20]) def test_inst_var_at(): # I am not entirely sure if this is what this primitive is From oscar at codespeak.net Thu Oct 25 16:18:36 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Thu, 25 Oct 2007 16:18:36 +0200 (CEST) Subject: [pypy-svn] r47934 - in pypy/dist/pypy: rlib rlib/test translator/c Message-ID: <20071025141836.DB8558120@code0.codespeak.net> Author: oscar Date: Thu Oct 25 16:18:35 2007 New Revision: 47934 Modified: pypy/dist/pypy/rlib/rarithmetic.py pypy/dist/pypy/rlib/test/test_rarithmetic.py pypy/dist/pypy/translator/c/primitive.py Log: (cbolz, oscar) added INFINITY and NAN; added tests; moved isinf and isnan from c backend to rarithmetic Modified: pypy/dist/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/dist/pypy/rlib/rarithmetic.py (original) +++ pypy/dist/pypy/rlib/rarithmetic.py Thu Oct 25 16:18:35 2007 @@ -45,9 +45,18 @@ LONG_MASK = _Ltest*2-1 LONG_TEST = _Ltest +INFINITY = 1e200 * 1e200 +NAN = INFINITY / INFINITY + def isinf(x): return x != 0.0 and x / 2 == x +# To get isnan, working x-platform and both on 2.3 and 2.4, is a +# horror. I think this works (for reasons I don't really want to talk +# about), and probably when implemented on top of pypy, too. +def isnan(v): + return v != v*1.0 or (v == 1.0 and v == 2.0) + def intmask(n): if isinstance(n, int): return int(n) # possibly bool->int Modified: pypy/dist/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/dist/pypy/rlib/test/test_rarithmetic.py Thu Oct 25 16:18:35 2007 @@ -331,3 +331,8 @@ class TestOOtype(BaseTestRarithmetic, OORtypeMixin): pass +def test_isinf(): + assert isinf(INFINITY) + +def test_isnan(): + assert isnan(NAN) Modified: pypy/dist/pypy/translator/c/primitive.py ============================================================================== --- pypy/dist/pypy/translator/c/primitive.py (original) +++ pypy/dist/pypy/translator/c/primitive.py Thu Oct 25 16:18:35 2007 @@ -1,7 +1,7 @@ import sys from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic from pypy.rlib.objectmodel import CDefinedIntSymbolic -from pypy.rlib.rarithmetic import r_longlong +from pypy.rlib.rarithmetic import r_longlong, isinf, isnan from pypy.rpython.lltypesystem.lltype import * from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem.llmemory import Address, \ @@ -81,15 +81,6 @@ else: return '%dLL' % value -def isinf(x): - return x != 0.0 and x / 2 == x - -# To get isnan, working x-platform and both on 2.3 and 2.4, is a -# horror. I think this works (for reasons I don't really want to talk -# about), and probably when implemented on top of pypy, too. -def isnan(v): - return v != v*1.0 or (v == 1.0 and v == 2.0) - def name_float(value, db): if isinf(value): if value > 0: From akuhn at codespeak.net Thu Oct 25 16:22:11 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Thu, 25 Oct 2007 16:22:11 +0200 (CEST) Subject: [pypy-svn] r47935 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025142211.487C58120@code0.codespeak.net> Author: akuhn Date: Thu Oct 25 16:22:10 2007 New Revision: 47935 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py Log: (akuhn, tverwaes, arigo) replaced use of struct with RPython code Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 16:22:10 2007 @@ -9,6 +9,21 @@ chr((integer >> 8) & 0xff) + chr((integer >> 0) & 0xff)) +def chrs2int(b): + assert len(b) == 4 + first = ord(b[0]) # big endian + if first & 0x80 != 0: + first = first - 0x100 + return first << 24 | ord(b[1]) << 16 | ord(b[2]) << 8 | ord(b[3]) + +def swapped_chrs2int(b): + assert len(b) == 4 + first = ord(b[3]) # little endian + if first & 0x80 != 0: + first = first - 0x100 + return first << 24 | ord(b[2]) << 16 | ord(b[1]) << 8 | ord(b[0]) + + def splitbits(integer, lengths): assert sum(lengths) <= 32 result = [] @@ -35,15 +50,12 @@ self.count = 0 def peek(self): - import struct if self.pos >= len(self.data): raise IndexError if self.swap: - format = " Author: akuhn Date: Thu Oct 25 16:24:43 2007 New Revision: 47936 Removed: pypy/dist/pypy/lang/smalltalk/specialobjects.txt Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: removed legacy text file, please refer to constants.py for constants Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 16:24:43 2007 @@ -3,12 +3,6 @@ from pypy.lang.smalltalk import objtable from pypy.rlib import objectmodel -def int2str(integer): - return (chr((integer >> 24) & 0xff) + - chr((integer >> 16) & 0xff) + - chr((integer >> 8) & 0xff) + - chr((integer >> 0) & 0xff)) - def chrs2int(b): assert len(b) == 4 first = ord(b[0]) # big endian @@ -21,22 +15,21 @@ first = ord(b[3]) # little endian if first & 0x80 != 0: first = first - 0x100 - return first << 24 | ord(b[2]) << 16 | ord(b[1]) << 8 | ord(b[0]) - + return first << 24 | ord(b[2]) << 16 | ord(b[1]) << 8 | ord(b[0]) def splitbits(integer, lengths): + #XXX we can later let the tool chain mask and unroll this assert sum(lengths) <= 32 result = [] for length in lengths: result.append(integer & (2**length - 1)) integer = integer >> length - #XXX we can later mask and unroll this return result # ____________________________________________________________ # -# Reads an image file and created all model objects +# Reads an image file and creates all model objects class Stream(object): """ Simple input stream """ From arigo at codespeak.net Thu Oct 25 16:25:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 16:25:52 +0200 (CEST) Subject: [pypy-svn] r47937 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025142552.C23F780E0@code0.codespeak.net> Author: arigo Date: Thu Oct 25 16:25:52 2007 New Revision: 47937 Modified: pypy/dist/pypy/lang/smalltalk/shadow.py Log: Read s_superclass and s_metaclass. That's all untested code - naughty us. Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Thu Oct 25 16:25:52 2007 @@ -47,6 +47,8 @@ def update_shadow(self): "Update the ClassShadow with data from the w_self class." + from pypy.lang.smalltalk import objtable + w_self = self.w_self # read and painfully decode the format classformat = unwrap_int(w_self.fetch(constants.CLASS_FORMAT_INDEX)) @@ -82,14 +84,21 @@ self.instance_kind = COMPILED_METHOD else: raise ClassShadowError("unknown format %d" % (format,)) - # XXX read s_superclass - # XXX read s_metaclass # XXX read the methoddict + # ... + + # for the rest, we need to reset invalid to False already so + # that cycles in the superclass and/or metaclass chains don't + # cause infinite recursion self.invalid = False - if self.s_superclass is not None: - self.s_superclass.check_for_updates() - if self.s_metaclass is not None: - self.s_metaclass.check_for_updates() + # read s_superclass + w_superclass = w_self.fetch(constants.CLASS_SUPERCLASS_INDEX) + if w_superclass is objtable.w_nil: + self.s_superclass = None + else: + self.s_superclass = w_superclass.as_class_get_shadow() + # read s_metaclass + self.s_metaclass = w_self.shadow_of_my_class() def new(self, extrasize=0): w_cls = self.w_self From oscar at codespeak.net Thu Oct 25 16:29:04 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Thu, 25 Oct 2007 16:29:04 +0200 (CEST) Subject: [pypy-svn] r47938 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025142904.A927E80E0@code0.codespeak.net> Author: oscar Date: Thu Oct 25 16:29:04 2007 New Revision: 47938 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (cfbolz, oscar) added log_n and tests Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 16:29:04 2007 @@ -202,6 +202,7 @@ FLOAT_SQUARE_ROOT = 55 FLOAT_SIN = 56 FLOAT_ARCTAN = 57 +FLOAT_LOG_N = 58 math_ops = { FLOAT_ADD: operator.add, @@ -250,6 +251,18 @@ w_res = objtable.wrap_float(math.atan(f)) return w_res + at primitive(FLOAT_LOG_N) + at stack(1) +def func(args, (w_float,)): + f = unwrap_float(w_float) + if f == 0: + res = -rarithmetic.INFINITY + elif f < 0: + res = rarithmetic.NAN + else: + res = math.log(f) + return objtable.wrap_float(res) + # ___________________________________________________________________________ # Subscript and Stream Primitives Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 16:29:04 2007 @@ -5,6 +5,7 @@ from pypy.lang.smalltalk import interpreter from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import objtable +from pypy.rlib.rarithmetic import INFINITY, NAN, isinf, isnan # Violates the guideline, but we use it A LOT to reference the primitive codes: import pypy.lang.smalltalk.primitives as p @@ -302,3 +303,10 @@ assert prim(p.FLOAT_ARCTAN, [0.0]).value == 0.0 assert float_equals(prim(p.FLOAT_ARCTAN, [1]), math.pi/4) assert float_equals(prim(p.FLOAT_ARCTAN, [1e99]), math.pi/2) + +def test_primitive_log_n(): + assert prim(p.FLOAT_LOG_N, [1.0]).value == 0.0 + assert prim(p.FLOAT_LOG_N, [math.e]).value == 1.0 + assert float_equals(prim(p.FLOAT_LOG_N, [10.0]), 2.302585092994046) + assert isinf(prim(p.FLOAT_LOG_N, [0.0]).value) # works also for negative infinity + assert isnan(prim(p.FLOAT_LOG_N, [-1.0]).value) From oscar at codespeak.net Thu Oct 25 16:34:02 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Thu, 25 Oct 2007 16:34:02 +0200 (CEST) Subject: [pypy-svn] r47939 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025143402.8AAE680E0@code0.codespeak.net> Author: oscar Date: Thu Oct 25 16:34:02 2007 New Revision: 47939 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (cfbolz, oscar) added exp and tests Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 16:34:02 2007 @@ -203,6 +203,7 @@ FLOAT_SIN = 56 FLOAT_ARCTAN = 57 FLOAT_LOG_N = 58 +FLOAT_EXP = 59 math_ops = { FLOAT_ADD: operator.add, @@ -263,6 +264,14 @@ res = math.log(f) return objtable.wrap_float(res) + at primitive(FLOAT_EXP) + at stack(1) +def func(args, (w_float,)): + f = unwrap_float(w_float) + w_res = objtable.wrap_float(math.exp(f)) + return w_res + + # ___________________________________________________________________________ # Subscript and Stream Primitives Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 16:34:02 2007 @@ -310,3 +310,9 @@ assert float_equals(prim(p.FLOAT_LOG_N, [10.0]), 2.302585092994046) assert isinf(prim(p.FLOAT_LOG_N, [0.0]).value) # works also for negative infinity assert isnan(prim(p.FLOAT_LOG_N, [-1.0]).value) + +def test_primitive_exp(): + assert float_equals(prim(p.FLOAT_EXP, [-1.0]), 1/math.e) + assert prim(p.FLOAT_EXP, [0]).value == 1 + assert float_equals(prim(p.FLOAT_EXP, [1]), math.e) + assert float_equals(prim(p.FLOAT_EXP, [math.log(10)]), 10) From oscar at codespeak.net Thu Oct 25 16:35:26 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Thu, 25 Oct 2007 16:35:26 +0200 (CEST) Subject: [pypy-svn] r47940 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071025143526.4639B80E0@code0.codespeak.net> Author: oscar Date: Thu Oct 25 16:35:26 2007 New Revision: 47940 Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (cfbolz, oscar) changed some magic numbers in tests to expressions Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 16:35:26 2007 @@ -291,7 +291,7 @@ def test_primitive_square_root(): assert prim(p.FLOAT_SQUARE_ROOT, [4.0]).value == 2.0 - assert float_equals(prim(p.FLOAT_SQUARE_ROOT, [2.0]), 1.414213562373095) + assert float_equals(prim(p.FLOAT_SQUARE_ROOT, [2.0]), math.sqrt(2)) prim_fails(p.FLOAT_SQUARE_ROOT, [-2.0]) def test_primitive_sin(): @@ -307,7 +307,7 @@ def test_primitive_log_n(): assert prim(p.FLOAT_LOG_N, [1.0]).value == 0.0 assert prim(p.FLOAT_LOG_N, [math.e]).value == 1.0 - assert float_equals(prim(p.FLOAT_LOG_N, [10.0]), 2.302585092994046) + assert float_equals(prim(p.FLOAT_LOG_N, [10.0]), math.log(10)) assert isinf(prim(p.FLOAT_LOG_N, [0.0]).value) # works also for negative infinity assert isnan(prim(p.FLOAT_LOG_N, [-1.0]).value) From oscar at codespeak.net Thu Oct 25 16:49:11 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Thu, 25 Oct 2007 16:49:11 +0200 (CEST) Subject: [pypy-svn] r47942 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025144911.864158125@code0.codespeak.net> Author: oscar Date: Thu Oct 25 16:49:10 2007 New Revision: 47942 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (cfbolz, oscar) times_two_power Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 16:49:10 2007 @@ -199,6 +199,7 @@ FLOAT_MULTIPLY = 49 FLOAT_DIVIDE = 50 FLOAT_TRUNCATED = 51 +FLOAT_TIMES_TWO_POWER = 54 FLOAT_SQUARE_ROOT = 55 FLOAT_SIN = 56 FLOAT_ARCTAN = 57 @@ -229,6 +230,14 @@ w_res = objtable.wrap_int(int(f)) return w_res + at primitive(FLOAT_TIMES_TWO_POWER) + at stack(2) +def func(args, (w_rcvr,w_arg,)): + rcvr = unwrap_float(w_rcvr) + arg = unwrap_int(w_arg) + w_res = objtable.wrap_float(math.ldexp(rcvr,arg)) + return w_res + @primitive(FLOAT_SQUARE_ROOT) @stack(1) def func(args, (w_float,)): Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 16:49:10 2007 @@ -316,3 +316,15 @@ assert prim(p.FLOAT_EXP, [0]).value == 1 assert float_equals(prim(p.FLOAT_EXP, [1]), math.e) assert float_equals(prim(p.FLOAT_EXP, [math.log(10)]), 10) + +def equals_ttp(rcvr,arg,res): + return float_equals(prim(p.FLOAT_TIMES_TWO_POWER, [rcvr,arg]), res) + +def test_times_two_power(): + assert equals_ttp(1,1,2) + assert equals_ttp(1.5,1,3) + assert equals_ttp(2,4,32) + assert equals_ttp(0,2,0) + assert equals_ttp(-1,2,-4) + assert equals_ttp(1.5,0,1.5) + assert equals_ttp(1.5,-1,0.75) From xoraxax at codespeak.net Thu Oct 25 16:53:58 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 25 Oct 2007 16:53:58 +0200 (CEST) Subject: [pypy-svn] r47943 - in pypy/dist/pypy: objspace/std translator/cli translator/jvm Message-ID: <20071025145358.14F5F8125@code0.codespeak.net> Author: xoraxax Date: Thu Oct 25 16:53:58 2007 New Revision: 47943 Modified: pypy/dist/pypy/objspace/std/formatting.py pypy/dist/pypy/translator/cli/ilgenerator.py pypy/dist/pypy/translator/jvm/generator.py Log: Removed three other isnan or isinf occurences in the code. Modified: pypy/dist/pypy/objspace/std/formatting.py ============================================================================== --- pypy/dist/pypy/objspace/std/formatting.py (original) +++ pypy/dist/pypy/objspace/std/formatting.py Thu Oct 25 16:53:58 2007 @@ -2,7 +2,7 @@ String formatting routines. """ from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.rarithmetic import ovfcheck, formatd_overflow +from pypy.rlib.rarithmetic import ovfcheck, formatd_overflow, isnan, isinf from pypy.interpreter.error import OperationError from pypy.tool.sourcetools import func_with_new_name @@ -489,12 +489,3 @@ hex_num_helper = format_num_helper_generator('%x', '0123456789abcdef') -# isinf isn't too hard... -def isinf(v): - return v != 0 and v*2.0 == v - -# To get isnan, working x-platform and both on 2.3 and 2.4, is a -# horror. I think this works (for reasons I don't really want to talk -# about), and probably when implemented on top of pypy, too. -def isnan(v): - return v != v*1.0 or (v == 1.0 and v == 2.0) Modified: pypy/dist/pypy/translator/cli/ilgenerator.py ============================================================================== --- pypy/dist/pypy/translator/cli/ilgenerator.py (original) +++ pypy/dist/pypy/translator/cli/ilgenerator.py Thu Oct 25 16:53:58 2007 @@ -1,17 +1,13 @@ from pypy.rpython.lltypesystem.lltype import Signed, Unsigned, Void, Bool, Float from pypy.rpython.lltypesystem.lltype import SignedLongLong, UnsignedLongLong from pypy.rlib.objectmodel import CDefinedIntSymbolic +from pypy.rlib.rarithmetic import isnan, isinf from pypy.rpython.ootypesystem import ootype from pypy.translator.oosupport.metavm import Generator from pypy.translator.oosupport.constant import push_constant from pypy.objspace.flow import model as flowmodel from pypy.translator.cli.support import string_literal -def isnan(v): - return v != v*1.0 or (v == 1.0 and v == 2.0) - -def isinf(v): - return v!=0 and (v == v*2) class CodeGenerator(object): def __init__(self, out, indentstep = 4, startblock = '{', endblock = '}'): Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Thu Oct 25 16:53:58 2007 @@ -4,6 +4,7 @@ from pypy.translator.oosupport.function import render_sub_op from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import CDefinedIntSymbolic +from pypy.rlib.rarithmetic import isnan, isinf from pypy.translator.oosupport.constant import push_constant import pypy.translator.jvm.typesystem as jvmtype from pypy.translator.jvm.typesystem import \ @@ -14,14 +15,6 @@ jObjectArray, jPyPyInterlink, jPyPyCustomDict, jPyPyEquals, \ jPyPyHashCode, jMap, jPyPyWeakRef, jSystem, jll_os -# ___________________________________________________________________________ -# Miscellaneous helper functions - -def _isnan(v): - return v != v*1.0 or (v == 1.0 and v == 2.0) - -def _isinf(v): - return v!=0 and (v == v*2) # ___________________________________________________________________________ # JVM Opcodes: @@ -1109,9 +1102,9 @@ self.emit(LDC2, value) def _push_double_constant(self, value): - if _isnan(value): + if isnan(value): DOUBLENAN.load(self) - elif _isinf(value): + elif isinf(value): if value > 0: DOUBLEPOSINF.load(self) else: DOUBLENEGINF.load(self) elif value == 0.0: From oscar at codespeak.net Thu Oct 25 17:01:31 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Thu, 25 Oct 2007 17:01:31 +0200 (CEST) Subject: [pypy-svn] r47944 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025150131.C35768131@code0.codespeak.net> Author: oscar Date: Thu Oct 25 17:01:31 2007 New Revision: 47944 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (cfbolz, oscar) FullGC Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 17:01:31 2007 @@ -196,9 +196,11 @@ _FLOAT_OFFSET = 40 FLOAT_ADD = 41 FLOAT_SUBTRACT = 42 +# NB: 43 ... 48 are implemented above FLOAT_MULTIPLY = 49 FLOAT_DIVIDE = 50 FLOAT_TRUNCATED = 51 +# OPTIONAL: 52, 53 FLOAT_TIMES_TWO_POWER = 54 FLOAT_SQUARE_ROOT = 55 FLOAT_SIN = 56 @@ -506,6 +508,16 @@ return # ___________________________________________________________________________ +# Squeak Miscellaneous Primitives (128-149) +FULL_GC = 130 + + at primitive(FULL_GC) + at stack(1) # Squeak pops the arg and ignores it ... go figure +def func(args, (w_arg,)): + from pypy.rlib import rgc + rgc.collect() + +# ___________________________________________________________________________ # Boolean Primitives LESSTHAN = 3 Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 17:01:31 2007 @@ -328,3 +328,8 @@ assert equals_ttp(-1,2,-4) assert equals_ttp(1.5,0,1.5) assert equals_ttp(1.5,-1,0.75) + +def test_full_gc(): + # Should not fail :-) + prim(p.FULL_GC, [42]) # Dummy arg + From oscar at codespeak.net Thu Oct 25 17:05:40 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Thu, 25 Oct 2007 17:05:40 +0200 (CEST) Subject: [pypy-svn] r47945 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025150540.0586F812D@code0.codespeak.net> Author: oscar Date: Thu Oct 25 17:05:40 2007 New Revision: 47945 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (cfbolz, oscar) incremental gc (faked) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 17:05:40 2007 @@ -510,12 +510,18 @@ # ___________________________________________________________________________ # Squeak Miscellaneous Primitives (128-149) FULL_GC = 130 +INC_GC = 131 +def fake_bytes_left(): + return 2**20 # XXX we don't know how to do this :-( + + at primitive(INC_GC) # XXX the same for now @primitive(FULL_GC) @stack(1) # Squeak pops the arg and ignores it ... go figure def func(args, (w_arg,)): from pypy.rlib import rgc rgc.collect() + return fake_bytes_left() # ___________________________________________________________________________ # Boolean Primitives Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 17:05:40 2007 @@ -329,6 +329,10 @@ assert equals_ttp(1.5,0,1.5) assert equals_ttp(1.5,-1,0.75) +def test_inc_gc(): + # Should not fail :-) + prim(p.INC_GC, [42]) # Dummy arg + def test_full_gc(): # Should not fail :-) prim(p.FULL_GC, [42]) # Dummy arg From fijal at codespeak.net Thu Oct 25 17:14:56 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 25 Oct 2007 17:14:56 +0200 (CEST) Subject: [pypy-svn] r47946 - pypy/dist/pypy/translator/backendopt Message-ID: <20071025151456.1C79E812B@code0.codespeak.net> Author: fijal Date: Thu Oct 25 17:14:55 2007 New Revision: 47946 Modified: pypy/dist/pypy/translator/backendopt/inline.py Log: Add a possibility to iterate over all callsites Modified: pypy/dist/pypy/translator/backendopt/inline.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/inline.py (original) +++ pypy/dist/pypy/translator/backendopt/inline.py Thu Oct 25 17:14:55 2007 @@ -77,7 +77,8 @@ graph = getattr(funcobj, 'graph', None) # accept a function or a graph as 'inline_func' - if (graph is calling_what or + if (calling_what is None or + graph is calling_what or getattr(funcobj, '_callable', None) is calling_what): yield graph, block, i From arigo at codespeak.net Thu Oct 25 17:16:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 17:16:59 +0200 (CEST) Subject: [pypy-svn] r47947 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025151659.B57F88122@code0.codespeak.net> Author: arigo Date: Thu Oct 25 17:16:59 2007 New Revision: 47947 Added: pypy/dist/pypy/lang/smalltalk/test/test_shadow.py (contents, props changed) Modified: pypy/dist/pypy/lang/smalltalk/classtable.py pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/shadow.py pypy/dist/pypy/lang/smalltalk/test/test_classtable.py Log: (toon, arigo, akuhn around) * removed s_metaclass from the ClassShadow, where it is not really useful * our first test for update_shadow() Modified: pypy/dist/pypy/lang/smalltalk/classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/classtable.py Thu Oct 25 17:16:59 2007 @@ -4,13 +4,12 @@ def bootstrap_class(instsize, w_superclass=None, w_metaclass=None, name='?', format=shadow.POINTERS, varsized=False): from pypy.lang.smalltalk import model - w_class = model.W_PointersObject(None, 0) # a dummy placeholder for testing + w_class = model.W_PointersObject(w_metaclass, 0) + # a dummy placeholder for testing s = shadow.ClassShadow(w_class) s.methoddict = {} if w_superclass is not None: s.s_superclass = w_superclass.as_class_get_shadow() - if w_metaclass is not None: - s.s_metaclass = w_metaclass.as_class_get_shadow() s.name = name s.instance_size = instsize s.instance_kind = format @@ -53,12 +52,11 @@ w_Metaclass = classtable["w_Metaclass"] w_ProtoObjectClass.as_class_get_shadow().s_superclass = \ w_Class.as_class_get_shadow() - # at this point, all classes that still lack a w_metaclass are themselves + # at this point, all classes that still lack a w_class are themselves # metaclasses for nm, w_cls_obj in classtable.items(): - s = w_cls_obj.as_class_get_shadow() - if s.s_metaclass is None: - s.s_metaclass = w_Metaclass.as_class_get_shadow() + if w_cls_obj.w_class is None: + w_cls_obj.w_class = w_Metaclass create_classtable() def copy_in_globals_classes_known_to_the_vm(): Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Thu Oct 25 17:16:59 2007 @@ -12,7 +12,7 @@ CLASS_SUPERCLASS_INDEX = 0 CLASS_METHODDICT_INDEX = 1 CLASS_FORMAT_INDEX = 2 -CLASS_NAME_INDEX = 6 +CLASS_NAME_INDEX = 6 # in the mini.image, at least METHODDICT_VALUES_INDEX = 1 METHODDICT_NAMES_INDEX = 2 Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 17:16:59 2007 @@ -174,11 +174,13 @@ return len(self.bytes) def __str__(self): - return "".join(self.bytes) + return self.as_string() def __repr__(self): - return "" % ("".join(self.bytes),) + return "" % (self.as_string(),) + def as_string(self): + return "".join(self.bytes) def invariant(self): if not W_AbstractObjectWithClassReference.invariant(self): Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Thu Oct 25 17:16:59 2007 @@ -37,8 +37,7 @@ def invalidate(self): self.methoddict = {} self.s_superclass = None # the ClassShadow of the super class - self.s_metaclass = None # the ClassShadow of the meta class - self.name = '?' # take care when initing this, metaclasses do not have a name! + self.name = None self.invalid = True def check_for_updates(self): @@ -62,7 +61,7 @@ # compute the instance size (really the size, not the number of bytes) instsize_lo = (classformat >> 1) & 0x3F instsize_hi = (classformat >> (9 + 1)) & 0xC0 - self.instance_size = instsize_lo | instsize_hi + self.instance_size = (instsize_lo | instsize_hi) - 1 # subtract hdr # decode the instSpec format = (classformat >> 7) & 15 self.instance_varsized = format >= 2 @@ -84,6 +83,11 @@ self.instance_kind = COMPILED_METHOD else: raise ClassShadowError("unknown format %d" % (format,)) + # read the name + if w_self.size() > constants.CLASS_NAME_INDEX: + w_name = w_self.fetch(constants.CLASS_NAME_INDEX) + if isinstance(w_name, model.W_BytesObject): + self.name = w_name.as_string() # XXX read the methoddict # ... @@ -97,8 +101,6 @@ self.s_superclass = None else: self.s_superclass = w_superclass.as_class_get_shadow() - # read s_metaclass - self.s_metaclass = w_self.shadow_of_my_class() def new(self, extrasize=0): w_cls = self.w_self Modified: pypy/dist/pypy/lang/smalltalk/test/test_classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_classtable.py Thu Oct 25 17:16:59 2007 @@ -1,26 +1,24 @@ from pypy.lang.smalltalk import classtable -def ismetaclass(shadow): - # Heuristic to detect if this is the shadow of a metaclass. Don't use - # apart from in this test file, because classtable['w_Metaclass'] is +def ismetaclass(w_cls): + # Heuristic to detect if this is a metaclass. Don't use apart + # from in this test file, because classtable['w_Metaclass'] is # bogus after loading an image. - return shadow.s_metaclass.w_self is classtable.classtable['w_Metaclass'] + return w_cls.w_class is classtable.classtable['w_Metaclass'] def test_every_class_is_an_instance_of_a_metaclass(): for (nm, w_cls) in classtable.classtable.items(): - shadow = w_cls.as_class_get_shadow() - assert ismetaclass(shadow) or ismetaclass(shadow.s_metaclass) + assert ismetaclass(w_cls) or ismetaclass(w_cls.w_class) def test_every_metaclass_inherits_from_class_and_behavior(): s_Class = classtable.classtable['w_Class'].as_class_get_shadow() s_Behavior = classtable.classtable['w_Behavior'].as_class_get_shadow() for (nm, w_cls) in classtable.classtable.items(): - shadow = w_cls.as_class_get_shadow() - if ismetaclass(shadow): + if ismetaclass(w_cls): + shadow = w_cls.as_class_get_shadow() assert shadow.inherits_from(s_Class) assert s_Class.inherits_from(s_Behavior) def test_metaclass_of_metaclass_is_an_instance_of_metaclass(): - s_Metaclass = classtable.classtable['w_Metaclass'].as_class_get_shadow() - assert s_Metaclass.s_metaclass.s_metaclass is s_Metaclass - + w_Metaclass = classtable.classtable['w_Metaclass'] + assert w_Metaclass.w_class.w_class is w_Metaclass Added: pypy/dist/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Thu Oct 25 17:16:59 2007 @@ -0,0 +1,28 @@ +from pypy.lang.smalltalk import model, shadow, classtable, constants, objtable + +w_Object = classtable.classtable['w_Object'] +w_Metaclass = classtable.classtable['w_Metaclass'] + +def build_smalltalk_class(name, format, w_superclass=w_Object, + w_classofclass=None): + if w_classofclass is None: + w_classofclass = build_smalltalk_class(None, 0x94, + w_superclass.w_class, + w_Metaclass) + size = constants.CLASS_NAME_INDEX + 1 + w_class = model.W_PointersObject(w_classofclass, size) + w_class.store(constants.CLASS_SUPERCLASS_INDEX, w_superclass) + #w_class.store(constants.CLASS_METHODDICT_INDEX, ...) + w_class.store(constants.CLASS_FORMAT_INDEX, objtable.wrap_int(format)) + if name is not None: + w_class.store(constants.CLASS_NAME_INDEX, objtable.wrap_string(name)) + return w_class + +def test_empty_class(): + w_class = build_smalltalk_class("Empty", 0x2) + classshadow = w_class.as_class_get_shadow() + assert classshadow.instance_kind == shadow.POINTERS + assert not classshadow.isvariable() + assert classshadow.instsize() == 0 + assert classshadow.name == "Empty" + assert classshadow.s_superclass is w_Object.as_class_get_shadow() From akuhn at codespeak.net Thu Oct 25 17:24:42 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Thu, 25 Oct 2007 17:24:42 +0200 (CEST) Subject: [pypy-svn] r47948 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025152442.E10BA8122@code0.codespeak.net> Author: akuhn Date: Thu Oct 25 17:24:42 2007 New Revision: 47948 Modified: pypy/dist/pypy/lang/smalltalk/classtable.py Log: documented would be correct minimal setup for classtable Modified: pypy/dist/pypy/lang/smalltalk/classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/classtable.py Thu Oct 25 17:24:42 2007 @@ -32,9 +32,24 @@ classtable[name] = shadow return shadow + # A complete minimal setup (including Behavior) would look like this + # + # class: superclass: metaclass: + # ------------------- ------------------- ------------------- + # Object *nil Object class + # Behavior Object Behavior class + # ClassDescription Behavior ClassDescription class + # Class ClassDescription Class class + # Metaclass ClassDescription Metaclass class + # Object class *Class *Metaclass + # Behavior class Object class *Metaclass + # ClassDescription cl Behavior class *Metaclass + # Class class ClassDescription cl *Metaclass + # Metaclass class ClassDescription cl *Metaclass + # Class Name Super class name cls_nm_tbl = [ - ["w_Object", "w_ProtoObject"], + ["w_Object", "w_ProtoObject"], # there is not ProtoObject in mini.image ["w_Behavior", "w_Object"], ["w_ClassDescription", "w_Behavior"], ["w_Class", "w_ClassDescription"], From arigo at codespeak.net Thu Oct 25 17:29:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 17:29:51 +0200 (CEST) Subject: [pypy-svn] r47949 - pypy/dist/pypy/objspace/flow/test Message-ID: <20071025152951.655F68122@code0.codespeak.net> Author: arigo Date: Thu Oct 25 17:29:51 2007 New Revision: 47949 Modified: pypy/dist/pypy/objspace/flow/test/test_objspace.py Log: An extra (passing) test for operator.xor. Modified: pypy/dist/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/dist/pypy/objspace/flow/test/test_objspace.py Thu Oct 25 17:29:51 2007 @@ -489,6 +489,7 @@ operator.__ne__(x,3) operator.__gt__(x,3) operator.__ge__(x,3) + operator.xor(x,3) # the following ones are constant-folded operator.eq(2,3) operator.__gt__(2,3) @@ -497,10 +498,10 @@ x = self.codetest(self.specialcases) from pypy.translator.simplify import join_blocks join_blocks(x) - assert len(x.startblock.operations) == 13 + assert len(x.startblock.operations) == 14 for op in x.startblock.operations: assert op.opname in ['lt', 'le', 'eq', 'ne', - 'gt', 'ge', 'is_'] + 'gt', 'ge', 'is_', 'xor'] assert len(op.args) == 2 assert op.args[1].value == 3 From tverwaes at codespeak.net Thu Oct 25 17:38:48 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 17:38:48 +0200 (CEST) Subject: [pypy-svn] r47951 - in pypy/dist/pypy/lang/smalltalk: . test tool Message-ID: <20071025153848.77F5E80E5@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 17:38:48 2007 New Revision: 47951 Modified: pypy/dist/pypy/lang/smalltalk/shadow.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: added tests for class shadow names in image Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Thu Oct 25 17:38:48 2007 @@ -71,12 +71,12 @@ self.instance_kind = WEAK_POINTERS elif format == 6: self.instance_kind = WORDS - if self.instance_kind != 0: + if self.instance_size != 0: raise ClassShadowError("can't have both words and a non-zero " "base instance size") elif 8 <= format <= 11: self.instance_kind = BYTES - if self.instance_kind != 0: + if self.instance_size != 0: raise ClassShadowError("can't have both bytes and a non-zero " "base instance size") elif 12 <= format <= 15: Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 17:38:48 2007 @@ -6,7 +6,9 @@ from pypy.lang.smalltalk import model from pypy.lang.smalltalk import constants from pypy.lang.smalltalk import interpreter - +from pypy.lang.smalltalk import objtable +from pypy.lang.smalltalk import classtable +from pypy.lang.smalltalk import shadow # lazy initialization of test data, ie ImageReader and Float class def setup_module(module): @@ -140,6 +142,18 @@ SO_SEMAPHORE_CLASS = 18 SO_CHARACTER_CLASS = 19""" +def test_name_of_shadow_of_specials(): + image = get_image() + w_doesnot = image.special(constants.SO_DOES_NOT_UNDERSTAND) + assert repr(w_doesnot.shadow_of_my_class()) == "" + assert repr(objtable.w_nil.shadow_of_my_class()) == "" + assert repr(objtable.w_mone.shadow_of_my_class()) == "" + assert repr(objtable.w_zero.shadow_of_my_class()) == "" + assert repr(objtable.w_one.shadow_of_my_class()) == "" + assert repr(objtable.w_two.shadow_of_my_class()) == "" + assert repr(objtable.w_true.shadow_of_my_class()) == "" + assert repr(objtable.w_false.shadow_of_my_class()) == "" + def test_special_classes0(): image = get_image() w = image.special(constants.SO_DOES_NOT_UNDERSTAND) @@ -212,7 +226,6 @@ test_lookup_abs_in_integer(-3) def test_map_mirrors_to_classtable(): - from pypy.lang.smalltalk import classtable, shadow, objtable w_compiledmethod_class = image.special(constants.SO_COMPILEDMETHOD_CLASS) assert w_compiledmethod_class is classtable.w_CompiledMethod w_nil = image.special(constants.SO_NIL) Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Thu Oct 25 17:38:48 2007 @@ -50,7 +50,7 @@ def testSelector(): image = create_squeakimage() w_doesnot = image.special(sqc.SO_DOES_NOT_UNDERSTAND) - print str(w_doesnot.shadow_of_my_class()) + assert repr(w_doesnot.shadow_of_my_class()) == "" def test_do(): testSelector() From fijal at codespeak.net Thu Oct 25 17:45:08 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 25 Oct 2007 17:45:08 +0200 (CEST) Subject: [pypy-svn] r47953 - in pypy/dist/pypy: rpython/memory/gctransform translator/c Message-ID: <20071025154508.590C580E5@code0.codespeak.net> Author: fijal Date: Thu Oct 25 17:45:07 2007 New Revision: 47953 Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py pypy/dist/pypy/translator/c/database.py pypy/dist/pypy/translator/c/funcgen.py pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/c/node.py Log: Speed up slightly inline_helpers, not walking it all over and over. Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Thu Oct 25 17:45:07 2007 @@ -109,6 +109,7 @@ def __init__(self, translator, inline=False): self.translator = translator self.seen_graphs = {} + self.prepared = False self.minimal_transform = {} if translator: self.mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper) @@ -118,6 +119,7 @@ if translator and inline: self.lltype_to_classdef = translator.rtyper.lltype_to_classdef_mapping() self.graphs_to_inline = {} + self.graph_dependencies = {} if self.MinimalGCTransformer: self.minimalgctransformer = self.MinimalGCTransformer(self) else: @@ -134,12 +136,23 @@ self.seen_graphs[graph] = True self.minimal_transform[graph] = True + def prepare_inline_helpers(self, graphs): + from pypy.translator.backendopt.inline import iter_callsites + for graph in graphs: + self.graph_dependencies[graph] = {} + for called, block, i in iter_callsites(graph, None): + if called in self.graphs_to_inline: + self.graph_dependencies[graph][called] = True + self.prepared = True + def inline_helpers(self, graph): + if not self.prepared: + raise Exception("Need to call prepare_inline_helpers first") if self.inline: raise_analyzer = RaiseAnalyzer(self.translator) - for inline_graph in self.graphs_to_inline: + to_enum = self.graph_dependencies.get(graph, self.graphs_to_inline) + for inline_graph in to_enum: try: - # XXX quite inefficient: we go over the function lots of times inline.inline_function(self.translator, inline_graph, graph, self.lltype_to_classdef, raise_analyzer, Modified: pypy/dist/pypy/translator/c/database.py ============================================================================== --- pypy/dist/pypy/translator/c/database.py (original) +++ pypy/dist/pypy/translator/c/database.py Thu Oct 25 17:45:07 2007 @@ -383,3 +383,12 @@ return not fnobj._safe_not_sandboxed else: return "if_external" + + def prepare_inline_helpers(self): + all_nodes = self.globalcontainers() + funcnodes = [node for node in all_nodes if node.nodekind == 'func'] + graphs = [] + for node in funcnodes: + for graph in node.graphs_to_patch(): + graphs.append(graph) + self.gctransformer.prepare_inline_helpers(graphs) Modified: pypy/dist/pypy/translator/c/funcgen.py ============================================================================== --- pypy/dist/pypy/translator/c/funcgen.py (original) +++ pypy/dist/pypy/translator/c/funcgen.py Thu Oct 25 17:45:07 2007 @@ -127,6 +127,9 @@ lltypes[id(v)] = T, typename self.lltypes = lltypes + def graphs_to_patch(self): + yield self.graph + def implementation_end(self): self.lltypes = None self.vars = None Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Thu Oct 25 17:45:07 2007 @@ -173,6 +173,12 @@ # actually generating the source. if db is None: db = self.build_database() + graphs = [] + for node in db.containerlist: + if isinstance(node, FuncNode): + for graph in node.graphs_to_patch(): + graphs.append(graph) + db.gctransformer.prepare_inline_helpers(graphs) for node in db.containerlist: if isinstance(node, FuncNode): for funcgen in node.funcgens: @@ -688,6 +694,7 @@ # sg = SourceGenerator(database, preimplementationlines) sg.set_strategy(targetdir) + database.prepare_inline_helpers() sg.gen_readable_parts_of_source(f) sg.write_extra_sources(sources) Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Thu Oct 25 17:45:07 2007 @@ -10,7 +10,8 @@ from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring from pypy.translator.c.support import cdecl, forward_cdecl, somelettersfrom from pypy.translator.c.support import c_char_array_constant, barebonearray -from pypy.translator.c.primitive import PrimitiveType, isinf, isnan +from pypy.translator.c.primitive import PrimitiveType +from pypy.rlib.rarithmetic import isinf, isnan from pypy.translator.c import extfunc @@ -739,6 +740,11 @@ for s in self.funcgen_implementation(funcgen): yield s + def graphs_to_patch(self): + for funcgen in self.funcgens: + for i in funcgen.graphs_to_patch(): + yield i + def funcgen_implementation(self, funcgen): funcgen.implementation_begin() # recompute implementationtypename as the argnames may have changed From arigo at codespeak.net Thu Oct 25 17:46:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 17:46:17 +0200 (CEST) Subject: [pypy-svn] r47954 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071025154617.693578121@code0.codespeak.net> Author: arigo Date: Thu Oct 25 17:46:17 2007 New Revision: 47954 Modified: pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Log: (akuhn, arigo, toon around, mini.image) Test the decoding of the format from the header done in update_shadow(). Modified: pypy/dist/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Thu Oct 25 17:46:17 2007 @@ -18,11 +18,21 @@ w_class.store(constants.CLASS_NAME_INDEX, objtable.wrap_string(name)) return w_class -def test_empty_class(): - w_class = build_smalltalk_class("Empty", 0x2) +def basicshape(name, format, kind, varsized, instsize): + w_class = build_smalltalk_class(name, format) classshadow = w_class.as_class_get_shadow() - assert classshadow.instance_kind == shadow.POINTERS - assert not classshadow.isvariable() - assert classshadow.instsize() == 0 - assert classshadow.name == "Empty" + assert classshadow.instance_kind == kind + assert classshadow.isvariable() == varsized + assert classshadow.instsize() == instsize + assert classshadow.name == name assert classshadow.s_superclass is w_Object.as_class_get_shadow() + +def test_basic_shape(): + yield basicshape, "Empty", 0x02, shadow.POINTERS, False, 0 + yield basicshape, "Seven", 0x90, shadow.POINTERS, False, 7 + yield basicshape, "Seventyseven", 0x1009C, shadow.POINTERS, False, 77 + yield basicshape, "EmptyVar", 0x102, shadow.POINTERS, True, 0 + yield basicshape, "VarSeven", 0x190, shadow.POINTERS, True, 7 + yield basicshape, "Bytes", 0x402, shadow.BYTES, True, 0 + yield basicshape, "Words", 0x302, shadow.WORDS, True, 0 + yield basicshape, "CompiledMeth", 0xE02, shadow.COMPILED_METHOD, True, 0 From arigo at codespeak.net Thu Oct 25 17:57:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 17:57:31 +0200 (CEST) Subject: [pypy-svn] r47955 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025155731.A76B08121@code0.codespeak.net> Author: arigo Date: Thu Oct 25 17:57:30 2007 New Revision: 47955 Modified: pypy/dist/pypy/lang/smalltalk/model.py Log: (toon, akuhn, arigo) Use the shadow in this __str__(). Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 17:57:30 2007 @@ -85,11 +85,10 @@ return self.w_class def __str__(self): - # XXX use the shadow of my class - if self.size() >= 9: - return ''.join(self.fetch(constants.CLASS_NAME_INDEX).bytes) + " class" + if isinstance(self, W_PointersObject) and self._shadow is not None: + return "%s class" % (self.as_class_get_shadow().name or '?',) else: - return "a " + ''.join(self.getclass().fetch(constants.CLASS_NAME_INDEX).bytes) + return "a %s" % (self.shadow_of_my_class().name or '?',) def invariant(self): return (W_AbstractObjectWithIdentityHash.invariant(self) and From tverwaes at codespeak.net Thu Oct 25 18:00:39 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 18:00:39 +0200 (CEST) Subject: [pypy-svn] r47956 - in pypy/dist/pypy/lang/smalltalk: . tool Message-ID: <20071025160039.C2BAA8121@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 18:00:39 2007 New Revision: 47956 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: (toon, arigo, akuhn) repr of AbstractObjectWithClassReference Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 18:00:39 2007 @@ -84,6 +84,9 @@ def getclass(self): return self.w_class + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, self) + def __str__(self): if isinstance(self, W_PointersObject) and self._shadow is not None: return "%s class" % (self.as_class_get_shadow().name or '?',) Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Thu Oct 25 18:00:39 2007 @@ -1,41 +1,41 @@ import autopath import py -from pypy.lang.smalltalk import squeakimage as sq -from pypy.lang.smalltalk import constants as sqc -from pypy.lang.smalltalk import model as sqm -from pypy.lang.smalltalk import interpreter as sqi +from pypy.lang.smalltalk import squeakimage +from pypy.lang.smalltalk import constants +from pypy.lang.smalltalk import model +from pypy.lang.smalltalk import interpreter mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') def get_miniimage(): - return sq.ImageReader(sq.Stream(mini_image.open())) + return squeakimage.ImageReader(squeakimage.Stream(mini_image.open())) def create_squeakimage(): example = get_miniimage() example.initialize() - image = sq.SqueakImage() + image = squeakimage.SqueakImage() image.from_reader(example) return image def printStringsInImage(): image = create_squeakimage() for each in image.objects: - if isinstance(each,sqm.W_BytesObject): + if isinstance(each,model.W_BytesObject): print each.bytes def testCompiledMethods(): image = create_squeakimage() amethod = None - w_smallint_class = image.special(sqc.SO_SMALLINTEGER_CLASS) + w_smallint_class = image.special(constants.SO_SMALLINTEGER_CLASS) - interp = sqi.Interpreter() + interp = interpreter.Interpreter() amethod = w_smallint_class.lookup("abs") # First literal of the abs method is # a real smalltalk int - w_frame = amethod.createFrame(sqm.W_SmallInteger(3), []) + w_frame = amethod.createFrame(model.W_SmallInteger(3), []) interp.activeContext = w_frame print amethod @@ -44,13 +44,14 @@ try: interp.step() print interp.activeContext.stack - except sqi.ReturnFromTopLevel, e: + except interpreter.ReturnFromTopLevel, e: return e.object def testSelector(): image = create_squeakimage() - w_doesnot = image.special(sqc.SO_DOES_NOT_UNDERSTAND) + w_doesnot = image.special(constants.SO_DOES_NOT_UNDERSTAND) assert repr(w_doesnot.shadow_of_my_class()) == "" + print w_doesnot.getclass().fetch(constants.CLASS_METHODDICT_INDEX)._vars def test_do(): testSelector() From arigo at codespeak.net Thu Oct 25 18:29:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 18:29:26 +0200 (CEST) Subject: [pypy-svn] r47959 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025162926.CED93813B@code0.codespeak.net> Author: arigo Date: Thu Oct 25 18:29:26 2007 New Revision: 47959 Modified: pypy/dist/pypy/lang/smalltalk/classtable.py pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/shadow.py pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Log: A test to load and decode the method dictionary. Modified: pypy/dist/pypy/lang/smalltalk/classtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/classtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/classtable.py Thu Oct 25 18:29:26 2007 @@ -82,7 +82,8 @@ # ___________________________________________________________________________ # Other classes -def define_cls(cls_nm, supercls_nm, instvarsize=0, format=shadow.POINTERS): +def define_cls(cls_nm, supercls_nm, instvarsize=0, format=shadow.POINTERS, + varsized=False): assert cls_nm.startswith("w_") meta_nm = cls_nm + "Class" meta_super_nm = supercls_nm + "Class" @@ -97,6 +98,7 @@ classtable[supercls_nm], w_meta_cls, format=format, + varsized=varsized, name=cls_nm[2:]) define_cls("w_Magnitude", "w_Object") @@ -108,12 +110,14 @@ define_cls("w_Collection", "w_Object") define_cls("w_SequencableCollection", "w_Collection") define_cls("w_ArrayedCollection", "w_SequencableCollection") +define_cls("w_Array", "w_ArrayedCollection", varsized=True) define_cls("w_String", "w_ArrayedCollection", format=shadow.BYTES) define_cls("w_UndefinedObject", "w_Object") define_cls("w_Boolean", "w_Object") define_cls("w_True", "w_Boolean") define_cls("w_False", "w_Boolean") define_cls("w_ByteArray", "w_ArrayedCollection", format=shadow.BYTES) +define_cls("w_MethodDict", "w_Object", instvarsize=2, varsized=True) define_cls("w_CompiledMethod", "w_ByteArray", format=shadow.COMPILED_METHOD) define_cls("w_MethodContext", "w_Object") define_cls("w_ContextPart", "w_Object") Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Thu Oct 25 18:29:26 2007 @@ -14,6 +14,7 @@ CLASS_FORMAT_INDEX = 2 CLASS_NAME_INDEX = 6 # in the mini.image, at least +METHODDICT_TALLY_INDEX = 0 METHODDICT_VALUES_INDEX = 1 METHODDICT_NAMES_INDEX = 2 @@ -79,6 +80,7 @@ # XXX more missing? classes_in_special_object_table = { "SmallInteger": SO_SMALLINTEGER_CLASS, + "Array": SO_ARRAY_CLASS, "String": SO_STRING_CLASS, "Float": SO_FLOAT_CLASS, "MethodContext": SO_METHODCONTEXT_CLASS, Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Thu Oct 25 18:29:26 2007 @@ -88,9 +88,18 @@ w_name = w_self.fetch(constants.CLASS_NAME_INDEX) if isinstance(w_name, model.W_BytesObject): self.name = w_name.as_string() - # XXX read the methoddict - # ... - + # read the methoddict + w_methoddict = w_self.fetch(constants.CLASS_METHODDICT_INDEX) + w_values = w_methoddict.fetch(constants.METHODDICT_VALUES_INDEX) + size = w_methoddict.size() - constants.METHODDICT_NAMES_INDEX + for i in range(size): + w_selector = w_methoddict.fetch(constants.METHODDICT_NAMES_INDEX+i) + if w_selector is not objtable.w_nil: + if not isinstance(w_selector, model.W_BytesObject): + raise ClassShadowError("bogus selector in method dict") + selector = w_selector.as_string() + w_compiledmethod = w_values.fetch(i) + self.methoddict[selector] = w_compiledmethod # for the rest, we need to reset invalid to False already so # that cycles in the superclass and/or metaclass chains don't # cause infinite recursion Modified: pypy/dist/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Thu Oct 25 18:29:26 2007 @@ -1,18 +1,42 @@ +import random from pypy.lang.smalltalk import model, shadow, classtable, constants, objtable w_Object = classtable.classtable['w_Object'] w_Metaclass = classtable.classtable['w_Metaclass'] +w_MethodDict = classtable.classtable['w_MethodDict'] +w_Array = classtable.classtable['w_Array'] + +def build_methoddict(methods): + size = int(len(methods) * 1.5) + w_methoddict = w_MethodDict.as_class_get_shadow().new(size) + w_array = w_Array.as_class_get_shadow().new(size) + for i in range(size): + w_array.store(i, objtable.w_nil) + w_methoddict.store(constants.METHODDICT_NAMES_INDEX+i, objtable.w_nil) + w_tally = objtable.wrap_int(len(methods)) + w_methoddict.store(constants.METHODDICT_TALLY_INDEX, w_tally) + w_methoddict.store(constants.METHODDICT_VALUES_INDEX, w_array) + positions = range(size) + random.shuffle(positions) + for selector, w_compiledmethod in methods.items(): + pos = positions.pop() + w_selector = objtable.wrap_string(selector) + w_methoddict.store(constants.METHODDICT_NAMES_INDEX+pos, w_selector) + w_array.store(pos, w_compiledmethod) + #print w_methoddict._vars + return w_methoddict def build_smalltalk_class(name, format, w_superclass=w_Object, - w_classofclass=None): + w_classofclass=None, methods={}): if w_classofclass is None: w_classofclass = build_smalltalk_class(None, 0x94, w_superclass.w_class, w_Metaclass) + w_methoddict = build_methoddict(methods) size = constants.CLASS_NAME_INDEX + 1 w_class = model.W_PointersObject(w_classofclass, size) w_class.store(constants.CLASS_SUPERCLASS_INDEX, w_superclass) - #w_class.store(constants.CLASS_METHODDICT_INDEX, ...) + w_class.store(constants.CLASS_METHODDICT_INDEX, w_methoddict) w_class.store(constants.CLASS_FORMAT_INDEX, objtable.wrap_int(format)) if name is not None: w_class.store(constants.CLASS_NAME_INDEX, objtable.wrap_string(name)) @@ -32,7 +56,15 @@ yield basicshape, "Seven", 0x90, shadow.POINTERS, False, 7 yield basicshape, "Seventyseven", 0x1009C, shadow.POINTERS, False, 77 yield basicshape, "EmptyVar", 0x102, shadow.POINTERS, True, 0 + yield basicshape, "VarTwo", 0x3986, shadow.POINTERS, True, 2 yield basicshape, "VarSeven", 0x190, shadow.POINTERS, True, 7 yield basicshape, "Bytes", 0x402, shadow.BYTES, True, 0 yield basicshape, "Words", 0x302, shadow.WORDS, True, 0 yield basicshape, "CompiledMeth", 0xE02, shadow.COMPILED_METHOD, True, 0 + +def test_methoddict(): + methods = {'foo': 'foo_method', + 'bar': 'bar_method'} + w_class = build_smalltalk_class("Demo", 0x90, methods=methods) + classshadow = w_class.as_class_get_shadow() + assert classshadow.methoddict == methods From tverwaes at codespeak.net Thu Oct 25 18:35:29 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 18:35:29 +0200 (CEST) Subject: [pypy-svn] r47961 - in pypy/dist/pypy/lang/smalltalk: . test tool Message-ID: <20071025163529.0A200813D@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 18:35:29 2007 New Revision: 47961 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: (toon, arigo, cfbolz) invalidate shadows after image loading Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 18:35:29 2007 @@ -319,6 +319,8 @@ w_pointersobject._vars = [g_object.w_object for g_object in self.pointers] w_pointersobject.w_class = self.g_class.w_object w_pointersobject.hash = self.chunk.hash12 + if w_pointersobject._shadow is not None: + w_pointersobject._shadow.invalidate() def fillin_wordsobject(self, w_wordsobject): w_wordsobject.words = self.chunk.data Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 18:35:29 2007 @@ -220,7 +220,7 @@ return e.object def test_lookup_neg_abs_in_integer(): - py.test.skip("TOFIX methodlookup 'negated' fails in shadow SmallInteger") + #py.test.skip("TOFIX methodlookup 'negated' fails in shadow SmallInteger") # Fails due to same reason because of which # classmirror-methodlookup fails test_lookup_abs_in_integer(-3) Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Thu Oct 25 18:35:29 2007 @@ -51,7 +51,14 @@ image = create_squeakimage() w_doesnot = image.special(constants.SO_DOES_NOT_UNDERSTAND) assert repr(w_doesnot.shadow_of_my_class()) == "" + print w_doesnot.getclass().fetch(constants.CLASS_METHODDICT_INDEX).shadow_of_my_class().instance_kind + print w_doesnot.getclass().fetch(constants.CLASS_METHODDICT_INDEX).shadow_of_my_class().instance_size + print print w_doesnot.getclass().fetch(constants.CLASS_METHODDICT_INDEX)._vars + print + print w_doesnot.getclass().fetch(constants.CLASS_METHODDICT_INDEX)._vars[constants.METHODDICT_NAMES_INDEX:] + print + print w_doesnot.getclass().fetch(constants.CLASS_METHODDICT_INDEX)._vars[constants.METHODDICT_VALUES_INDEX]._vars def test_do(): testSelector() From arigo at codespeak.net Thu Oct 25 18:43:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 18:43:23 +0200 (CEST) Subject: [pypy-svn] r47962 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025164323.990648137@code0.codespeak.net> Author: arigo Date: Thu Oct 25 18:43:23 2007 New Revision: 47962 Modified: pypy/dist/pypy/lang/smalltalk/shadow.py pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Log: (toon, arigo) Keep the wrapped Symbol objects as keys of the methoddict of the class shadows. Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Thu Oct 25 18:43:23 2007 @@ -95,11 +95,8 @@ for i in range(size): w_selector = w_methoddict.fetch(constants.METHODDICT_NAMES_INDEX+i) if w_selector is not objtable.w_nil: - if not isinstance(w_selector, model.W_BytesObject): - raise ClassShadowError("bogus selector in method dict") - selector = w_selector.as_string() w_compiledmethod = w_values.fetch(i) - self.methoddict[selector] = w_compiledmethod + self.methoddict[w_selector] = w_compiledmethod # for the rest, we need to reset invalid to False already so # that cycles in the superclass and/or metaclass chains don't # cause infinite recursion Modified: pypy/dist/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Thu Oct 25 18:43:23 2007 @@ -18,9 +18,8 @@ w_methoddict.store(constants.METHODDICT_VALUES_INDEX, w_array) positions = range(size) random.shuffle(positions) - for selector, w_compiledmethod in methods.items(): + for w_selector, w_compiledmethod in methods.items(): pos = positions.pop() - w_selector = objtable.wrap_string(selector) w_methoddict.store(constants.METHODDICT_NAMES_INDEX+pos, w_selector) w_array.store(pos, w_compiledmethod) #print w_methoddict._vars @@ -63,8 +62,8 @@ yield basicshape, "CompiledMeth", 0xE02, shadow.COMPILED_METHOD, True, 0 def test_methoddict(): - methods = {'foo': 'foo_method', - 'bar': 'bar_method'} + methods = {objtable.wrap_string('foo'): 'foo_method', + objtable.wrap_string('bar'): 'bar_method'} w_class = build_smalltalk_class("Demo", 0x90, methods=methods) classshadow = w_class.as_class_get_shadow() assert classshadow.methoddict == methods From tverwaes at codespeak.net Thu Oct 25 18:46:02 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 18:46:02 +0200 (CEST) Subject: [pypy-svn] r47963 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071025164602.D4AD08137@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 18:46:02 2007 New Revision: 47963 Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: (arigo, tverwaes) now the abs of -3 works using shadow methodlookup Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 18:46:02 2007 @@ -193,7 +193,6 @@ w_object = model.W_SmallInteger(int) - # XXX # Should get this from w_object w_smallint_class = image.special(constants.SO_SMALLINTEGER_CLASS) w_method = w_smallint_class.lookup("abs") @@ -220,9 +219,6 @@ return e.object def test_lookup_neg_abs_in_integer(): - #py.test.skip("TOFIX methodlookup 'negated' fails in shadow SmallInteger") - # Fails due to same reason because of which - # classmirror-methodlookup fails test_lookup_abs_in_integer(-3) def test_map_mirrors_to_classtable(): From fijal at codespeak.net Thu Oct 25 18:50:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 25 Oct 2007 18:50:54 +0200 (CEST) Subject: [pypy-svn] r47964 - pypy/dist/pypy/module/zipimport Message-ID: <20071025165054.9EB40813A@code0.codespeak.net> Author: fijal Date: Thu Oct 25 18:50:54 2007 New Revision: 47964 Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py Log: Whack until we have pypy-c starting. The zipimporter seems to be invoked recursively for the same path. deny such possibility. This would be a no-issue when we would have interp-level zipfile module. Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/interp_zipimport.py Thu Oct 25 18:50:54 2007 @@ -167,9 +167,13 @@ def descr_new_zipimporter(space, w_type, name): try: - return zip_importer_cache[name] + result = zip_importer_cache[name] + if result is None: + raise OperationError(space.w_ImportError, space.wrap( + "Cannot import %s from zipfile, recursion detected or" + "already tried and failed" % (name,))) except KeyError: - pass + zip_importer_cache[name] = None ok = False parts = name.split(os.path.sep) filename = "" # make annotator happy @@ -197,11 +201,11 @@ w_ZipFile = space.getattr(w_zipfile, space.wrap('ZipFile')) try: w_dir = space.call(w_ZipFile, space.newlist([space.wrap(filename)])) - except OperationError: # we catch everything as this function + except OperationError, e: # we catch everything as this function raise OperationError(space.w_ImportError, space.wrap( "%s seems not to be a zipfile" % (filename,))) result = space.wrap(W_ZipImporter(space, name, w_dir, w_zipfile)) - zip_importer_cache[filename] = result + zip_importer_cache[name] = result return result descr_new_zipimporter.unwrap_spec = [ObjSpace, W_Root, str] From fijal at codespeak.net Thu Oct 25 18:52:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 25 Oct 2007 18:52:57 +0200 (CEST) Subject: [pypy-svn] r47965 - pypy/dist/lib-python/modified-2.4.1/test Message-ID: <20071025165257.A078A813A@code0.codespeak.net> Author: fijal Date: Thu Oct 25 18:52:54 2007 New Revision: 47965 Added: pypy/dist/lib-python/modified-2.4.1/test/test_zipimport.py - copied, changed from r47270, pypy/dist/lib-python/2.4.1/test/test_zipimport.py Log: Makes no sense to clear this dictionary as we don't expose it. From tverwaes at codespeak.net Thu Oct 25 18:53:01 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 18:53:01 +0200 (CEST) Subject: [pypy-svn] r47966 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071025165301.AB630813D@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 18:53:00 2007 New Revision: 47966 Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: (arigo, tverwaes) fixing abs evaluation returnvalue-test Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 18:53:00 2007 @@ -216,7 +216,8 @@ interp.step() print interp.w_active_context.stack except interpreter.ReturnFromTopLevel, e: - return e.object + assert e.object.value == abs(int) + return def test_lookup_neg_abs_in_integer(): test_lookup_abs_in_integer(-3) From arigo at codespeak.net Thu Oct 25 19:04:56 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 19:04:56 +0200 (CEST) Subject: [pypy-svn] r47967 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025170456.4A85F812D@code0.codespeak.net> Author: arigo Date: Thu Oct 25 19:04:55 2007 New Revision: 47967 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/shadow.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py pypy/dist/pypy/lang/smalltalk/test/test_model.py Log: (toon, arigo) Move the testing version of lookup() out of W_PointerObject into test_miniimage. Raise MethodNotFound instead of returning None when a method is not found. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 19:04:55 2007 @@ -133,7 +133,7 @@ def _sendSelector(self, selector, argcount, interp, receiver, receiverclassshadow): method = receiverclassshadow.lookup(selector) - assert method + # XXX catch MethodNotFound here and send doesNotUnderstand: if method.primitive: func = primitives.prim_table[method.primitive] try: Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 19:04:55 2007 @@ -3,9 +3,6 @@ from pypy.rlib.rarithmetic import intmask from pypy.lang.smalltalk import constants -class MethodNotFound(Exception): - pass - class W_Object(object): def size(self): @@ -131,36 +128,6 @@ shadow.check_for_updates() return shadow - def compiledmethodnamed(self, methodname): - # XXX kill me. Temporary, for testing - w_methoddict = self.fetch(constants.CLASS_METHODDICT_INDEX)._vars - names = w_methoddict[constants.METHODDICT_NAMES_INDEX:] - values = w_methoddict[constants.METHODDICT_VALUES_INDEX]._vars - for var in names: - if isinstance(var, W_BytesObject): - if str(var) == str(methodname): - return values[names.index(var)] - raise MethodNotFound - - def lookup(self, methodname): - # XXX kill me. Temporary, for testing - from pypy.lang.smalltalk import objtable - in_class = self - while in_class != None: - try: - return in_class.compiledmethodnamed(methodname) - except MethodNotFound: - # Current hack because we don't have a ref to the real - # nil yet... XXX XXX XXX - try: - new_class = in_class._vars[constants.CLASS_SUPERCLASS_INDEX] - if new_class is objtable.w_nil: - raise IndexError - else: - in_class = new_class - except IndexError: - return self.lookup("doesNotUnderstand:") - class W_BytesObject(W_AbstractObjectWithClassReference): def __init__(self, w_class, size): W_AbstractObjectWithClassReference.__init__(self, w_class) Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Thu Oct 25 19:04:55 2007 @@ -17,6 +17,9 @@ WEAK_POINTERS = 3 COMPILED_METHOD = 4 +class MethodNotFound(Exception): + pass + class ClassShadowError(Exception): pass @@ -168,7 +171,7 @@ elif self.s_superclass != None: return self.s_superclass.lookup(selector) else: - return None + raise MethodNotFound def installmethod(self, selector, method): "NOT_RPYTHON" # this is only for testing. Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 19:04:55 2007 @@ -33,7 +33,30 @@ def get_float_class(): image = get_image() return image.special(constants.SO_FLOAT_CLASS) - + +# ------ custom lookup implementations -------------------------------- + +def compiledmethodnamed(w_class, methodname): + w_methoddict = w_class.fetch(constants.CLASS_METHODDICT_INDEX)._vars + names = w_methoddict[constants.METHODDICT_NAMES_INDEX:] + values = w_methoddict[constants.METHODDICT_VALUES_INDEX]._vars + for var in names: + if isinstance(var, model.W_BytesObject): + if str(var) == str(methodname): + return values[names.index(var)] + raise shadow.MethodNotFound + +def lookup(w_class, methodname): + in_class = w_class + while in_class != None: + try: + return compiledmethodnamed(in_class, methodname) + except shadow.MethodNotFound: + pass + in_class = in_class._vars[constants.CLASS_SUPERCLASS_INDEX] + if in_class is objtable.w_nil: + raise shadow.MethodNotFound + # ------ tests ------------------------------------------ def test_miniimageexists(): @@ -195,7 +218,7 @@ # Should get this from w_object w_smallint_class = image.special(constants.SO_SMALLINTEGER_CLASS) - w_method = w_smallint_class.lookup("abs") + w_method = lookup(w_smallint_class, "abs") # XXX # currently still using highlevel lookup directly pointing to Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Thu Oct 25 19:04:55 2007 @@ -1,5 +1,6 @@ import py from pypy.lang.smalltalk import model, shadow +from pypy.lang.smalltalk.shadow import MethodNotFound import pypy.lang.smalltalk.classtable as ct mockclass = ct.bootstrap_class @@ -56,10 +57,10 @@ subshadow.methoddict["foo"] = 3 assert shadow.lookup("foo") == 1 assert shadow.lookup("bar") == 2 - assert shadow.lookup("zork") == None + py.test.raises(MethodNotFound, shadow.lookup, "zork") assert subshadow.lookup("foo") == 3 assert subshadow.lookup("bar") == 2 - assert subshadow.lookup("zork") == None + py.test.raises(MethodNotFound, subshadow.lookup, "zork") def test_w_compiledin(): w_super = mockclass(0) From arigo at codespeak.net Thu Oct 25 19:36:16 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 19:36:16 +0200 (CEST) Subject: [pypy-svn] r47970 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025173616.EC2BD8137@code0.codespeak.net> Author: arigo Date: Thu Oct 25 19:36:16 2007 New Revision: 47970 Modified: pypy/dist/pypy/lang/smalltalk/shadow.py Log: Avoid repr'ing as "". Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Thu Oct 25 19:36:16 2007 @@ -163,7 +163,7 @@ # Methods for querying the format word, taken from the blue book: def __repr__(self): - return "" % (self.name,) + return "" % (self.name or '?',) def lookup(self, selector): if selector in self.methoddict: From jacob at codespeak.net Thu Oct 25 19:51:18 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Thu, 25 Oct 2007 19:51:18 +0200 (CEST) Subject: [pypy-svn] r47972 - pypy/branch/dist-future-fixing/lib-python/modified-2.4.1 Message-ID: <20071025175118.A84788144@code0.codespeak.net> Author: jacob Date: Thu Oct 25 19:51:18 2007 New Revision: 47972 Modified: pypy/branch/dist-future-fixing/lib-python/modified-2.4.1/__future__.py Log: Restored the CO_GENERATOR_ALLOWED constant to its proper value. Modified: pypy/branch/dist-future-fixing/lib-python/modified-2.4.1/__future__.py ============================================================================== --- pypy/branch/dist-future-fixing/lib-python/modified-2.4.1/__future__.py (original) +++ pypy/branch/dist-future-fixing/lib-python/modified-2.4.1/__future__.py Thu Oct 25 19:51:18 2007 @@ -62,7 +62,7 @@ # they're not exported in __all__, because they don't really belong to # this module. CO_NESTED = 0x0010 # nested_scopes -CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000) +CO_GENERATOR_ALLOWED = 0x0100 # generators CO_FUTURE_DIVISION = 0x2000 # division CO_FUTURE_ABSIMPORT = 0x4000 # absolute_import CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement From jacob at codespeak.net Thu Oct 25 19:52:32 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Thu, 25 Oct 2007 19:52:32 +0200 (CEST) Subject: [pypy-svn] r47973 - pypy/branch/dist-future-fixing/pypy/module/recparser Message-ID: <20071025175232.D377E8145@code0.codespeak.net> Author: jacob Date: Thu Oct 25 19:52:32 2007 New Revision: 47973 Modified: pypy/branch/dist-future-fixing/pypy/module/recparser/__init__.py Log: Removed reduntdant import. Modified: pypy/branch/dist-future-fixing/pypy/module/recparser/__init__.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/module/recparser/__init__.py (original) +++ pypy/branch/dist-future-fixing/pypy/module/recparser/__init__.py Thu Oct 25 19:52:32 2007 @@ -1,5 +1,4 @@ from pypy.interpreter.error import OperationError, debug_print -import pypy.interpreter.pyparser.pythonparse from pypy.interpreter.mixedmodule import MixedModule From jacob at codespeak.net Thu Oct 25 19:54:21 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Thu, 25 Oct 2007 19:54:21 +0200 (CEST) Subject: [pypy-svn] r47974 - pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/data Message-ID: <20071025175421.3387E8145@code0.codespeak.net> Author: jacob Date: Thu Oct 25 19:54:19 2007 New Revision: 47974 Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/data/Grammar2.5a Log: Reverted the 2.5a grammar to what it was before the wild hacking at the Vilnius sprint. This will break some faulty tests. Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/data/Grammar2.5a ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/data/Grammar2.5a (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/data/Grammar2.5a Thu Oct 25 19:54:19 2007 @@ -24,16 +24,10 @@ # file_input is a module or sequence of commands read from an input file; # eval_input is the input for the eval() and input() functions. # NB: compound_stmt in single_input is followed by extra NEWLINE! - -single_input: NEWLINE | future_import_list [';'] | [future_import_list ';'] simple_stmt | compound_stmt NEWLINE -file_input: (future_import_list [';'] NEWLINE)* (NEWLINE | stmt)* ENDMARKER +single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +file_input: (NEWLINE | stmt)* ENDMARKER eval_input: testlist NEWLINE* ENDMARKER -future_import_list: import_from_future (';' import_from_future)* -import_from_future: 'from' '__future__' 'import' ('(' future_import_as_names [','] ')' | future_import_as_names) -future_import_feature: NAME [('as'|NAME) NAME] -future_import_as_names: future_import_feature (',' future_import_feature)* - decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ funcdef: [decorators] 'def' NAME parameters ':' suite @@ -69,12 +63,13 @@ exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] try_stmt: ('try' ':' suite (except_clause ':' suite)+ #diagram:break ['else' ':' suite] | 'try' ':' suite 'finally' ':' suite) +with_stmt: 'with' test [ 'as' expr ] ':' suite # NB compile.c makes sure that the default except clause is last except_clause: 'except' [test [',' test]] suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT From jacob at codespeak.net Thu Oct 25 19:57:47 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Thu, 25 Oct 2007 19:57:47 +0200 (CEST) Subject: [pypy-svn] r47975 - in pypy/branch/dist-future-fixing/pypy/interpreter/pyparser: . test Message-ID: <20071025175747.E963D8145@code0.codespeak.net> Author: jacob Date: Thu Oct 25 19:57:46 2007 New Revision: 47975 Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py Log: Made the future automaton use the __future__.py module directly. Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py Thu Oct 25 19:57:46 2007 @@ -64,7 +64,6 @@ def __init__(self, string): self.s = string - self.end = len(string) self.pos = 0 self.docstringConsumed = False self.flags = 0 @@ -237,10 +236,47 @@ self.getMore(parenList=parenList) def setFlag(self, feature): - if feature == "division": - self.flags |= CO_FUTURE_DIVISION - elif feature == "generators": - self.flags |= CO_GENERATOR_ALLOWED - elif feature == "with_statement": - self.flags |= CO_FUTURE_WITH_STATEMENT - + try: + self.flags |= futureFlags.compiler_features[feature] + except IndexError: + pass + +from codeop import PyCF_DONT_IMPLY_DEDENT +from pypy.interpreter.error import OperationError + +from pypy.tool import stdlib___future__ as future + +class FutureFlags(object): + def __init__(self, version): + compiler_flags = 0 + self.compiler_features = {} + self.mandatory_flags = 0 + for fname in future.all_feature_names: + feature = getattr(future, fname) + if version >= feature.getOptionalRelease(): + flag = feature.compiler_flag + compiler_flags |= flag + self.compiler_features[fname] = flag + if version >= feature.getMandatoryRelease(): + self.mandatory_flags |= feature.compiler_flag + self.allowed_flags = compiler_flags | PyCF_DONT_IMPLY_DEDENT + + def get_flag_names(self, space, flags): + if flags & ~self.allowed_flags: + raise OperationError(space.w_ValueError, + space.wrap("compile(): unrecognized flags")) + flag_names = [] + for name, value in self.compiler_features.items(): + if flags & value: + flag_names.append(name) + return flag_names + +# XXX This is a hack to deal with the fact that we currently are +# using the Python 2.4.1 libraries even when running Python 2.5 +# and that we have a hacked __future__ module. +from pypy.config.pypyoption import get_pypy_config +config = get_pypy_config(translating=False) +if config.objspace.pyversion == '2.4': + futureFlags = FutureFlags((2, 4, 4, 'final', 0)) +else: + futureFlags = FutureFlags((2, 5, 0, 'final', 0)) Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py Thu Oct 25 19:57:46 2007 @@ -1,7 +1,6 @@ import py import pypy.interpreter.pyparser.future as future -from pypy.interpreter.astcompiler.consts import CO_GENERATOR_ALLOWED, \ - CO_FUTURE_DIVISION, CO_FUTURE_WITH_STATEMENT +from pypy.tool import stdlib___future__ as fut def run(s): f = future.FutureAutomaton(s) @@ -15,7 +14,7 @@ s = '"Docstring\\" "\nfrom __future__ import division\n' f = run(s) assert f.pos == len(s) - assert f.flags == CO_FUTURE_DIVISION + assert f.flags == fut.CO_FUTURE_DIVISION def test_comment(): s = '# A comment about nothing ;\n' @@ -49,88 +48,88 @@ s = 'from __future__ import division\n' f = run(s) assert f.pos == len(s) - assert f.flags == CO_FUTURE_DIVISION + assert f.flags == fut.CO_FUTURE_DIVISION def test_froms(): s = 'from __future__ import division, generators, with_statement\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED | - CO_FUTURE_WITH_STATEMENT) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED | + fut.CO_FUTURE_WITH_STATEMENT) def test_from_as(): s = 'from __future__ import division as b\n' f = run(s) assert f.pos == len(s) - assert f.flags == CO_FUTURE_DIVISION + assert f.flags == fut.CO_FUTURE_DIVISION def test_froms_as(): s = 'from __future__ import division as b, generators as c\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED) def test_from_paren(): s = 'from __future__ import (division)\n' f = run(s) assert f.pos == len(s) - assert f.flags == CO_FUTURE_DIVISION + assert f.flags == fut.CO_FUTURE_DIVISION def test_froms_paren(): s = 'from __future__ import (division, generators)\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED) def test_froms_paren_as(): s = 'from __future__ import (division as b, generators,)\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED) def test_multiline(): s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,)\nfrom __future__ import with_statement\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED | - CO_FUTURE_WITH_STATEMENT) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED | + fut.CO_FUTURE_WITH_STATEMENT) def test_windows_style_lineendings(): s = '"abc" #def\r\n #ghi\r\nfrom __future__ import (division as b, generators,)\r\nfrom __future__ import with_statement\r\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED | - CO_FUTURE_WITH_STATEMENT) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED | + fut.CO_FUTURE_WITH_STATEMENT) def test_mac_style_lineendings(): s = '"abc" #def\r #ghi\rfrom __future__ import (division as b, generators,)\rfrom __future__ import with_statement\r' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED | - CO_FUTURE_WITH_STATEMENT) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED | + fut.CO_FUTURE_WITH_STATEMENT) def test_semicolon(): s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,); from __future__ import with_statement\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED | - CO_FUTURE_WITH_STATEMENT) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED | + fut.CO_FUTURE_WITH_STATEMENT) def test_full_chain(): s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,); from __future__ import with_statement\n' flags = future.getFutures(s) - assert flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED | - CO_FUTURE_WITH_STATEMENT) + assert flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED | + fut.CO_FUTURE_WITH_STATEMENT) def test_intervening_code(): s = 'from __future__ import (division as b, generators,)\nfrom sys import modules\nfrom __future__ import with_statement\n' flags = future.getFutures(s) - assert flags & CO_FUTURE_WITH_STATEMENT == 0 + assert flags & fut.CO_FUTURE_WITH_STATEMENT == 0 From niko at codespeak.net Thu Oct 25 19:57:53 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 25 Oct 2007 19:57:53 +0200 (CEST) Subject: [pypy-svn] r47976 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025175753.34F2D814E@code0.codespeak.net> Author: niko Date: Thu Oct 25 19:57:52 2007 New Revision: 47976 Modified: pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/shadow.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: 1. Refactor the W_MethodContext class into several classes: W_ContextPart, W_BlockContext, and W_MethodContext 2. Tweak the block primitives so that they actually work now, at least for simple cases Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Thu Oct 25 19:57:52 2007 @@ -21,14 +21,22 @@ ASSOCIATION_KEY_INDEX = 0 ASSOCIATION_VALUE_INDEX = 1 -BLKCTX_CALLER_INDEX = 0 -BLKCTX_INSTRUCTION_POINTER_INDEX = 1 -BLKCTX_STACK_POINTER_INDEX = 2 +CTXPART_SENDER_INDEX = 0 +CTXPART_PC_INDEX = 1 +CTXPART_STACKP_INDEX = 2 + +# Extends CTXPART_* BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX = 3 BLKCTX_INITIAL_IP_INDEX = 4 BLKCTX_HOME_INDEX = 5 BLKCTX_TEMP_FRAME_START = 6 +# Extends CTXPART_* +MTHDCTX_METHOD = 3 +MTHDCTX_RECEIVER_MAP = 4 +MTHDCTX_RECEIVER = 5 +MTHDCTX_TEMP_FRAME_START = 6 + # ----- special objects indices ------- SO_NIL = 0 @@ -83,6 +91,7 @@ "Array": SO_ARRAY_CLASS, "String": SO_STRING_CLASS, "Float": SO_FLOAT_CLASS, + "BlockContext": SO_BLOCKCONTEXT_CLASS, "MethodContext": SO_METHODCONTEXT_CLASS, "Character": SO_CHARACTER_CLASS, "ByteArray": SO_BYTEARRAY_CLASS, Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 19:57:52 2007 @@ -1,6 +1,7 @@ import py from pypy.lang.smalltalk import model, constants, primitives from pypy.lang.smalltalk import objtable +from pypy.lang.smalltalk.model import W_ContextPart class MissingBytecode(NotImplementedError): @@ -9,55 +10,49 @@ class IllegalStoreError(Exception): """Illegal Store.""" +class IllegalFetchError(Exception): + """Illegal Fetch.""" -class W_MethodContext(model.W_AbstractObjectWithIdentityHash): - def __init__(self, method, receiver, arguments, sender = None): - self.method = method - self.receiver = receiver - self.sender = sender - self.stack = [] - self.temps = arguments + [None] * method.tempsize - self.pc = 0 - - def getclass(self): - from pypy.lang.smalltalk.classtable import w_MethodContext - return w_MethodContext +class Interpreter: - def pop(self): - return self.stack.pop() + TRUE = objtable.w_true + FALSE = objtable.w_false + NIL = objtable.w_nil + MONE = objtable.w_mone + ZERO = objtable.w_zero + ONE = objtable.w_one + TWO = objtable.w_two + + def __init__(self): + self.w_active_context = None - def push(self, w_v): - self.stack.append(w_v) + + def interpret(self): + try: + while True: + self.step() + except ReturnFromTopLevel, e: + return e.object - def top(self): - return self.peek(0) + def step(self): + next = self.w_active_context.getNextBytecode() + bytecodeimpl = BYTECODE_TABLE[next] + bytecodeimpl(self.w_active_context, self) - def peek(self, idx): - return self.stack[-(idx+1)] - - def pop_n(self, n): - self.stack = self.stack[:len(self.stack)-n] - - def getByte(self): - bytecode = self.method.bytes[self.pc] - currentBytecode = ord(bytecode) - self.pc = self.pc + 1 - return currentBytecode - - def getNextBytecode(self): - self.currentBytecode = self.getByte() - return self.currentBytecode - - def gettemp(self, index): - return self.temps[index] +class ReturnFromTopLevel(Exception): + def __init__(self, object): + self.object = object - def settemp(self, index, w_value): - self.temps[index] = w_value +# ___________________________________________________________________________ +# Bytecode Implementations: +# +# "self" is always a W_ContextPart instance. +class __extend__(W_ContextPart): # push bytecodes def pushReceiverVariableBytecode(self, interp): index = self.currentBytecode & 15 - self.push(self.receiver.fetch(index)) + self.push(self.w_receiver.fetch(index)) def pushTemporaryVariableBytecode(self, interp): index = self.currentBytecode & 15 @@ -65,19 +60,19 @@ def pushLiteralConstantBytecode(self, interp): index = self.currentBytecode & 31 - self.push(self.method.getliteral(index)) + self.push(self.w_method().getliteral(index)) def pushLiteralVariableBytecode(self, interp): # this bytecode assumes that literals[index] is an Association # which is an object with two named vars, and fetches the second # named var (the value). index = self.currentBytecode & 31 - association = self.method.getliteral(index) + association = self.w_method().getliteral(index) self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) def storeAndPopReceiverVariableBytecode(self, interp): index = self.currentBytecode & 7 - self.receiver.store(index, self.pop()) + self.w_receiver.store(index, self.pop()) def storeAndPopTemporaryVariableBytecode(self, interp): index = self.currentBytecode & 7 @@ -85,7 +80,7 @@ # push bytecodes def pushReceiverBytecode(self, interp): - self.push(self.receiver) + self.push(self.w_receiver) def pushConstantTrueBytecode(self, interp): self.push(interp.TRUE) @@ -107,27 +102,27 @@ def pushConstantTwoBytecode(self, interp): self.push(interp.TWO) - + def pushActiveContextBytecode(self, interp): self.push(self) def duplicateTopBytecode(self, interp): self.push(self.top()) - + # send, return bytecodes def sendLiteralSelectorBytecode(self, interp): - selector = self.method.getliteral(self.currentBytecode & 15) + selector = self.w_method().getliteral(self.currentBytecode & 15) argcount = ((self.currentBytecode >> 4) & 3) - 1 self._sendSelfSelector(selector, argcount, interp) - + def _sendSelfSelector(self, selector, argcount, interp): receiver = self.peek(argcount) self._sendSelector(selector, argcount, interp, receiver, receiver.shadow_of_my_class()) def _sendSuperSelector(self, selector, argcount, interp): - s_compiledin = self.method.w_compiledin.as_class_get_shadow() - self._sendSelector(selector, argcount, interp, self.receiver, + s_compiledin = self.w_method().w_compiledin.as_class_get_shadow() + self._sendSelector(selector, argcount, interp, self.w_receiver, s_compiledin.s_superclass) def _sendSelector(self, selector, argcount, interp, @@ -149,13 +144,13 @@ self.pop_n(argcount + 1) def _return(self, object, interp): - if self.sender is None: # for tests, when returning from the top-level context + if self.w_sender is None: # for tests, when returning from the top-level context raise ReturnFromTopLevel(object) - self.sender.push(object) - interp.w_active_context = self.sender + self.w_sender.push(object) + interp.w_active_context = self.w_sender def returnReceiver(self, interp): - self._return(self.receiver, interp) + self._return(self.w_receiver, interp) def returnTrue(self, interp): self._return(interp.TRUE, interp) @@ -169,9 +164,8 @@ def returnTopFromMethod(self, interp): self._return(self.top(), interp) - def returnTopFromBlock(self, interp): - raise MissingBytecode + self._return(self.top(), interp) def unknownBytecode(self, interp): raise MissingBytecode @@ -183,25 +177,25 @@ def extendedPushBytecode(self, interp): variableType, variableIndex = self.extendedVariableTypeAndIndex() if variableType == 0: - self.push(self.receiver.fetch(variableIndex)) + self.push(self.w_receiver.fetch(variableIndex)) elif variableType == 1: self.push(self.gettemp(variableIndex)) elif variableType == 2: - self.push(self.method.getliteral(variableIndex)) + self.push(self.w_method().getliteral(variableIndex)) elif variableType == 3: - association = self.method.getliteral(variableIndex) + association = self.w_method().getliteral(variableIndex) self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) def extendedStoreBytecode(self, interp): variableType, variableIndex = self.extendedVariableTypeAndIndex() if variableType == 0: - self.receiver.store(variableIndex, self.top()) + self.w_receiver.store(variableIndex, self.top()) elif variableType == 1: self.settemp(variableIndex, self.top()) elif variableType == 2: raise IllegalStoreError elif variableType == 3: - association = self.method.getliteral(variableIndex) + association = self.w_method().getliteral(variableIndex) association.store(constants.ASSOCIATION_VALUE_INDEX, self.top()) def extendedStoreAndPopBytecode(self, interp): @@ -210,7 +204,7 @@ def getExtendedSelectorArgcount(self): descriptor = self.getByte() - return (self.method.getliteral(descriptor & 31)), (descriptor >> 5) + return (self.w_method().getliteral(descriptor & 31)), (descriptor >> 5) def singleExtendedSendBytecode(self, interp): selector, argcount = self.getExtendedSelectorArgcount() @@ -222,28 +216,28 @@ opType = second >> 5 if opType == 0: # selfsend - self._sendSelfSelector(self.method.getliteral(third), + self._sendSelfSelector(self.w_method().getliteral(third), second & 31, interp) elif opType == 1: # supersend - self._sendSuperSelector(self.method.getliteral(third), + self._sendSuperSelector(self.w_method().getliteral(third), second & 31, interp) elif opType == 2: # pushReceiver - self.push(self.receiver.fetch(third)) + self.push(self.w_receiver.fetch(third)) elif opType == 3: # pushLiteralConstant - self.push(self.method.getliteral(third)) + self.push(self.w_method().getliteral(third)) elif opType == 4: # pushLiteralVariable - association = self.method.getliteral(third) + association = self.w_method().getliteral(third) self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) elif opType == 5: - self.receiver.store(third, self.top()) + self.w_receiver.store(third, self.top()) elif opType == 6: - self.receiver.store(third, self.pop()) + self.w_receiver.store(third, self.pop()) elif opType == 7: - association = self.method.getliteral(third) + association = self.w_method().getliteral(third) association.store(constants.ASSOCIATION_VALUE_INDEX, self.top()) def singleExtendedSuperBytecode(self, interp): @@ -252,7 +246,7 @@ def secondExtendedSendBytecode(self, interp): descriptor = self.getByte() - selector = self.method.getliteral(descriptor & 63) + selector = self.w_method().getliteral(descriptor & 63) argcount = descriptor >> 6 self._sendSelfSelector(selector, argcount, interp) @@ -398,119 +392,94 @@ self._sendSelfSelector("y", 0, interp) -class Interpreter: - - TRUE = objtable.w_true - FALSE = objtable.w_false - NIL = objtable.w_nil - MONE = objtable.w_mone - ZERO = objtable.w_zero - ONE = objtable.w_one - TWO = objtable.w_two - - def __init__(self): - self.w_active_context = None - - - def interpret(self): - try: - while True: - self.step() - except ReturnFromTopLevel, e: - return e.object - - def step(self): - next = self.w_active_context.getNextBytecode() - bytecodeimpl = BYTECODE_TABLE[next] - bytecodeimpl(self.w_active_context, self) - -class ReturnFromTopLevel(Exception): - def __init__(self, object): - self.object = object - BYTECODE_RANGES = [ - ( 0, 15, W_MethodContext.pushReceiverVariableBytecode), - ( 16, 31, W_MethodContext.pushTemporaryVariableBytecode), - ( 32, 63, W_MethodContext.pushLiteralConstantBytecode), - ( 64, 95, W_MethodContext.pushLiteralVariableBytecode), - ( 96, 103, W_MethodContext.storeAndPopReceiverVariableBytecode), - (104, 111, W_MethodContext.storeAndPopTemporaryVariableBytecode), - (112, W_MethodContext.pushReceiverBytecode), - (113, W_MethodContext.pushConstantTrueBytecode), - (114, W_MethodContext.pushConstantFalseBytecode), - (115, W_MethodContext.pushConstantNilBytecode), - (116, W_MethodContext.pushConstantMinusOneBytecode), - (117, W_MethodContext.pushConstantZeroBytecode), - (118, W_MethodContext.pushConstantOneBytecode), - (119, W_MethodContext.pushConstantTwoBytecode), - (120, W_MethodContext.returnReceiver), - (121, W_MethodContext.returnTrue), - (122, W_MethodContext.returnFalse), - (123, W_MethodContext.returnNil), - (124, W_MethodContext.returnTopFromMethod), - (125, W_MethodContext.returnTopFromBlock), - (126, W_MethodContext.unknownBytecode), - (127, W_MethodContext.unknownBytecode), - (128, W_MethodContext.extendedPushBytecode), - (129, W_MethodContext.extendedStoreBytecode), - (130, W_MethodContext.extendedStoreAndPopBytecode), - (131, W_MethodContext.singleExtendedSendBytecode), - (132, W_MethodContext.doubleExtendedDoAnythingBytecode), - (133, W_MethodContext.singleExtendedSuperBytecode), - (134, W_MethodContext.secondExtendedSendBytecode), - (135, W_MethodContext.popStackBytecode), - (136, W_MethodContext.duplicateTopBytecode), - (137, W_MethodContext.pushActiveContextBytecode), - (138, 143, W_MethodContext.experimentalBytecode), - (144, 151, W_MethodContext.shortUnconditionalJump), - (152, 159, W_MethodContext.shortConditionalJump), - (160, 167, W_MethodContext.longUnconditionalJump), - (168, 171, W_MethodContext.longJumpIfTrue), - (172, 175, W_MethodContext.longJumpIfFalse), - (176, W_MethodContext.bytecodePrimAdd), - (177, W_MethodContext.bytecodePrimSubtract), - (178, W_MethodContext.bytecodePrimLessThan), - (179, W_MethodContext.bytecodePrimGreaterThan), - (180, W_MethodContext.bytecodePrimLessOrEqual), - (181, W_MethodContext.bytecodePrimGreaterOrEqual), - (182, W_MethodContext.bytecodePrimEqual), - (183, W_MethodContext.bytecodePrimNotEqual), - (184, W_MethodContext.bytecodePrimMultiply), - (185, W_MethodContext.bytecodePrimDivide), - (186, W_MethodContext.bytecodePrimMod), - (187, W_MethodContext.bytecodePrimMakePoint), - (188, W_MethodContext.bytecodePrimBitShift), - (189, W_MethodContext.bytecodePrimDiv), - (190, W_MethodContext.bytecodePrimBitAnd), - (191, W_MethodContext.bytecodePrimBitOr), - (192, W_MethodContext.bytecodePrimAt), - (193, W_MethodContext.bytecodePrimAtPut), - (194, W_MethodContext.bytecodePrimSize), - (195, W_MethodContext.bytecodePrimNext), - (196, W_MethodContext.bytecodePrimNextPut), - (197, W_MethodContext.bytecodePrimAtEnd), - (198, W_MethodContext.bytecodePrimEquivalent), - (199, W_MethodContext.bytecodePrimClass), - (200, W_MethodContext.bytecodePrimBlockCopy), - (201, W_MethodContext.bytecodePrimValue), - (202, W_MethodContext.bytecodePrimValueWithArg), - (203, W_MethodContext.bytecodePrimDo), - (204, W_MethodContext.bytecodePrimNew), - (205, W_MethodContext.bytecodePrimNewWithArg), - (206, W_MethodContext.bytecodePrimPointX), - (207, W_MethodContext.bytecodePrimPointY), - (208, 255, W_MethodContext.sendLiteralSelectorBytecode), + ( 0, 15, W_ContextPart.pushReceiverVariableBytecode), + ( 16, 31, W_ContextPart.pushTemporaryVariableBytecode), + ( 32, 63, W_ContextPart.pushLiteralConstantBytecode), + ( 64, 95, W_ContextPart.pushLiteralVariableBytecode), + ( 96, 103, W_ContextPart.storeAndPopReceiverVariableBytecode), + (104, 111, W_ContextPart.storeAndPopTemporaryVariableBytecode), + (112, W_ContextPart.pushReceiverBytecode), + (113, W_ContextPart.pushConstantTrueBytecode), + (114, W_ContextPart.pushConstantFalseBytecode), + (115, W_ContextPart.pushConstantNilBytecode), + (116, W_ContextPart.pushConstantMinusOneBytecode), + (117, W_ContextPart.pushConstantZeroBytecode), + (118, W_ContextPart.pushConstantOneBytecode), + (119, W_ContextPart.pushConstantTwoBytecode), + (120, W_ContextPart.returnReceiver), + (121, W_ContextPart.returnTrue), + (122, W_ContextPart.returnFalse), + (123, W_ContextPart.returnNil), + (124, W_ContextPart.returnTopFromMethod), + (125, W_ContextPart.returnTopFromBlock), + (126, W_ContextPart.unknownBytecode), + (127, W_ContextPart.unknownBytecode), + (128, W_ContextPart.extendedPushBytecode), + (129, W_ContextPart.extendedStoreBytecode), + (130, W_ContextPart.extendedStoreAndPopBytecode), + (131, W_ContextPart.singleExtendedSendBytecode), + (132, W_ContextPart.doubleExtendedDoAnythingBytecode), + (133, W_ContextPart.singleExtendedSuperBytecode), + (134, W_ContextPart.secondExtendedSendBytecode), + (135, W_ContextPart.popStackBytecode), + (136, W_ContextPart.duplicateTopBytecode), + (137, W_ContextPart.pushActiveContextBytecode), + (138, 143, W_ContextPart.experimentalBytecode), + (144, 151, W_ContextPart.shortUnconditionalJump), + (152, 159, W_ContextPart.shortConditionalJump), + (160, 167, W_ContextPart.longUnconditionalJump), + (168, 171, W_ContextPart.longJumpIfTrue), + (172, 175, W_ContextPart.longJumpIfFalse), + (176, W_ContextPart.bytecodePrimAdd), + (177, W_ContextPart.bytecodePrimSubtract), + (178, W_ContextPart.bytecodePrimLessThan), + (179, W_ContextPart.bytecodePrimGreaterThan), + (180, W_ContextPart.bytecodePrimLessOrEqual), + (181, W_ContextPart.bytecodePrimGreaterOrEqual), + (182, W_ContextPart.bytecodePrimEqual), + (183, W_ContextPart.bytecodePrimNotEqual), + (184, W_ContextPart.bytecodePrimMultiply), + (185, W_ContextPart.bytecodePrimDivide), + (186, W_ContextPart.bytecodePrimMod), + (187, W_ContextPart.bytecodePrimMakePoint), + (188, W_ContextPart.bytecodePrimBitShift), + (189, W_ContextPart.bytecodePrimDiv), + (190, W_ContextPart.bytecodePrimBitAnd), + (191, W_ContextPart.bytecodePrimBitOr), + (192, W_ContextPart.bytecodePrimAt), + (193, W_ContextPart.bytecodePrimAtPut), + (194, W_ContextPart.bytecodePrimSize), + (195, W_ContextPart.bytecodePrimNext), + (196, W_ContextPart.bytecodePrimNextPut), + (197, W_ContextPart.bytecodePrimAtEnd), + (198, W_ContextPart.bytecodePrimEquivalent), + (199, W_ContextPart.bytecodePrimClass), + (200, W_ContextPart.bytecodePrimBlockCopy), + (201, W_ContextPart.bytecodePrimValue), + (202, W_ContextPart.bytecodePrimValueWithArg), + (203, W_ContextPart.bytecodePrimDo), + (204, W_ContextPart.bytecodePrimNew), + (205, W_ContextPart.bytecodePrimNewWithArg), + (206, W_ContextPart.bytecodePrimPointX), + (207, W_ContextPart.bytecodePrimPointY), + (208, 255, W_ContextPart.sendLiteralSelectorBytecode), ] def initialize_bytecode_table(): result = [None] * 256 for entry in BYTECODE_RANGES: + def dump_func(f): + def wrapped(*args): + print "Bytecode: %s" % (f.__name__) + return f(*args) + return wrapped if len(entry) == 2: positions = [entry[0]] else: positions = range(entry[0], entry[1]+1) for pos in positions: - result[pos] = entry[-1] + result[pos] = dump_func(entry[-1]) assert None not in result return result Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 19:57:52 2007 @@ -2,6 +2,7 @@ from pypy.rlib import rrandom from pypy.rlib.rarithmetic import intmask from pypy.lang.smalltalk import constants +from pypy.tool.pairtype import extendabletype class W_Object(object): @@ -36,6 +37,11 @@ def __repr__(self): return "W_SmallInteger(%d)" % self.value + +def unwrap_int(w_value): + if isinstance(w_value, W_SmallInteger): + return w_value.value + raise ClassShadowError("expected a W_SmallInteger, got %s" % (w_value,)) class W_Float(W_Object): def __init__(self, value): @@ -216,7 +222,6 @@ return self.literals[index + 1] # header of compiledmethod at index 0 def createFrame(self, receiver, arguments, sender = None): - from pypy.lang.smalltalk.interpreter import W_MethodContext assert len(arguments) == self.argsize return W_MethodContext(self, receiver, arguments, sender) @@ -238,3 +243,155 @@ self.tempsize is not None and hasattr(self, 'primitive') and self.primitive is not None) + +class W_ContextPart(W_AbstractObjectWithIdentityHash): + + __metaclass__ = extendabletype + + def __init__(self, w_home, w_sender): + self.stack = [] + self.pc = 0 + assert isinstance(w_home, W_MethodContext) + self.w_home = w_home + self.w_sender = w_sender + + def getclass(self): + from pypy.lang.smalltalk.classtable import w_ContextPart + return w_ContextPart + + # ______________________________________________________________________ + # Imitate the primitive accessors + + def fetch(self, index): + if index == CTXPART_SENDER_INDEX: + return self.w_sender + elif index == CTXPART_PC_INDEX: + return objtable.wrap_int(self.pc) + elif index == CTXPART_STACKP_INDEX: + return objtable.wrap_int(len(self.stack)) + + # Invalid! + raise IllegalFetchError + + # ______________________________________________________________________ + # Method that contains the bytecode for this method/block context + + def w_method(self): + return self.w_home._w_method + + def getByte(self): + bytecode = self.w_method().bytes[self.pc] + currentBytecode = ord(bytecode) + self.pc = self.pc + 1 + return currentBytecode + + def getNextBytecode(self): + self.currentBytecode = self.getByte() + return self.currentBytecode + + # ______________________________________________________________________ + # Temporary Variables + # + # Are always fetched relative to the home method context. + + def gettemp(self, index): + return self.w_home.temps[index] + + def settemp(self, index, w_value): + self.w_home.temps[index] = w_value + + # ______________________________________________________________________ + # Stack Manipulation + + def pop(self): + return self.stack.pop() + + def push(self, w_v): + self.stack.append(w_v) + + def push_all(self, lst): + " Equivalent to 'for x in lst: self.push(x)' where x is a lst " + self.stack += lst + + def top(self): + return self.peek(0) + + def peek(self, idx): + return self.stack[-(idx+1)] + + def pop_n(self, n): + res = self.stack[len(self.stack)-n:] + self.stack = self.stack[:len(self.stack)-n] + return res + +class W_BlockContext(W_ContextPart): + + def __init__(self, w_home, w_sender, argcnt, initialip): + W_ContextPart.__init__(self, w_home, w_sender) + self.argcnt = argcnt + self.initialip = initialip + + def expected_argument_count(self): + return self.argcnt + + def getclass(self): + from pypy.lang.smalltalk.classtable import w_BlockContext + return w_BlockContext + + def fetch(self, index): + if index == BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX: + return objtable.wrap_int(self.argcnt) + elif index == BLKCTX_INITIAL_IP_INDEX: + return objtable.wrap_int(self.initialip) + elif index == BLKCTX_HOME_INDEX: + return self.w_home + elif index >= BLKCTX_TEMP_FRAME_START: + stack_index = len(self.stack) - index - 1 + return self.stack[stack_index] + else: + return W_ContextPart.fetch(index) + + def store(self, index, value): + if index == BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX: + self.argcnt = unwrap_int(self.argcnt) + elif index == BLKCTX_INITIAL_IP_INDEX: + self.pc = unwrap_int(self.argcnt) + elif index == BLKCTX_HOME_INDEX: + self.w_home = value + elif index >= BLKCTX_TEMP_FRAME_START: + stack_index = len(self.stack) - index - 1 + self.stack[stack_index] = value + else: + return W_ContextPart.fetch(index) + +class W_MethodContext(W_ContextPart): + def __init__(self, w_method, w_receiver, arguments, w_sender = None): + W_ContextPart.__init__(self, self, w_sender) + self._w_method = w_method + self.w_receiver = w_receiver + self.temps = arguments + [None] * w_method.tempsize + + def getclass(self): + from pypy.lang.smalltalk.classtable import w_MethodContext + return w_MethodContext + + def fetch(self, index): + if index == MTHDCTX_METHOD: + return self.w_method() + elif index == MTHDCTX_RECEIVER_MAP: # what is this thing? + return objtable.w_nil + elif index == MTHDCTX_RECEIVER: + return self.w_receiver + elif index >= MTHDCTX_TEMP_FRAME_START: + # First set of indices are temporary variables: + offset = index - MTHDCTX_TEMP_FRAME_START + if offset < len(self.temps): + return self.temps[offset] + + # After that comes the stack: + offset -= len(self.temps) + stack_index = len(self.stack) - offset - 1 + return self.stack[stack_index] + else: + return W_ContextPart.fetch(index) + Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 19:57:52 2007 @@ -5,15 +5,6 @@ from pypy.lang.smalltalk import objtable from pypy.rlib import rarithmetic -from pypy.lang.smalltalk.constants import \ - BLKCTX_CALLER_INDEX, \ - BLKCTX_INSTRUCTION_POINTER_INDEX, \ - BLKCTX_STACK_POINTER_INDEX, \ - BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX, \ - BLKCTX_INITIAL_IP_INDEX, \ - BLKCTX_HOME_INDEX, \ - BLKCTX_TEMP_FRAME_START - class PrimitiveFailedError(Exception): pass @@ -69,6 +60,8 @@ def decorator(wrapped): def result(args): frame = args.interp.w_active_context + if len(frame.stack) < n: + raise PrimitiveFailedError() items = frame.stack[len(frame.stack)-n:] res = wrapped(args, items) frame.pop_n(n) # only if no exception occurs! @@ -623,58 +616,51 @@ @primitive(PRIMITIVE_BLOCK_COPY) @stack(2) def func(args, (w_context, w_argcnt)): - raise PrimitiveNotYetWrittenError() frame = args.interp.w_active_context + argcnt = unwrap_int(w_argcnt) # From B.B.: If receiver is a MethodContext, then it becomes # the new BlockContext's home context. Otherwise, the home # context of the receiver is used for the new BlockContext. - if w_context.getclass() == classtable.w_BlockContext: - w_method_context = w_context.fetch(BLKCTX_HOME_INDEX) - else: - w_method_context = w_context + # Note that in our impl, MethodContext.w_home == self + if not isinstance(w_context, model.W_ContextPart): + raise PrimitiveFailedError() + w_method_context = w_context.w_home # The block bytecodes are stored inline: so we skip past the - # byteodes to invoke this primitive to find them (hence +3) - w_new_context = classtable.w_BlockContext.as_class_get_shadow().new( - unwrap_int(w_argcnt)) - initialip = frame.pc + 3 - - # Initialize various fields. - w_new_context.store(BLKCTX_INITIAL_IP_INDEX, initialip) - w_new_context.store(BLKCTX_INSTRUCTION_POINTER_INDEX, initialip) - w_new_context.store(BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX, w_argcnt) - w_new_context.store(BLKCTX_HOME_INDEX, w_method_context) + # byteodes to invoke this primitive to find them (hence +2) + initialip = frame.pc + 2 + w_new_context = model.W_BlockContext( + w_method_context, objtable.w_nil, argcnt, initialip) return w_new_context @primitive(PRIMITIVE_VALUE) def func(args): - raise PrimitiveNotYetWrittenError() # If nargs == 4, stack looks like: # 3 2 1 0 # Rcvr | Arg 0 | Arg1 | Arg 2 # - w_block_ctx = args.interp.w_active_context.peek(args.argument_count-1) - - w_exp_arg_cnt = w_block_ctx.fetch(BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX) - exp_arg_cnt = unwrap_int(w_exp_arg_cnt) + frame = args.interp.w_active_context + + # Validate that we have a block on the stack and that it received + # the proper number of arguments: + w_block_ctx = frame.peek(args.argument_count-1) + if not isinstance(w_block_ctx, model.W_BlockContext): + raise PrimitiveFailedError() + exp_arg_cnt = w_block_ctx.expected_argument_count() if args.argument_count != exp_arg_cnt: raise PrimitiveFailedError() - # Copy the values from the stack such that the most recently pushed - # item (index 0) ends up in slot BLKCTX_TEMP_FRAME_START + nargs - 1 - for idx in range(exp_arg_cnt - 1): - w_block_ctx.store( - BLKCTX_TEMP_FRAME_START+idx, - w_block_ctx.fetch(exp_arg_cnt - idx - 1)) + # Initialize the block stack from the contents of the stack: + # Don't bother to copy the 'self' argument + block_args = frame.pop_n(exp_arg_cnt - 1) + w_block_ctx.push_all(block_args) # Set some fields - w_initial_ip = w_block_ctx.fetch(BLKCTX_INITIAL_IP_INDEX) - w_block_ctx.store(BLKCTX_INSTRUCTION_POINTER_INDEX, w_initial_ip) - w_block_ctx.store(BLKCTX_STACK_POINTER_INDEX, w_exp_arg_cnt) - w_block_ctx.store(BLKCTX_CALLER_INDEX, args.interp.w_active_context) + w_block_ctx.pc = w_block_ctx.initialip + w_block_ctx.w_sender = frame args.interp.w_active_context = w_block_ctx @primitive(PRIMITIVE_VALUE_WITH_ARGS) Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Thu Oct 25 19:57:52 2007 @@ -17,18 +17,14 @@ WEAK_POINTERS = 3 COMPILED_METHOD = 4 +unwrap_int = model.unwrap_int + class MethodNotFound(Exception): pass class ClassShadowError(Exception): pass -def unwrap_int(w_value): - if isinstance(w_value, model.W_SmallInteger): - return w_value.value - raise ClassShadowError("expected a W_SmallInteger, got %s" % (w_value,)) - - class ClassShadow(AbstractShadow): """A shadow for Smalltalk objects that are classes (i.e. used as the class of another Smalltalk object). @@ -112,7 +108,14 @@ self.s_superclass = w_superclass.as_class_get_shadow() def new(self, extrasize=0): + from pypy.lang.smalltalk import classtable w_cls = self.w_self + + if w_cls == classtable.w_BlockContext: + return model.W_BlockContext(None, None, 0, 0) + elif w_cls == classtable.w_MethodContext: + return model.W_MethodContext(None, None, []) + if self.instance_kind == POINTERS: return model.W_PointersObject(w_cls, self.instance_size+extrasize) elif self.instance_kind == WORDS: Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Thu Oct 25 19:57:52 2007 @@ -42,7 +42,7 @@ w_method = model.W_CompiledMethod(0, bytes="hello", argsize=2, tempsize=1) w_frame = w_method.createFrame("receiver", ["foo", "bar"]) - assert w_frame.receiver == "receiver" + assert w_frame.w_receiver == "receiver" assert w_frame.gettemp(0) == "foo" assert w_frame.gettemp(1) == "bar" assert w_frame.gettemp(2) == None @@ -75,7 +75,7 @@ def test_pushReceiverBytecode(): interp = new_interpreter(pushReceiverBytecode) interp.step() - assert interp.w_active_context.top() == interp.w_active_context.receiver + assert interp.w_active_context.top() == interp.w_active_context.w_receiver def test_pushReceiverVariableBytecode(bytecode = (pushReceiverVariableBytecode(0) + pushReceiverVariableBytecode(1) + @@ -104,7 +104,7 @@ pushLiteralConstantBytecode(1) + pushLiteralConstantBytecode(2)): interp = new_interpreter(bytecode) - interp.w_active_context.method.literals = fakeliterals("a", "b", "c") + interp.w_active_context.w_method().literals = fakeliterals("a", "b", "c") interp.step() interp.step() interp.step() @@ -115,7 +115,7 @@ w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(bytecode) - interp.w_active_context.method.literals = fakeliterals(w_association) + interp.w_active_context.w_method().literals = fakeliterals(w_association) interp.step() assert interp.w_active_context.stack == ["myvalue"] @@ -125,7 +125,7 @@ for index in range(8): w_object = shadow.new() interp = new_interpreter(pushConstantTrueBytecode + bytecode(index)) - interp.w_active_context.receiver = w_object + interp.w_active_context.w_receiver = w_object interp.step() interp.step() if popped: @@ -213,14 +213,14 @@ shadow.installmethod("foo", model.W_CompiledMethod(0, pushConstantOneBytecode + bytecode)) interp = new_interpreter(bytecodes) - interp.w_active_context.method.literals = fakeliterals("foo") + interp.w_active_context.w_method().literals = fakeliterals("foo") interp.w_active_context.push(w_object) callerContext = interp.w_active_context interp.step() - assert interp.w_active_context.sender == callerContext + assert interp.w_active_context.w_sender == callerContext assert interp.w_active_context.stack == [] - assert interp.w_active_context.receiver == w_object - assert interp.w_active_context.method == shadow.methoddict["foo"] + assert interp.w_active_context.w_receiver == w_object + assert interp.w_active_context.w_method() == shadow.methoddict["foo"] assert callerContext.stack == [] interp.step() interp.step() @@ -240,7 +240,7 @@ shadow.installmethod("fib:", method) w_object = shadow.new() interp = new_interpreter(sendLiteralSelectorBytecode(16) + returnTopFromMethod) - interp.w_active_context.method.literals = fakeliterals("fib:") + interp.w_active_context.w_method().literals = fakeliterals("fib:") interp.w_active_context.push(w_object) interp.w_active_context.push(wrap_int(8)) result = interp.interpret() @@ -253,7 +253,7 @@ s_smallintclass.installmethod("sub", prim_meth) try: interp = new_interpreter(sendLiteralSelectorBytecode(1 + 16)) - interp.w_active_context.method.literals = fakeliterals("foo", "sub") + interp.w_active_context.w_method().literals = fakeliterals("foo", "sub") interp.w_active_context.push(wrap_int(50)) interp.w_active_context.push(wrap_int(8)) callerContext = interp.w_active_context @@ -340,7 +340,7 @@ w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(pushConstantOneBytecode + bytecode) - interp.w_active_context.method.literals = fakeliterals(w_association) + interp.w_active_context.w_method().literals = fakeliterals(w_association) interp.step() interp.step() assert w_association.fetch(1) == interp.ONE @@ -364,8 +364,8 @@ interp.w_active_context.push(w_object) interp.w_active_context.push(interp.ONE) interp.step() - assert interp.w_active_context.method == shadow.methoddict["+"] - assert interp.w_active_context.receiver is w_object + assert interp.w_active_context.w_method() == shadow.methoddict["+"] + assert interp.w_active_context.w_receiver is w_object assert interp.w_active_context.gettemp(0) == interp.ONE assert interp.w_active_context.stack == [] @@ -407,16 +407,16 @@ meth1.literals = fakeliterals("foo") meth2.literals = fakeliterals("foo") interp = new_interpreter(bytecodes) - interp.w_active_context.method.literals = fakeliterals("foo") + interp.w_active_context.w_method().literals = fakeliterals("foo") interp.w_active_context.push(w_object) for w_specificclass in [w_class, w_super, w_supersuper]: callerContext = interp.w_active_context interp.step() - assert interp.w_active_context.sender == callerContext + assert interp.w_active_context.w_sender == callerContext assert interp.w_active_context.stack == [] - assert interp.w_active_context.receiver == w_object + assert interp.w_active_context.w_receiver == w_object meth = w_specificclass.as_class_get_shadow().methoddict["foo"] - assert interp.w_active_context.method == meth + assert interp.w_active_context.w_method() == meth assert callerContext.stack == [] def test_secondExtendedSendBytecode(): @@ -450,15 +450,16 @@ def test_block_copy_and_value(): - py.test.skip("block_copy_and_value not working yet") - bc_3_plus_4 = [ 137, 117, 200, 164, 4, 32, 33, 176, 125, 201, 124] - bc_x_plus_x_plus_1 = [ 137, 118, 200, 164, 7, 104, 16, 16, 176, 118, 176, 125, 32, 202, 124 ] - bc_x_plus_y = [ 137, 119, 200, 164, 6, 105, 104, 16, 17, 176, 125, 33, 34, 240, 124 ] + #bc_x_plus_x_plus_1 = [ 137, 118, 200, 164, 7, 104, 16, 16, 176, 118, 176, 125, 32, 202, 124 ] + #bc_x_plus_y = [ 137, 119, 200, 164, 6, 105, 104, 16, 17, 176, 125, 33, 34, 240, 124 ] - for bcodes in [ bc_3_plus_4, bc_x_plus_x_plus_1, bc_x_plus_y ]: + for bcodes in [ bc_3_plus_4 ]: #, bc_x_plus_x_plus_1, bc_x_plus_y ]: bcode = "".join([chr(x) for x in bcodes]) interp = new_interpreter(bcode) + interp.w_active_context.w_method().literals = \ + fakeliterals(wrap_int(3), + wrap_int(4)) res = interp.interpret() - assert res == wrap_int(7) + assert res.value == 7 Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Oct 25 19:57:52 2007 @@ -12,7 +12,7 @@ mockclass = classtable.bootstrap_class -class MockFrame(interpreter.W_MethodContext): +class MockFrame(model.W_MethodContext): def __init__(self, stack): self.stack = stack From jacob at codespeak.net Thu Oct 25 20:00:02 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Thu, 25 Oct 2007 20:00:02 +0200 (CEST) Subject: [pypy-svn] r47977 - in pypy/branch/dist-future-fixing/pypy/interpreter/pyparser: . test Message-ID: <20071025180002.CE5D38147@code0.codespeak.net> Author: jacob Date: Thu Oct 25 20:00:02 2007 New Revision: 47977 Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfgrammar.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pytoken.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_lookahead.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_parser.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/tuplebuilder.py Log: Refactored the parser bits to reduce the dependency mesh. This is only half way done, but it should work in its present state. Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py Thu Oct 25 20:00:02 2007 @@ -1099,10 +1099,11 @@ self.d = len(rule_stack) class AstBuilder(BaseGrammarBuilder): - """A builder that directly produce the AST""" + """A builder that directly produces the AST""" def __init__(self, parser, grammar_version, debug=0, space=None): - BaseGrammarBuilder.__init__(self, parser, debug) + BaseGrammarBuilder.__init__(self, debug) + self.parser = parser self.rule_stack = [] self.space = space self.source_encoding = None Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfgrammar.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfgrammar.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfgrammar.py Thu Oct 25 20:00:02 2007 @@ -27,7 +27,7 @@ group: '(' alternative ')' star? """ p = GRAMMAR_GRAMMAR - p.add_token('EOF','EOF') + p.add_token(Token('EOF','EOF')) # star: '*' | '+' star = p.Alternative_n( "star", [p.Token_n('TOK_STAR', '*'), p.Token_n('TOK_ADD', '+')] ) Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py Thu Oct 25 20:00:02 2007 @@ -1,4 +1,4 @@ -from grammar import Token, GrammarProxy +from grammar import Token, EmptyToken, GrammarProxy from grammar import AbstractBuilder, AbstractContext @@ -72,7 +72,7 @@ """ if not isinstance(other, Token): raise RuntimeError("Unexpected token type") - if other is self.parser.EmptyToken: + if other is EmptyToken: return False if other.codename != self.codename: return False @@ -93,7 +93,7 @@ class EBNFBuilder(AbstractBuilder): """Build a grammar tree""" def __init__(self, gram_parser, dest_parser): - AbstractBuilder.__init__(self, dest_parser) + AbstractBuilder.__init__(self) self.gram = gram_parser self.rule_stack = [] self.seqcounts = [] # number of items in the current sequence @@ -105,10 +105,13 @@ self.current_rule_name = "" self.tokens = {} self.keywords = [] - NAME = dest_parser.add_token('NAME') + NAME = dest_parser.add_token(Token(dest_parser, 'NAME')) # NAME = dest_parser.tokens['NAME'] self.tokens[NAME] = NameToken(dest_parser, keywords=self.keywords) + # XXX Temporary. We should be able to get rid of it later + self.parser = dest_parser + def context(self): return EBNFBuilderContext(len(self.rule_stack), self.seqcounts, self.altcounts) Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py Thu Oct 25 20:00:02 2007 @@ -7,13 +7,13 @@ KleeneStar : as in S -> A* or S -> A+ Token : a lexer token """ -try: - from pypy.interpreter.baseobjspace import Wrappable - from pypy.interpreter.pyparser.pytoken import NULLTOKEN -except ImportError: - # allows standalone testing - Wrappable = object - NULLTOKEN = -1 # None + +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.pyparser.pytoken import NULLTOKEN +#except ImportError: +# # allows standalone testing +# Wrappable = object +# NULLTOKEN = -1 # None from syntaxtree import SyntaxNode, TempSyntaxNode, TokenNode @@ -105,13 +105,12 @@ class AbstractBuilder(Wrappable): """Abstract base class for builder objects""" - def __init__(self, parser, debug=0 ): + def __init__(self, debug=0 ): # This attribute is here for convenience self.debug = debug # the parser that represent the grammar used # Commented the assert: this eases the testing #assert isinstance( parser, Parser ) - self.parser = parser def context(self): """Return an opaque context object""" @@ -145,8 +144,8 @@ """Base/default class for a builder""" # XXX (adim): this is trunk's keyword management keywords = None - def __init__(self, parser, debug=0 ): - AbstractBuilder.__init__(self, parser, debug ) + def __init__(self, debug=0 ): + AbstractBuilder.__init__(self, debug ) # stacks contain different objects depending on the builder class # to be RPython they should not be defined in the base class self.stack = [] @@ -209,10 +208,8 @@ symbols = {} # dirty trick to provide a symbols mapping while printing (and not putting it in every object) - def __init__(self, parser, codename): + def __init__(self, codename): # the rule name - assert isinstance(parser, Parser) - self.parser = parser # integer mapping to either a token value or rule symbol value self.codename = codename self.args = [] @@ -249,7 +246,7 @@ pos1 = source.get_pos() in_first_set = self.match_first_set(builder, token) if not in_first_set: # and not EmptyToken in self.first_set: - if self.parser.EmptyToken in self.first_set: + if EmptyToken in self.first_set: ret = builder.sequence(self, source, 0 ) if self._trace: self._debug_display(token, level, 'eee' ) @@ -364,7 +361,8 @@ class GrammarProxy(GrammarElement): def __init__(self, parser, rule_name, codename=-1 ): - GrammarElement.__init__(self, parser, codename ) + GrammarElement.__init__(self, codename ) + self.parser = parser self.rule_name = rule_name self.object = None @@ -372,10 +370,10 @@ """Helper function used to represent the grammar. mostly used for debugging the grammar itself""" name = self.parser.symbol_repr(self.codename) - repr = "Proxy("+name + repr = 'Proxy(' + name if self.object: - repr+=","+self.object.display(1) - repr += ")" + repr += ',' + self.object.display(1) + repr += ')' return repr @@ -383,7 +381,8 @@ class Alternative(GrammarElement): """Represents an alternative in a grammar rule (as in S -> A | B | C)""" def __init__(self, parser, name, args): - GrammarElement.__init__(self, parser, name ) + GrammarElement.__init__(self, name ) + self.parser = parser self.args = args self._reordered = False for i in self.args: @@ -401,7 +400,7 @@ # to see if this solve our problems with infinite recursion for rule in self.args: if USE_LOOKAHEAD: - if not rule.match_first_set(builder, tok) and self.parser.EmptyToken not in rule.first_set: + if not rule.match_first_set(builder, tok) and EmptyToken not in rule.first_set: if self._trace: print "Skipping impossible rule: %s" % (rule,) continue @@ -443,7 +442,7 @@ # is only needed for warning / debugging purposes tokens_set = [] for rule in self.args: - if self.parser.EmptyToken in rule.first_set: + if EmptyToken in rule.first_set: empty_set.append(rule) else: not_empty_set.append(rule) @@ -452,7 +451,7 @@ # It will check if a token is part of several first sets of # a same alternative for token in rule.first_set: - if token is not self.parser.EmptyToken and token in tokens_set: + if token is not EmptyToken and token in tokens_set: print "Warning, token %s in\n\t%s's first set is " \ " part of a previous rule's first set in " \ " alternative\n\t%s" % (token, rule, self) @@ -485,7 +484,8 @@ class Sequence(GrammarElement): """Reprensents a Sequence in a grammar rule (as in S -> A B C)""" def __init__(self, parser, name, args): - GrammarElement.__init__(self, parser, name ) + GrammarElement.__init__(self, name ) + self.parser = parser self.args = args for i in self.args: assert isinstance( i, GrammarElement ) @@ -531,16 +531,16 @@ for rule in self.args: if not rule.first_set: break - if self.parser.EmptyToken in self.first_set: - self.first_set.remove( self.parser.EmptyToken ) + if EmptyToken in self.first_set: + self.first_set.remove(EmptyToken) - # del self.first_set[self.parser.EmptyToken] + # del self.first_set[EmptyToken] # while we're in this loop, keep agregating possible tokens for t in rule.first_set: if t not in self.first_set: self.first_set.append(t) # self.first_set[t] = 1 - if self.parser.EmptyToken not in rule.first_set: + if EmptyToken not in rule.first_set: break def validate( self, syntax_node ): @@ -561,7 +561,8 @@ class KleeneStar(GrammarElement): """Represents a KleeneStar in a grammar rule as in (S -> A+) or (S -> A*)""" def __init__(self, parser, name, _min = 0, _max = -1, rule=None): - GrammarElement.__init__( self, parser, name ) + GrammarElement.__init__( self, name ) + self.parser = parser self.args = [rule] self.min = _min if _max == 0: @@ -569,7 +570,7 @@ self.max = _max self.star = "x" if self.min == 0: - self.first_set.append( self.parser.EmptyToken ) + self.first_set.append(EmptyToken) # self.first_set[self.parser.EmptyToken] = 1 def _match(self, source, builder, level=0): @@ -626,15 +627,15 @@ def calc_first_set(self): """returns the list of possible next tokens if S -> A*: - LAH(S) = Union( LAH(A), self.parser.EmptyToken ) + LAH(S) = Union( LAH(A), EmptyToken ) if S -> A+: LAH(S) = LAH(A) """ rule = self.args[0] self.first_set = rule.first_set[:] # self.first_set = dict(rule.first_set) - if self.min == 0 and self.parser.EmptyToken not in self.first_set: - self.first_set.append(self.parser.EmptyToken) + if self.min == 0 and EmptyToken not in self.first_set: + self.first_set.append(EmptyToken) # self.first_set[self.parser.EmptyToken] = 1 def validate( self, syntax_node ): @@ -655,7 +656,8 @@ class Token(GrammarElement): """Represents a Token in a grammar rule (a lexer token)""" def __init__(self, parser, codename, value=None): - GrammarElement.__init__(self, parser, codename) + GrammarElement.__init__(self, codename) + self.parser = parser self.value = value self.first_set = [self] # self.first_set = {self: 1} @@ -707,9 +709,9 @@ """ if not isinstance(other, Token): raise RuntimeError("Unexpected token type") - if other is self.parser.EmptyToken: + if other is EmptyToken: return False - # XXX (adim): this is trunk's keyword management + # XXX (adim): this is trunk's keyWrappableword management # if (self.value is not None and builder.keywords is not None # and self.value not in builder.keywords): # return False @@ -736,6 +738,7 @@ return False +EmptyToken = Token(None, -1, None) class Parser(object): def __init__(self): @@ -745,7 +748,6 @@ self.sym_name = {} # mapping symbol code -> symbol name self.symbols = {} # mapping symbol name -> symbol code self.tokens = { 'NULLTOKEN' : -1 } - self.EmptyToken = Token( self, -1, None ) self.tok_name = {} self.tok_values = {} self.tok_rvalues = {} @@ -781,20 +783,22 @@ return val return self.symbols[ sym ] - def add_token( self, tok, value = None ): + def add_token(self, token): + name = token.codename + value = token.value # assert isinstance( tok, str ) - if not tok in self.tokens: - val = self._sym_count + if not name in self.tokens: + number = self._sym_count self._sym_count += 1 - self.tokens[tok] = val - self.tok_name[val] = tok + self.tokens[name] = number + self.tok_name[number] = name if value is not None: - self.tok_values[value] = val + self.tok_values[value] = number # XXX : this reverse mapping seemed only to be used # because of pycodegen visitAugAssign - self.tok_rvalues[val] = value - return val - return self.tokens[ tok ] + self.tok_rvalues[number] = value + return number + return self.tokens[name] def load_symbols( self, symbols ): for _value, _name in symbols.items(): @@ -868,7 +872,11 @@ def Token_n(self, name, value = None ): # assert isinstance( name, str) # assert value is None or isinstance( value, str) - name_id = self.add_token(name, value) + + # XXX What is the significance of the name_id? Needs to be found + # out for full refactoring of this code. + t = Token(self, name, value) + name_id = self.add_token(t) return Token(self, name_id, value) # Debugging functions Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py Thu Oct 25 20:00:02 2007 @@ -11,7 +11,8 @@ from pypy.interpreter.pyparser.error import SyntaxError from pypy.interpreter.pyparser.pythonlexer import Source, match_encoding_declaration from pypy.interpreter.astcompiler.consts import CO_FUTURE_WITH_STATEMENT -import pypy.interpreter.pyparser.pysymbol as pysymbol +# XXX seems dead +#import pypy.interpreter.pyparser.pysymbol as pysymbol import pypy.interpreter.pyparser.pytoken as pytoken import pypy.interpreter.pyparser.ebnfparse as ebnfparse from pypy.interpreter.pyparser.ebnflexer import GrammarSource @@ -190,6 +191,7 @@ ## grammar.build_first_sets(ebnfbuilder.all_rules) ## return space.wrap( ebnfbuilder.root_rules ) +# XXX Unused? def grammar_rules( space ): w_rules = space.newdict() parser = make_pyparser(space.config.objspace.pyversion) Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pytoken.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pytoken.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pytoken.py Thu Oct 25 20:00:02 2007 @@ -13,69 +13,66 @@ # tok_rpunct = {} -def setup_tokens( parser ): +def setup_tokens(parser): # global tok_rpunct # For compatibility, this produces the same constant values as Python 2.4. - parser.add_token( 'ENDMARKER' ) - parser.add_token( 'NAME' ) - parser.add_token( 'NUMBER' ) - parser.add_token( 'STRING' ) - parser.add_token( 'NEWLINE' ) - parser.add_token( 'INDENT' ) - parser.add_token( 'DEDENT' ) - parser.add_token( 'LPAR', "(" ) - parser.add_token( 'RPAR', ")" ) - parser.add_token( 'LSQB', "[" ) - parser.add_token( 'RSQB', "]" ) - parser.add_token( 'COLON', ":" ) - parser.add_token( 'COMMA', "," ) - parser.add_token( 'SEMI', ";" ) - parser.add_token( 'PLUS', "+" ) - parser.add_token( 'MINUS', "-" ) - parser.add_token( 'STAR', "*" ) - parser.add_token( 'SLASH', "/" ) - parser.add_token( 'VBAR', "|" ) - parser.add_token( 'AMPER', "&" ) - parser.add_token( 'LESS', "<" ) - parser.add_token( 'GREATER', ">" ) - parser.add_token( 'EQUAL', "=" ) - parser.add_token( 'DOT', "." ) - parser.add_token( 'PERCENT', "%" ) - parser.add_token( 'BACKQUOTE', "`" ) - parser.add_token( 'LBRACE', "{" ) - parser.add_token( 'RBRACE', "}" ) - parser.add_token( 'EQEQUAL', "==" ) - ne = parser.add_token( 'NOTEQUAL', "!=" ) + from grammar import Token + parser.add_token(Token(parser, 'ENDMARKER' )) + parser.add_token(Token(parser, 'NAME' )) + parser.add_token(Token(parser, 'NUMBER' )) + parser.add_token(Token(parser, 'STRING' )) + parser.add_token(Token(parser, 'NEWLINE' )) + parser.add_token(Token(parser, 'INDENT' )) + parser.add_token(Token(parser, 'DEDENT' )) + parser.add_token(Token(parser, 'LPAR', "(" )) + parser.add_token(Token(parser, 'RPAR', ")" )) + parser.add_token(Token(parser, 'LSQB', "[" )) + parser.add_token(Token(parser, 'RSQB', "]" )) + parser.add_token(Token(parser, 'COLON', ":" )) + parser.add_token(Token(parser, 'COMMA', "," )) + parser.add_token(Token(parser, 'SEMI', ";" )) + parser.add_token(Token(parser, 'PLUS', "+" )) + parser.add_token(Token(parser, 'MINUS', "-" )) + parser.add_token(Token(parser, 'STAR', "*" )) + parser.add_token(Token(parser, 'SLASH', "/" )) + parser.add_token(Token(parser, 'VBAR', "|" )) + parser.add_token(Token(parser, 'AMPER', "&" )) + parser.add_token(Token(parser, 'LESS', "<" )) + parser.add_token(Token(parser, 'GREATER', ">" )) + parser.add_token(Token(parser, 'EQUAL', "=" )) + parser.add_token(Token(parser, 'DOT', "." )) + parser.add_token(Token(parser, 'PERCENT', "%" )) + parser.add_token(Token(parser, 'BACKQUOTE', "`" )) + parser.add_token(Token(parser, 'LBRACE', "{" )) + parser.add_token(Token(parser, 'RBRACE', "}" )) + parser.add_token(Token(parser, 'EQEQUAL', "==" )) + ne = parser.add_token(Token(parser, 'NOTEQUAL', "!=" )) parser.tok_values["<>"] = ne - parser.add_token( 'LESSEQUAL', "<=" ) - parser.add_token( 'GREATEREQUAL', ">=" ) - parser.add_token( 'TILDE', "~" ) - parser.add_token( 'CIRCUMFLEX', "^" ) - parser.add_token( 'LEFTSHIFT', "<<" ) - parser.add_token( 'RIGHTSHIFT', ">>" ) - parser.add_token( 'DOUBLESTAR', "**" ) - parser.add_token( 'PLUSEQUAL', "+=" ) - parser.add_token( 'MINEQUAL', "-=" ) - parser.add_token( 'STAREQUAL', "*=" ) - parser.add_token( 'SLASHEQUAL', "/=" ) - parser.add_token( 'PERCENTEQUAL', "%=" ) - parser.add_token( 'AMPEREQUAL', "&=" ) - parser.add_token( 'VBAREQUAL', "|=" ) - parser.add_token( 'CIRCUMFLEXEQUAL', "^=" ) - parser.add_token( 'LEFTSHIFTEQUAL', "<<=" ) - parser.add_token( 'RIGHTSHIFTEQUAL', ">>=" ) - parser.add_token( 'DOUBLESTAREQUAL', "**=" ) - parser.add_token( 'DOUBLESLASH', "//" ) - parser.add_token( 'DOUBLESLASHEQUAL',"//=" ) - parser.add_token( 'AT', "@" ) - parser.add_token( 'OP' ) - parser.add_token( 'ERRORTOKEN' ) + parser.add_token(Token(parser, 'LESSEQUAL', "<=" )) + parser.add_token(Token(parser, 'GREATEREQUAL', ">=" )) + parser.add_token(Token(parser, 'TILDE', "~" )) + parser.add_token(Token(parser, 'CIRCUMFLEX', "^" )) + parser.add_token(Token(parser, 'LEFTSHIFT', "<<" )) + parser.add_token(Token(parser, 'RIGHTSHIFT', ">>" )) + parser.add_token(Token(parser, 'DOUBLESTAR', "**" )) + parser.add_token(Token(parser, 'PLUSEQUAL', "+=" )) + parser.add_token(Token(parser, 'MINEQUAL', "-=" )) + parser.add_token(Token(parser, 'STAREQUAL', "*=" )) + parser.add_token(Token(parser, 'SLASHEQUAL', "/=" )) + parser.add_token(Token(parser, 'PERCENTEQUAL', "%=" )) + parser.add_token(Token(parser, 'AMPEREQUAL', "&=" )) + parser.add_token(Token(parser, 'VBAREQUAL', "|=" )) + parser.add_token(Token(parser, 'CIRCUMFLEXEQUAL', "^=" )) + parser.add_token(Token(parser, 'LEFTSHIFTEQUAL', "<<=" )) + parser.add_token(Token(parser, 'RIGHTSHIFTEQUAL', ">>=" )) + parser.add_token(Token(parser, 'DOUBLESTAREQUAL', "**=" )) + parser.add_token(Token(parser, 'DOUBLESLASH', "//" )) + parser.add_token(Token(parser, 'DOUBLESLASHEQUAL',"//=" )) + parser.add_token(Token(parser, 'AT', "@" )) + parser.add_token(Token(parser, 'OP' )) + parser.add_token(Token(parser, 'ERRORTOKEN' )) # extra PyPy-specific tokens - parser.add_token( "COMMENT" ) - parser.add_token( "NL" ) + parser.add_token(Token(parser, "COMMENT" )) + parser.add_token(Token(parser, "NL" )) - # tok_rpunct = parser.tok_values.copy() - # for _name, _value in parser.tokens.items(): - # globals()[_name] = _value - # setattr(parser, _name, _value) Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_lookahead.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_lookahead.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_lookahead.py Thu Oct 25 20:00:02 2007 @@ -1,7 +1,7 @@ -from pypy.interpreter.pyparser.grammar import Alternative, Sequence, KleeneStar, \ - Token, Parser +from pypy.interpreter.pyparser.grammar import Alternative,\ + Sequence, KleeneStar, Token, EmptyToken, Parser -class TestLookAheadBasics: +class TestLookAheadBasics(object): def setup_method(self, method): self.parser = Parser() @@ -31,7 +31,7 @@ kstar2 = self.parser.KleeneStar_n("k2", 0, 3, tok1) self.parser.build_first_sets() assert kstar1.first_set == [tok1] - assert kstar2.first_set == [tok1, self.parser.EmptyToken] + assert kstar2.first_set == [tok1, EmptyToken] def test_maybe_empty_sequence(self): @@ -43,7 +43,7 @@ k2 = self.parser.KleeneStar_n("k2", 0, 2, tok2) seq = self.parser.Sequence_n( "seq", [k1, k2]) self.parser.build_first_sets() - assert seq.first_set == [tok1, tok2, self.parser.EmptyToken] + assert seq.first_set == [tok1, tok2, EmptyToken] def test_not_empty_sequence(self): @@ -68,7 +68,7 @@ -class TestLookAhead: +class TestLookAhead(object): def setup_method(self, method): p = self.parser = Parser() @@ -86,7 +86,7 @@ p = self.parser LOW = p.tokens['LOW'] CAP = p.tokens['CAP'] - for s in [Token(p, LOW, 'low'), p.EmptyToken, Token(p, CAP, 'cap')]: + for s in [Token(p, LOW, 'low'), EmptyToken, Token(p, CAP, 'cap')]: assert s in self.A.first_set assert s in self.B.first_set assert s in self.C.first_set Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_parser.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_parser.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_parser.py Thu Oct 25 20:00:02 2007 @@ -1,5 +1,5 @@ from pypy.interpreter.pyparser.asthelper import get_atoms -from pypy.interpreter.pyparser.grammar import Parser +from pypy.interpreter.pyparser.grammar import Parser, Token from pypy.interpreter.pyparser import error from fakes import FakeSpace @@ -7,7 +7,7 @@ def test_symbols(): p = Parser() x1 = p.add_symbol('sym') - x2 = p.add_token('tok') + x2 = p.add_token(Token(p, 'tok')) x3 = p.add_anon_symbol(':sym') x4 = p.add_anon_symbol(':sym1') # test basic numbering assumption @@ -20,7 +20,7 @@ assert x3 < 0 y1 = p.add_symbol('sym') assert y1 == x1 - y2 = p.add_token('tok') + y2 = p.add_token(Token(p, 'tok')) assert y2 == x2 y3 = p.add_symbol(':sym') assert y3 == x3 Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/tuplebuilder.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/tuplebuilder.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/tuplebuilder.py Thu Oct 25 20:00:02 2007 @@ -57,13 +57,17 @@ """A builder that directly produce the AST""" def __init__(self, parser, debug=0, lineno=True): - AbstractBuilder.__init__(self, parser, debug) + AbstractBuilder.__init__(self, debug) # This attribute is here for convenience self.source_encoding = None self.lineno = lineno self.stack = [] - self.space_token = ( self.parser.tokens['NEWLINE'], self.parser.tokens['INDENT'], - self.parser.tokens['DEDENT'], self.parser.tokens['ENDMARKER'] ) + self.space_token = (parser.tokens['NEWLINE'], + parser.tokens['INDENT'], + parser.tokens['DEDENT'], + parser.tokens['ENDMARKER'] ) + + self.parser = parser def context(self): """Returns the state of the builder to be restored later""" From akuhn at codespeak.net Thu Oct 25 20:02:19 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Thu, 25 Oct 2007 20:02:19 +0200 (CEST) Subject: [pypy-svn] r47978 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025180219.6BCD98136@code0.codespeak.net> Author: akuhn Date: Thu Oct 25 20:02:18 2007 New Revision: 47978 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: added code for testing the current image (Squeak3.9-final-7067.image), loading works\! Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 20:02:18 2007 @@ -90,8 +90,14 @@ self.fillin_w_objects() def read_header(self): - version = self.stream.next() - if version != 0x1966: raise NotImplementedError #XXX swap here if 0x66190000 + version = self.stream.peek() + if version != 0x1966: + self.stream.swap = True + version = self.stream.peek() + if version != 0x1966: + raise CorrupImageError + version = self.stream.next() + #------ headersize = self.stream.next() self.endofmemory = self.stream.next() # endofmemory = bodysize self.oldbaseaddress = self.stream.next() @@ -103,12 +109,15 @@ self.stream.skipbytes(headersize - (9 * 4)) def read_body(self): + import sys self.stream.reset_count() while self.stream.count < self.endofmemory: chunk, pos = self.read_object() + if len(self.chunklist) % 1000 == 0: sys.stderr.write('#') self.chunklist.append(chunk) self.chunks[pos + self.oldbaseaddress] = chunk self.stream.close() + self.swap = self.stream.swap #save for later del self.stream return self.chunklist # return for testing @@ -328,17 +337,28 @@ w_wordsobject.hash = self.chunk.hash12 # XXX check this def fillin_bytesobject(self, w_bytesobject): - bbytes = [] - for each in self.chunk.data: - bbytes.append(chr((each >> 24) & 0xff)) - bbytes.append(chr((each >> 16) & 0xff)) - bbytes.append(chr((each >> 8) & 0xff)) - bbytes.append(chr((each >> 0) & 0xff)) w_bytesobject.w_class = self.g_class.w_object + w_bytesobject.bytes = self.get_bytes() + w_bytesobject.hash = self.chunk.hash12 # XXX check this + + def get_bytes(self): + bytes = [] + if self.owner.swap: + for each in self.chunk.data: + bytes.append(chr((each >> 0) & 0xff)) + bytes.append(chr((each >> 8) & 0xff)) + bytes.append(chr((each >> 16) & 0xff)) + bytes.append(chr((each >> 24) & 0xff)) + else: + for each in self.chunk.data: + bytes.append(chr((each >> 24) & 0xff)) + bytes.append(chr((each >> 16) & 0xff)) + bytes.append(chr((each >> 8) & 0xff)) + bytes.append(chr((each >> 0) & 0xff)) #strange, for example range(4)[:0] returns [] instead of [0,1,2,3]! #hence what we have to write list[:-odd] as list[:len(list)-odd] instead :( - w_bytesobject.bytes = bbytes[:len(bbytes)-(self.format & 3)] # omit odd bytes - w_bytesobject.hash = self.chunk.hash12 # XXX check this + return bytes[:len(bytes)-(self.format & 3)] # omit odd bytes + def fillin_compiledmethod(self, w_compiledmethod): header = self.chunk.data[0] @@ -353,19 +373,9 @@ _, primitive, literalsize, islarge, tempsize, numargs, highbit = ( splitbits(header, [1,9,8,1,6,4,1])) primitive = primitive + (highbit << 10) ##XXX todo, check this - # -------------------- literals = [self.decode_pointer(pointer).w_object for pointer in self.chunk.data[:literalsize+1]] - # -------------------- - bbytes = [] - for each in self.chunk.data: - bbytes.append(chr((each >> 24) & 0xff)) - bbytes.append(chr((each >> 16) & 0xff)) - bbytes.append(chr((each >> 8) & 0xff)) - bbytes.append(chr((each >> 0) & 0xff)) - #strange, for example range(4)[:0] returns [] instead of [0,1,2,3]! - #hence what we have to write list[:-odd] as list[:len(list)-odd] instead :( - bbytes = bbytes[(literalsize + 1)*4:len(bbytes)-(self.format & 3)] # omit literals & odd bytes + bbytes = self.get_bytes()[(literalsize + 1)*4:] # XXX assert mirrorcache.get_or_build(self.g_class.w_object) is # ct.m_CompiledMethod w_compiledmethod.__init__( @@ -374,7 +384,6 @@ argsize = numargs, tempsize = tempsize, primitive = primitive) - w_compiledmethod.literals = literals From arigo at codespeak.net Thu Oct 25 20:08:03 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 20:08:03 +0200 (CEST) Subject: [pypy-svn] r47979 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025180803.3AE258137@code0.codespeak.net> Author: arigo Date: Thu Oct 25 20:08:02 2007 New Revision: 47979 Modified: pypy/dist/pypy/lang/smalltalk/shadow.py Log: "!=" => "is not". Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Thu Oct 25 20:08:02 2007 @@ -171,7 +171,7 @@ def lookup(self, selector): if selector in self.methoddict: return self.methoddict[selector] - elif self.s_superclass != None: + elif self.s_superclass is not None: return self.s_superclass.lookup(selector) else: raise MethodNotFound From tverwaes at codespeak.net Thu Oct 25 20:22:03 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 20:22:03 +0200 (CEST) Subject: [pypy-svn] r47980 - in pypy/dist/pypy/lang/smalltalk: . tool Message-ID: <20071025182203.3275F813B@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 20:22:02 2007 New Revision: 47980 Modified: pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: adding constants for MESSAGE_CLASS Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Thu Oct 25 20:22:02 2007 @@ -18,6 +18,10 @@ METHODDICT_VALUES_INDEX = 1 METHODDICT_NAMES_INDEX = 2 +MESSAGE_SELECTOR_INDEX = 0 +MESSAGE_ARGUMENTS_INDEX = 1 +MESSAGE_LOOKUP_CLASS_INDEX = 2 + ASSOCIATION_KEY_INDEX = 0 ASSOCIATION_VALUE_INDEX = 1 @@ -36,7 +40,6 @@ MTHDCTX_RECEIVER_MAP = 4 MTHDCTX_RECEIVER = 5 MTHDCTX_TEMP_FRAME_START = 6 - # ----- special objects indices ------- SO_NIL = 0 Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Thu Oct 25 20:22:02 2007 @@ -3,9 +3,11 @@ from pypy.lang.smalltalk import squeakimage from pypy.lang.smalltalk import constants from pypy.lang.smalltalk import model +from pypy.lang.smalltalk import objtable +from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import interpreter -mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') +mini_image = py.magic.autopath().dirpath().dirpath().join('tool/squeak3.9.image') def get_miniimage(): return squeakimage.ImageReader(squeakimage.Stream(mini_image.open())) @@ -22,7 +24,8 @@ image = create_squeakimage() for each in image.objects: if isinstance(each,model.W_BytesObject): - print each.bytes + print each.shadow_of_my_class() + print each.as_string() def testCompiledMethods(): image = create_squeakimage() @@ -47,6 +50,45 @@ except interpreter.ReturnFromTopLevel, e: return e.object +def testDoesNotUnderstand(): + image = create_squeakimage() + amethod = None + + w_doesnot = image.special(constants.SO_DOES_NOT_UNDERSTAND) + w_object = objtable.wrap_int(3) + w_message_class = image.special(constants.SO_MESSAGE_CLASS) + s_message_class = w_message_class.as_class_get_shadow() + + #Build message argument + w_message = s_message_class.new(1) + w_message.store(constants.MESSAGE_SELECTOR_INDEX, objtable.wrap_string("zork")) + w_aarray = classtable.w_Array.as_class_get_shadow().new(0) + w_message.store(constants.MESSAGE_ARGUMENTS_INDEX, w_aarray) + if s_message_class.instsize() > constants.MESSAGE_LOOKUP_CLASS_INDEX: + w_message.store(constants.MESSAGE_LOOKUP_CLASS_INDEX, w_object.getclass()) + + s_class = w_object.shadow_of_my_class() + w_method = s_class.lookup(w_doesnot) + + interp = interpreter.Interpreter() + + # First literal of the abs method is + # a real smalltalk int + w_frame = w_method.createFrame(w_object, [w_message]) + print "WFRAME: %r" % (w_frame,) + interp.w_active_context = w_frame + + print w_method + + while True: + try: + print "Stackbefore: %r" % (interp.w_active_context.stack,) + interp.step() + print "Stackafter: %r" % (interp.w_active_context.stack,) + except interpreter.ReturnFromTopLevel, e: + return e.object + + def testSelector(): image = create_squeakimage() w_doesnot = image.special(constants.SO_DOES_NOT_UNDERSTAND) @@ -61,8 +103,9 @@ print w_doesnot.getclass().fetch(constants.CLASS_METHODDICT_INDEX)._vars[constants.METHODDICT_VALUES_INDEX]._vars def test_do(): - testSelector() - #printStringsInImage() + #testSelector() + printStringsInImage() + #testDoesNotUnderstand() if __name__ == '__main__': test_do() From cfbolz at codespeak.net Thu Oct 25 20:31:17 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 25 Oct 2007 20:31:17 +0200 (CEST) Subject: [pypy-svn] r47981 - in pypy/dist/pypy/lang/smalltalk: . test tool Message-ID: <20071025183117.566EF813B@code0.codespeak.net> Author: cfbolz Date: Thu Oct 25 20:31:17 2007 New Revision: 47981 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: python coding style: createFrame -> create_frame Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 20:31:17 2007 @@ -140,7 +140,7 @@ self.push(w_result) return arguments = self.stack[len(self.stack)-argcount:] - interp.w_active_context = method.createFrame(receiver, arguments, self) + interp.w_active_context = method.create_frame(receiver, arguments, self) self.pop_n(argcount + 1) def _return(self, object, interp): Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 20:31:17 2007 @@ -221,7 +221,7 @@ def getliteral(self, index): return self.literals[index + 1] # header of compiledmethod at index 0 - def createFrame(self, receiver, arguments, sender = None): + def create_frame(self, receiver, arguments, sender = None): assert len(arguments) == self.argsize return W_MethodContext(self, receiver, arguments, sender) Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Thu Oct 25 20:31:17 2007 @@ -33,7 +33,7 @@ assert isinstance(bytes, str) w_method = model.W_CompiledMethod(0, bytes=bytes, argsize=2, tempsize=1) - w_frame = w_method.createFrame(receiver, ["foo", "bar"]) + w_frame = w_method.create_frame(receiver, ["foo", "bar"]) interp = interpreter.Interpreter() interp.w_active_context = w_frame return interp @@ -41,7 +41,7 @@ def test_create_frame(): w_method = model.W_CompiledMethod(0, bytes="hello", argsize=2, tempsize=1) - w_frame = w_method.createFrame("receiver", ["foo", "bar"]) + w_frame = w_method.create_frame("receiver", ["foo", "bar"]) assert w_frame.w_receiver == "receiver" assert w_frame.gettemp(0) == "foo" assert w_frame.gettemp(1) == "bar" Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 20:31:17 2007 @@ -229,7 +229,7 @@ # w_method = s_class.lookup("abs") assert w_method - w_frame = w_method.createFrame(w_object, []) + w_frame = w_method.create_frame(w_object, []) interp.w_active_context = w_frame print w_method Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Thu Oct 25 20:31:17 2007 @@ -38,7 +38,7 @@ amethod = w_smallint_class.lookup("abs") # First literal of the abs method is # a real smalltalk int - w_frame = amethod.createFrame(model.W_SmallInteger(3), []) + w_frame = amethod.create_frame(model.W_SmallInteger(3), []) interp.activeContext = w_frame print amethod @@ -74,7 +74,7 @@ # First literal of the abs method is # a real smalltalk int - w_frame = w_method.createFrame(w_object, [w_message]) + w_frame = w_method.create_frame(w_object, [w_message]) print "WFRAME: %r" % (w_frame,) interp.w_active_context = w_frame From arigo at codespeak.net Thu Oct 25 20:39:10 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 20:39:10 +0200 (CEST) Subject: [pypy-svn] r47982 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025183910.B1B3A8136@code0.codespeak.net> Author: arigo Date: Thu Oct 25 20:39:10 2007 New Revision: 47982 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/shadow.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Log: (arigo, toon around) Following some discussion, revert r47962 and go for methoddicts that contains RPython strings as keys. This makes it easier to look up random strings - the interpreter has no way to find the symbol corresponding to a string in general. This introduces a performance issue that we think we'll fix with a ShadowString. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 20:39:10 2007 @@ -111,7 +111,7 @@ # send, return bytecodes def sendLiteralSelectorBytecode(self, interp): - selector = self.w_method().getliteral(self.currentBytecode & 15) + selector = self.w_method().getliteralsymbol(self.currentBytecode & 15) argcount = ((self.currentBytecode >> 4) & 3) - 1 self._sendSelfSelector(selector, argcount, interp) @@ -204,7 +204,8 @@ def getExtendedSelectorArgcount(self): descriptor = self.getByte() - return (self.w_method().getliteral(descriptor & 31)), (descriptor >> 5) + return ((self.w_method().getliteralsymbol(descriptor & 31)), + (descriptor >> 5)) def singleExtendedSendBytecode(self, interp): selector, argcount = self.getExtendedSelectorArgcount() @@ -216,11 +217,11 @@ opType = second >> 5 if opType == 0: # selfsend - self._sendSelfSelector(self.w_method().getliteral(third), + self._sendSelfSelector(self.w_method().getliteralsymbol(third), second & 31, interp) elif opType == 1: # supersend - self._sendSuperSelector(self.w_method().getliteral(third), + self._sendSuperSelector(self.w_method().getliteralsymbol(third), second & 31, interp) elif opType == 2: # pushReceiver @@ -246,7 +247,7 @@ def secondExtendedSendBytecode(self, interp): descriptor = self.getByte() - selector = self.w_method().getliteral(descriptor & 63) + selector = self.w_method().getliteralsymbol(descriptor & 63) argcount = descriptor >> 6 self._sendSelfSelector(selector, argcount, interp) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 20:39:10 2007 @@ -221,6 +221,11 @@ def getliteral(self, index): return self.literals[index + 1] # header of compiledmethod at index 0 + def getliteralsymbol(self, index): + w_literal = self.getliteral(index) + assert isinstance(w_literal, W_BytesObject) + return w_literal.as_string() # XXX performance issue here + def create_frame(self, receiver, arguments, sender = None): assert len(arguments) == self.argsize return W_MethodContext(self, receiver, arguments, sender) Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Thu Oct 25 20:39:10 2007 @@ -94,8 +94,11 @@ for i in range(size): w_selector = w_methoddict.fetch(constants.METHODDICT_NAMES_INDEX+i) if w_selector is not objtable.w_nil: + if not isinstance(w_selector, model.W_BytesObject): + raise ClassShadowError("bogus selector in method dict") + selector = w_selector.as_string() w_compiledmethod = w_values.fetch(i) - self.methoddict[w_selector] = w_compiledmethod + self.methoddict[selector] = w_compiledmethod # for the rest, we need to reset invalid to False already so # that cycles in the superclass and/or metaclass chains don't # cause infinite recursion @@ -169,12 +172,12 @@ return "" % (self.name or '?',) def lookup(self, selector): - if selector in self.methoddict: - return self.methoddict[selector] - elif self.s_superclass is not None: - return self.s_superclass.lookup(selector) - else: - raise MethodNotFound + look_in_shadow = self + while selector not in look_in_shadow.methoddict: + look_in_shadow = look_in_shadow.s_superclass + if look_in_shadow is None: + raise MethodNotFound + return look_in_shadow.methoddict[selector] def installmethod(self, selector, method): "NOT_RPYTHON" # this is only for testing. Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Thu Oct 25 20:39:10 2007 @@ -26,8 +26,19 @@ setup() +def fakesymbol(s, _cache={}): + try: + return _cache[s] + except KeyError: + result = _cache[s] = objtable.wrap_string(s) + return result + def fakeliterals(*literals): - return ["methodheader"] + list(literals) + lst = ["methodheader"] + list(literals) + for i in range(len(lst)): + if isinstance(lst[i], str): + lst[i] = fakesymbol(lst[i]) + return lst def new_interpreter(bytes, receiver=objtable.w_nil): assert isinstance(bytes, str) @@ -108,7 +119,9 @@ interp.step() interp.step() interp.step() - assert interp.w_active_context.stack == ["a", "b", "c"] + assert interp.w_active_context.stack == [fakesymbol("a"), + fakesymbol("b"), + fakesymbol("c")] def test_pushLiteralVariableBytecode(bytecode=pushLiteralVariableBytecode(0)): w_association = mockclass(2).as_class_get_shadow().new() Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Oct 25 20:39:10 2007 @@ -34,29 +34,6 @@ image = get_image() return image.special(constants.SO_FLOAT_CLASS) -# ------ custom lookup implementations -------------------------------- - -def compiledmethodnamed(w_class, methodname): - w_methoddict = w_class.fetch(constants.CLASS_METHODDICT_INDEX)._vars - names = w_methoddict[constants.METHODDICT_NAMES_INDEX:] - values = w_methoddict[constants.METHODDICT_VALUES_INDEX]._vars - for var in names: - if isinstance(var, model.W_BytesObject): - if str(var) == str(methodname): - return values[names.index(var)] - raise shadow.MethodNotFound - -def lookup(w_class, methodname): - in_class = w_class - while in_class != None: - try: - return compiledmethodnamed(in_class, methodname) - except shadow.MethodNotFound: - pass - in_class = in_class._vars[constants.CLASS_SUPERCLASS_INDEX] - if in_class is objtable.w_nil: - raise shadow.MethodNotFound - # ------ tests ------------------------------------------ def test_miniimageexists(): @@ -218,15 +195,8 @@ # Should get this from w_object w_smallint_class = image.special(constants.SO_SMALLINTEGER_CLASS) - w_method = lookup(w_smallint_class, "abs") - - # XXX - # currently still using highlevel lookup directly pointing to - # class. Should work using classmirrors when the metaclass of - # SmallInt is correctly set - - # s_class = w_object.shadow_of_my_class() - # w_method = s_class.lookup("abs") + s_class = w_object.shadow_of_my_class() + w_method = s_class.lookup("abs") assert w_method w_frame = w_method.create_frame(w_object, []) Modified: pypy/dist/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Thu Oct 25 20:39:10 2007 @@ -18,8 +18,9 @@ w_methoddict.store(constants.METHODDICT_VALUES_INDEX, w_array) positions = range(size) random.shuffle(positions) - for w_selector, w_compiledmethod in methods.items(): + for selector, w_compiledmethod in methods.items(): pos = positions.pop() + w_selector = objtable.wrap_string(selector) w_methoddict.store(constants.METHODDICT_NAMES_INDEX+pos, w_selector) w_array.store(pos, w_compiledmethod) #print w_methoddict._vars @@ -62,8 +63,8 @@ yield basicshape, "CompiledMeth", 0xE02, shadow.COMPILED_METHOD, True, 0 def test_methoddict(): - methods = {objtable.wrap_string('foo'): 'foo_method', - objtable.wrap_string('bar'): 'bar_method'} + methods = {'foo': 'foo_method', + 'bar': 'bar_method'} w_class = build_smalltalk_class("Demo", 0x90, methods=methods) classshadow = w_class.as_class_get_shadow() assert classshadow.methoddict == methods From arigo at codespeak.net Thu Oct 25 20:42:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 20:42:27 +0200 (CEST) Subject: [pypy-svn] r47983 - pypy/dist/pypy/translator/goal Message-ID: <20071025184227.229698136@code0.codespeak.net> Author: arigo Date: Thu Oct 25 20:42:26 2007 New Revision: 47983 Modified: pypy/dist/pypy/translator/goal/targetfibsmalltalk.py Log: Fix the target. Modified: pypy/dist/pypy/translator/goal/targetfibsmalltalk.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetfibsmalltalk.py (original) +++ pypy/dist/pypy/translator/goal/targetfibsmalltalk.py Thu Oct 25 20:42:26 2007 @@ -9,21 +9,33 @@ def new_interpreter(bytes): assert isinstance(bytes, str) w_method = model.W_CompiledMethod(0, bytes=bytes) - w_frame = w_method.createFrame(objtable.w_nil, []) + w_frame = w_method.create_frame(objtable.w_nil, []) interp = interpreter.Interpreter() interp.w_active_context = w_frame return interp +def build(): + bytecode = ''.join(map(chr, [ 16, 119, 178, 154, 118, 164, 11, 112, 16, 118, 177, 224, 112, 16, 119, 177, 224, 176, 124 ])) + shadow = mockclass(0).as_class_get_shadow() + method = model.W_CompiledMethod(1, bytecode, 1) + method.literals = fakeliterals("fib:") + shadow.installmethod("fib:", method) + w_object = shadow.new() + interp = new_interpreter(sendLiteralSelectorBytecode(16) + returnTopFromMethod) + interp.w_active_context.w_method().literals = fakeliterals("fib:") + return interp, w_object + +def check_me(): + interp, w_object = build() + interp.w_active_context.push(w_object) + interp.w_active_context.push(wrap_int(8)) + result = interp.interpret() + assert primitives.unwrap_int(result) == 34 + print "check_me() ok" +check_me() -bytecode = ''.join(map(chr, [ 16, 119, 178, 154, 118, 164, 11, 112, 16, 118, 177, 224, 112, 16, 119, 177, 224, 176, 124 ])) -shadow = mockclass(0).as_class_get_shadow() -method = model.W_CompiledMethod(1, bytecode, 1) -method.literals = fakeliterals("fib:") -shadow.installmethod("fib:", method) -w_object = shadow.new() -interp = new_interpreter(sendLiteralSelectorBytecode(16) + returnTopFromMethod) -interp.w_active_context.method.literals = fakeliterals("fib:") +interp, w_object = build() def entry_point(argv): if len(argv) > 1: From tismer at codespeak.net Thu Oct 25 20:55:41 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Thu, 25 Oct 2007 20:55:41 +0200 (CEST) Subject: [pypy-svn] r47986 - pypy/dist/pypy/module/zipimport Message-ID: <20071025185541.CD09B8138@code0.codespeak.net> Author: tismer Date: Thu Oct 25 20:55:38 2007 New Revision: 47986 Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py Log: made zipfile fully windows-aware. Refactored a bit, renamed things to be clear what is a file name and what is a dotted module name. Removed unused data Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/interp_zipimport.py Thu Oct 25 20:55:38 2007 @@ -12,6 +12,10 @@ import os import stat +ZIPSEP = '/' +# note that zipfiles always use slash, but for OSes with other +# separators, we need to pretend that we had the os.sep. + zip_importer_cache = {} ENUMERATE_EXTS = unrolling_iterable( @@ -28,12 +32,6 @@ self.name = name self.w_dir = w_dir self.w_zipfile = w_zipfile - self.filelist_w = space.unpackiterable( - space.getattr(w_dir, space.wrap('filelist'))) - # XXX unicode? - self.namelist = [space.str_w(i) for i in space.unpackiterable( - space.call(space.getattr(w_dir, space.wrap('namelist')), - space.newlist([])))] def import_py_file(self, space, modname, filename, w_buf, pkgpath): buf = space.str_w(w_buf) @@ -61,82 +59,83 @@ space.setattr(w_mod, w('__loader__'), space.wrap(self)) return result - def get_module(self, space, name): + def have_modulefile(self, space, filename): + if ZIPSEP != os.path.sep: + filename = filename.replace(os.path.sep, ZIPSEP) w = space.wrap try: return space.call(space.getattr(self.w_dir, w('getinfo')), - space.newlist([w(name)])) + space.newlist([w(filename)])) except OperationError, e: if not e.match(space, space.w_KeyError): # should never happen raise e - def find_module(self, space, import_name, w_path=None): - import_name = import_name.replace('.', os.path.sep) + def find_module(self, space, fullname, w_path=None): + filename = self.mangle(fullname) for _, _, ext in ENUMERATE_EXTS: - if self.get_module(space, import_name + ext): + if self.have_modulefile(space, filename + ext): return space.wrap(self) find_module.unwrap_spec = ['self', ObjSpace, str, W_Root] def mangle(self, name): return name.replace('.', os.path.sep) - def load_module(self, space, name): + def load_module(self, space, fullname): w = space.wrap w_modules = space.sys.get('modules') try: - return space.getitem(w_modules, w(name)) + return space.getitem(w_modules, w(fullname)) except OperationError, e: pass - filename = self.mangle(name) + filename = self.mangle(fullname) w_ZipImportError = space.getattr(space.getbuiltinmodule('zipimport'), w('ZipImportError')) last_exc = None for compiled, is_package, ext in ENUMERATE_EXTS: try: - w_buf = self._get_data(space, w(filename + ext)) + fname = filename + ext + w_buf = self.get_data(space, fname) if is_package: pkgpath = self.name else: pkgpath = None if compiled: - return self.import_pyc_file(space, name, filename + ext, + return self.import_pyc_file(space, fullname, fname, w_buf, pkgpath) else: - return self.import_py_file(space, name, filename + ext, + return self.import_py_file(space, fullname, fname, w_buf, pkgpath) except OperationError, e: last_exc = e w_mods = space.sys.get('modules') - space.call_method(w_mods,'pop', w(name), space.w_None) + space.call_method(w_mods,'pop', w(fullname), space.w_None) if last_exc: raise OperationError(w_ZipImportError, last_exc.w_value) # should never happen I think return space.w_None load_module.unwrap_spec = ['self', ObjSpace, str] - def _get_data(self, space, w_filename): + def get_data(self, space, filename): + if ZIPSEP != os.path.sep: + filename = filename.replace(os.path.sep, ZIPSEP) w = space.wrap try: return space.call(space.getattr(self.w_dir, w('read')), - space.newlist([w_filename])) + space.newlist([w(filename)])) except OperationError, e: raise OperationError(space.w_IOError, e.w_value) - - def get_data(self, space, filename): - filename = self.mangle(filename) - return self._get_data(space, space.wrap(filename)) get_data.unwrap_spec = ['self', ObjSpace, str] - def get_code(self, space, filename): + def get_code(self, space, fullname): + filename = self.mangle(fullname) w = space.wrap - filename = self.mangle(filename) for compiled, _, ext in ENUMERATE_EXTS: - if self.get_module(space, filename + ext): + if self.have_modulefile(space, filename + ext): if compiled: - return self._get_data(space, space.wrap(filename + ext)) + return self.get_data(space, filename + ext) else: - w_source = self._get_data(space, space.wrap(filename + ext)) + w_source = self.get_data(space, filename + ext) w_code = space.builtin.call('compile', w_source, w(filename + ext), w('exec')) return w_code @@ -144,19 +143,21 @@ "Cannot find source or code for %s in %s" % (filename, self.name))) get_code.unwrap_spec = ['self', ObjSpace, str] - def get_source(self, space, filename): - filename = self.mangle(filename) + def get_source(self, space, fullname): + filename = self.mangle(fullname) for compiled, _, ext in ENUMERATE_EXTS: if not compiled: - if self.get_module(space, filename + ext): - return self._get_data(space, space.wrap(filename + ext)) + fname = filename + ext + if self.have_modulefile(space, fname): + return self.get_data(space, fname) raise OperationError(space.w_ImportError, space.wrap( "Cannot find source for %s in %s" % (filename, self.name))) get_source.unwrap_spec = ['self', ObjSpace, str] - def is_package(self, space, filename): + def is_package(self, space, fullname): + filename = self.mangle(fullname) for _, is_package, ext in ENUMERATE_EXTS: - if self.get_module(space, filename + ext): + if self.have_modulefile(space, filename + ext): return space.wrap(is_package) raise OperationError(space.w_ImportError, space.wrap( "Cannot find module %s in %s" % (filename, self.name))) From tverwaes at codespeak.net Thu Oct 25 21:19:05 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 21:19:05 +0200 (CEST) Subject: [pypy-svn] r47987 - in pypy/dist/pypy/lang/smalltalk: . tool Message-ID: <20071025191905.234F4813B@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 21:19:03 2007 New Revision: 47987 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: fixed supersends coming from compiledmethods loaded from image (fetching compiledin from assoc at the end of the literals) Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 21:19:03 2007 @@ -121,7 +121,7 @@ receiver, receiver.shadow_of_my_class()) def _sendSuperSelector(self, selector, argcount, interp): - s_compiledin = self.w_method().w_compiledin.as_class_get_shadow() + s_compiledin = self.w_method().compiledin().as_class_get_shadow() self._sendSelector(selector, argcount, interp, self.w_receiver, s_compiledin.s_superclass) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 21:19:03 2007 @@ -205,6 +205,7 @@ The trailer has two variant formats. In the first variant, the last byte is at least 252 and the last four bytes represent a source pointer into one of the sources files (see #sourcePointer). In the second variant, the last byte is less than 252, and the last several bytes are a compressed version of the names of the method's temporary variables. The number of bytes used for this purpose is the value of the last byte in the method. """ + def __init__(self, literalsize, bytes, argsize=0, tempsize=0, primitive=0, w_compiledin=None): self.literals = [None] * literalsize @@ -214,6 +215,13 @@ self.tempsize = tempsize self.primitive = primitive + def compiledin(self): + if self.w_compiledin == None: + # Last of the literals is an association with compiledin + # as a class + self.w_compiledin = self.literals[-1].fetch(constants.ASSOCIATION_VALUE_INDEX) + return self.w_compiledin + def getclass(self): from pypy.lang.smalltalk.classtable import w_CompiledMethod return w_CompiledMethod Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Oct 25 21:19:03 2007 @@ -386,8 +386,6 @@ primitive = primitive) w_compiledmethod.literals = literals - - class ImageChunk(object): def __init__(self, size, format, classid, hash12): self.size = size Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Thu Oct 25 21:19:03 2007 @@ -7,7 +7,7 @@ from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import interpreter -mini_image = py.magic.autopath().dirpath().dirpath().join('tool/squeak3.9.image') +mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') def get_miniimage(): return squeakimage.ImageReader(squeakimage.Stream(mini_image.open())) @@ -27,85 +27,43 @@ print each.shadow_of_my_class() print each.as_string() -def testCompiledMethods(): +def tinyBenchmarks(): image = create_squeakimage() - amethod = None - - w_smallint_class = image.special(constants.SO_SMALLINTEGER_CLASS) - interp = interpreter.Interpreter() - amethod = w_smallint_class.lookup("abs") - # First literal of the abs method is - # a real smalltalk int - w_frame = amethod.create_frame(model.W_SmallInteger(3), []) - interp.activeContext = w_frame - - print amethod - - while True: - try: - interp.step() - print interp.activeContext.stack - except interpreter.ReturnFromTopLevel, e: - return e.object - -def testDoesNotUnderstand(): - image = create_squeakimage() - amethod = None - - w_doesnot = image.special(constants.SO_DOES_NOT_UNDERSTAND) - w_object = objtable.wrap_int(3) - w_message_class = image.special(constants.SO_MESSAGE_CLASS) - s_message_class = w_message_class.as_class_get_shadow() - - #Build message argument - w_message = s_message_class.new(1) - w_message.store(constants.MESSAGE_SELECTOR_INDEX, objtable.wrap_string("zork")) - w_aarray = classtable.w_Array.as_class_get_shadow().new(0) - w_message.store(constants.MESSAGE_ARGUMENTS_INDEX, w_aarray) - if s_message_class.instsize() > constants.MESSAGE_LOOKUP_CLASS_INDEX: - w_message.store(constants.MESSAGE_LOOKUP_CLASS_INDEX, w_object.getclass()) + w_object = model.W_SmallInteger(0) + # Should get this from w_object + w_smallint_class = image.special(constants.SO_SMALLINTEGER_CLASS) s_class = w_object.shadow_of_my_class() - w_method = s_class.lookup(w_doesnot) - - interp = interpreter.Interpreter() + w_method = s_class.lookup("tinyBenchmarks") - # First literal of the abs method is - # a real smalltalk int - w_frame = w_method.create_frame(w_object, [w_message]) - print "WFRAME: %r" % (w_frame,) + assert w_method + w_frame = w_method.create_frame(w_object, []) interp.w_active_context = w_frame print w_method + print "Going to execute %d toplevel bytecodes" % (len(w_method.bytes),) + counter = 0 while True: try: - print "Stackbefore: %r" % (interp.w_active_context.stack,) interp.step() - print "Stackafter: %r" % (interp.w_active_context.stack,) + print interp.w_active_context.stack + if interp.w_active_context == w_frame: + counter += 1 + print "Executing toplevel bytecode nr: %d of %d" % (counter, len(w_method.bytes)) except interpreter.ReturnFromTopLevel, e: - return e.object + assert e.object.value == abs(int) + return -def testSelector(): - image = create_squeakimage() - w_doesnot = image.special(constants.SO_DOES_NOT_UNDERSTAND) - assert repr(w_doesnot.shadow_of_my_class()) == "" - print w_doesnot.getclass().fetch(constants.CLASS_METHODDICT_INDEX).shadow_of_my_class().instance_kind - print w_doesnot.getclass().fetch(constants.CLASS_METHODDICT_INDEX).shadow_of_my_class().instance_size - print - print w_doesnot.getclass().fetch(constants.CLASS_METHODDICT_INDEX)._vars - print - print w_doesnot.getclass().fetch(constants.CLASS_METHODDICT_INDEX)._vars[constants.METHODDICT_NAMES_INDEX:] - print - print w_doesnot.getclass().fetch(constants.CLASS_METHODDICT_INDEX)._vars[constants.METHODDICT_VALUES_INDEX]._vars def test_do(): #testSelector() - printStringsInImage() + #printStringsInImage() #testDoesNotUnderstand() + tinyBenchmarks() if __name__ == '__main__': test_do() From arigo at codespeak.net Thu Oct 25 21:19:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 21:19:57 +0200 (CEST) Subject: [pypy-svn] r47988 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025191957.5F120813B@code0.codespeak.net> Author: arigo Date: Thu Oct 25 21:19:57 2007 New Revision: 47988 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py Log: (akuhn, arigo, cfbolz around) Various bugs and type inconsistencies found by trying to translate. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 21:19:57 2007 @@ -10,9 +10,6 @@ class IllegalStoreError(Exception): """Illegal Store.""" -class IllegalFetchError(Exception): - """Illegal Fetch.""" - class Interpreter: TRUE = objtable.w_true @@ -470,17 +467,18 @@ def initialize_bytecode_table(): result = [None] * 256 for entry in BYTECODE_RANGES: - def dump_func(f): - def wrapped(*args): - print "Bytecode: %s" % (f.__name__) - return f(*args) - return wrapped + #def dump_func(f): + # def wrapped(*args): + # print "Bytecode: %s" % (f.__name__) + # return f(*args) + # return wrapped if len(entry) == 2: positions = [entry[0]] else: positions = range(entry[0], entry[1]+1) for pos in positions: - result[pos] = dump_func(entry[-1]) + #result[pos] = dump_func(entry[-1]) + result[pos] = entry[-1] assert None not in result return result Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 21:19:57 2007 @@ -37,11 +37,14 @@ def __repr__(self): return "W_SmallInteger(%d)" % self.value - + +class UnwrappingError(Exception): + pass + def unwrap_int(w_value): if isinstance(w_value, W_SmallInteger): return w_value.value - raise ClassShadowError("expected a W_SmallInteger, got %s" % (w_value,)) + raise UnwrappingError("expected a W_SmallInteger, got %s" % (w_value,)) class W_Float(W_Object): def __init__(self, value): @@ -267,24 +270,24 @@ assert isinstance(w_home, W_MethodContext) self.w_home = w_home self.w_sender = w_sender - - def getclass(self): - from pypy.lang.smalltalk.classtable import w_ContextPart - return w_ContextPart # ______________________________________________________________________ # Imitate the primitive accessors def fetch(self, index): - if index == CTXPART_SENDER_INDEX: + from pypy.lang.smalltalk import objtable + if index == constants.CTXPART_SENDER_INDEX: return self.w_sender - elif index == CTXPART_PC_INDEX: + elif index == constants.CTXPART_PC_INDEX: return objtable.wrap_int(self.pc) - elif index == CTXPART_STACKP_INDEX: + elif index == constants.CTXPART_STACKP_INDEX: return objtable.wrap_int(len(self.stack)) # Invalid! - raise IllegalFetchError + raise IndexError + + def store(self, index, value): + raise NotImplementedError # ______________________________________________________________________ # Method that contains the bytecode for this method/block context @@ -352,30 +355,33 @@ return w_BlockContext def fetch(self, index): - if index == BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX: + from pypy.lang.smalltalk import objtable + if index == constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX: return objtable.wrap_int(self.argcnt) - elif index == BLKCTX_INITIAL_IP_INDEX: + elif index == constants.BLKCTX_INITIAL_IP_INDEX: return objtable.wrap_int(self.initialip) - elif index == BLKCTX_HOME_INDEX: + elif index == constants.BLKCTX_HOME_INDEX: return self.w_home - elif index >= BLKCTX_TEMP_FRAME_START: + elif index >= constants.BLKCTX_TEMP_FRAME_START: stack_index = len(self.stack) - index - 1 return self.stack[stack_index] else: return W_ContextPart.fetch(index) def store(self, index, value): - if index == BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX: - self.argcnt = unwrap_int(self.argcnt) - elif index == BLKCTX_INITIAL_IP_INDEX: - self.pc = unwrap_int(self.argcnt) - elif index == BLKCTX_HOME_INDEX: + # THIS IS ALL UNTESTED CODE and we're a bit unhappy about it + # because it crashd the translation N times :-( + if index == constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX: + self.argcnt = unwrap_int(value) + elif index == constants.BLKCTX_INITIAL_IP_INDEX: + self.pc = unwrap_int(value) + elif index == constants.BLKCTX_HOME_INDEX: self.w_home = value - elif index >= BLKCTX_TEMP_FRAME_START: + elif index >= constants.BLKCTX_TEMP_FRAME_START: stack_index = len(self.stack) - index - 1 self.stack[stack_index] = value else: - return W_ContextPart.fetch(index) + W_ContextPart.store(index, value) class W_MethodContext(W_ContextPart): def __init__(self, w_method, w_receiver, arguments, w_sender = None): @@ -389,15 +395,16 @@ return w_MethodContext def fetch(self, index): - if index == MTHDCTX_METHOD: + from pypy.lang.smalltalk import objtable + if index == constants.MTHDCTX_METHOD: return self.w_method() - elif index == MTHDCTX_RECEIVER_MAP: # what is this thing? + elif index == constants.MTHDCTX_RECEIVER_MAP: # what is this thing? return objtable.w_nil - elif index == MTHDCTX_RECEIVER: + elif index == constants.MTHDCTX_RECEIVER: return self.w_receiver - elif index >= MTHDCTX_TEMP_FRAME_START: + elif index >= constants.MTHDCTX_TEMP_FRAME_START: # First set of indices are temporary variables: - offset = index - MTHDCTX_TEMP_FRAME_START + offset = index - constants.MTHDCTX_TEMP_FRAME_START if offset < len(self.temps): return self.temps[offset] Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 21:19:57 2007 @@ -100,9 +100,6 @@ ADD: operator.add, SUBTRACT: operator.sub, MULTIPLY: operator.mul, - BIT_AND: operator.and_, - BIT_OR: operator.or_, - BIT_XOR: operator.xor } for (code,op) in math_ops.items(): def make_func(op): @@ -118,6 +115,22 @@ return wrap_int(res) make_func(op) +bitwise_binary_ops = { + BIT_AND: operator.and_, + BIT_OR: operator.or_, + BIT_XOR: operator.xor, + } +for (code,op) in bitwise_binary_ops.items(): + def make_func(op): + @primitive(code) + @stack(2) + def func(args, (w_receiver, w_argument)): + receiver = unwrap_int(w_receiver) + argument = unwrap_int(w_argument) + res = op(receiver, argument) + return wrap_int(res) + make_func(op) + # #/ -- return the result of a division, only succeed if the division is exact @primitive(DIVIDE) @stack(2) @@ -453,7 +466,10 @@ @primitive(EQUIVALENT) @stack(2) def func(args, (w_arg, w_rcvr)): - return w_arg == w_rcvr + # XXX this is bogus in the presence of (our implementation of) become, + # as we might plan to implement become by copying all fields from one + # object to the other + return objtable.wrap_bool(w_arg is w_rcvr) @primitive(EQUIVALENT) @stack(1) @@ -506,7 +522,7 @@ INC_GC = 131 def fake_bytes_left(): - return 2**20 # XXX we don't know how to do this :-( + return wrap_int(2**20) # XXX we don't know how to do this :-( @primitive(INC_GC) # XXX the same for now @primitive(FULL_GC) @@ -593,10 +609,12 @@ (PUSH_ONE, objtable.w_one), (PUSH_TWO, objtable.w_two), ]: - @primitive(code) - @stack(1) - def func(args, stack, const=const): # n.b.: capture const - return const + def make_func(const): + @primitive(code) + @stack(1) + def func(args, stack): + return const + make_func(const) define_const_primitives() # ___________________________________________________________________________ From arigo at codespeak.net Thu Oct 25 21:21:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 21:21:57 +0200 (CEST) Subject: [pypy-svn] r47989 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025192157.0F5F5813B@code0.codespeak.net> Author: arigo Date: Thu Oct 25 21:21:56 2007 New Revision: 47989 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: More untested code found. Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 21:21:56 2007 @@ -52,6 +52,7 @@ def primitive(code): def decorator(func): + assert code not in prim_table prim_table[code] = func return func return decorator @@ -471,7 +472,7 @@ # object to the other return objtable.wrap_bool(w_arg is w_rcvr) - at primitive(EQUIVALENT) + at primitive(CLASS) @stack(1) def func(args, (w_obj,)): return w_obj.w_class From arigo at codespeak.net Thu Oct 25 21:22:56 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 21:22:56 +0200 (CEST) Subject: [pypy-svn] r47990 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025192256.568B0813B@code0.codespeak.net> Author: arigo Date: Thu Oct 25 21:22:56 2007 New Revision: 47990 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: Yet more untested code. (found by cfbolz) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 21:22:56 2007 @@ -329,7 +329,7 @@ def func(args, (w_obj,)): if not w_obj.shadow_of_my_class().isvariable(): raise PrimitiveFailedError() - return w_obj.size() + return wrap_int(w_obj.size()) @primitive(STRING_AT) @stack(2) From arigo at codespeak.net Thu Oct 25 21:41:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 21:41:40 +0200 (CEST) Subject: [pypy-svn] r47991 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025194140.BE6A2812D@code0.codespeak.net> Author: arigo Date: Thu Oct 25 21:41:40 2007 New Revision: 47991 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py Log: More translation fixes. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 21:41:40 2007 @@ -65,6 +65,7 @@ # named var (the value). index = self.currentBytecode & 31 association = self.w_method().getliteral(index) + assert isinstance(association, model.W_PointersObject) self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) def storeAndPopReceiverVariableBytecode(self, interp): @@ -181,6 +182,7 @@ self.push(self.w_method().getliteral(variableIndex)) elif variableType == 3: association = self.w_method().getliteral(variableIndex) + assert isinstance(association, model.W_PointersObject) self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) def extendedStoreBytecode(self, interp): @@ -193,6 +195,7 @@ raise IllegalStoreError elif variableType == 3: association = self.w_method().getliteral(variableIndex) + assert isinstance(association, model.W_PointersObject) association.store(constants.ASSOCIATION_VALUE_INDEX, self.top()) def extendedStoreAndPopBytecode(self, interp): @@ -229,6 +232,7 @@ elif opType == 4: # pushLiteralVariable association = self.w_method().getliteral(third) + assert isinstance(association, model.W_PointersObject) self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) elif opType == 5: self.w_receiver.store(third, self.top()) @@ -236,6 +240,7 @@ self.w_receiver.store(third, self.pop()) elif opType == 7: association = self.w_method().getliteral(third) + assert isinstance(association, model.W_PointersObject) association.store(constants.ASSOCIATION_VALUE_INDEX, self.top()) def singleExtendedSuperBytecode(self, interp): Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 21:41:40 2007 @@ -55,7 +55,7 @@ return w_Float def gethash(self): - return XXX # check this + return 41 # XXX check this def invariant(self): return self.value is not None # XXX but later: @@ -222,7 +222,9 @@ if self.w_compiledin == None: # Last of the literals is an association with compiledin # as a class - self.w_compiledin = self.literals[-1].fetch(constants.ASSOCIATION_VALUE_INDEX) + association = self.literals[-1] + assert isinstance(association, W_PointersObject) + self.w_compiledin = association.fetch(constants.ASSOCIATION_VALUE_INDEX) return self.w_compiledin def getclass(self): @@ -366,11 +368,11 @@ stack_index = len(self.stack) - index - 1 return self.stack[stack_index] else: - return W_ContextPart.fetch(index) + return W_ContextPart.fetch(self, index) def store(self, index, value): # THIS IS ALL UNTESTED CODE and we're a bit unhappy about it - # because it crashd the translation N times :-( + # because it crashd the translation N+3 times :-( if index == constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX: self.argcnt = unwrap_int(value) elif index == constants.BLKCTX_INITIAL_IP_INDEX: @@ -381,7 +383,7 @@ stack_index = len(self.stack) - index - 1 self.stack[stack_index] = value else: - W_ContextPart.store(index, value) + W_ContextPart.store(self, index, value) class W_MethodContext(W_ContextPart): def __init__(self, w_method, w_receiver, arguments, w_sender = None): @@ -413,5 +415,5 @@ stack_index = len(self.stack) - offset - 1 return self.stack[stack_index] else: - return W_ContextPart.fetch(index) + return W_ContextPart.fetch(self, index) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 21:41:40 2007 @@ -18,7 +18,7 @@ def subscript(idx, w_obj): if isinstance(w_obj, model.W_PointersObject): - return w_obj.getindexedvar(idx) + return w_obj.fetch(idx) elif isinstance(w_obj, model.W_WordsObject): return objtable.wrap_int(w_obj.getword(idx)) elif isinstance(w_obj, model.W_BytesObject): @@ -406,6 +406,8 @@ def func(args, (w_rcvr, w_idx)): # I *think* this is the correct behavior, but I'm not quite sure. # Might be restricted to fixed length fields? + # XXX this doesn't look correct. Our guess is that INST_VAR_AT + # is used to access *only* the fixed length fields. idx = unwrap_int(w_idx) shadow = w_rcvr.shadow_of_my_class() if idx < 0: @@ -426,7 +428,7 @@ def func(args, (w_rcvr,)): if isinstance(w_rcvr, model.W_SmallInteger): raise PrimitiveFailedError() - return w_rcvr.w_hash + return wrap_int(w_rcvr.gethash()) @primitive(STORE_STACKP) @stack(2) @@ -506,6 +508,7 @@ # or vice versa (?) # 3. Format of rcvr is different from format of argument + raise PrimitiveNotYetWrittenError() # XXX needs to work in the shadows if w_arg_class.format != w_rcvr_class.format: raise PrimitiveFailedError() From jacob at codespeak.net Thu Oct 25 21:54:33 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Thu, 25 Oct 2007 21:54:33 +0200 (CEST) Subject: [pypy-svn] r47992 - pypy/branch/dist-future-fixing/pypy/interpreter/pyparser Message-ID: <20071025195433.C95A6813A@code0.codespeak.net> Author: jacob Date: Thu Oct 25 21:54:31 2007 New Revision: 47992 Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py Log: Cosmetic changes. Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py Thu Oct 25 21:54:31 2007 @@ -62,7 +62,7 @@ SyntaxError.__init__(self, msg, lineno, offset, line) self.token_stack = token_stack -def generate_tokens( parser, lines, flags): +def generate_tokens(parser, lines, flags): """ This is a rewrite of pypy.module.parser.pytokenize.generate_tokens since the original function is not RPYTHON (uses yield) @@ -147,12 +147,17 @@ if not line: break column = 0 while pos < max: # measure leading whitespace - if line[pos] == ' ': column = column + 1 - elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize - elif line[pos] == '\f': column = 0 - else: break + if line[pos] == ' ': + column = column + 1 + elif line[pos] == '\t': + column = (column / tabsize + 1) * tabsize + elif line[pos] == '\f': + column = 0 + else: + break pos = pos + 1 - if pos == max: break + if pos == max: + break if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] == '#': @@ -270,9 +275,10 @@ start = whiteSpaceDFA.recognize(line, pos) if start < 0: start = pos - if start Author: arigo Date: Thu Oct 25 21:56:20 2007 New Revision: 47993 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py Log: (akuhn, arigo, rtype) Final translation fixes. It translates! It runs! It gives the right answer! Yippee. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 21:56:20 2007 @@ -137,7 +137,9 @@ # the primitive succeeded self.push(w_result) return - arguments = self.stack[len(self.stack)-argcount:] + start = len(self.stack) - argcount + assert start >= 0 # XXX check in the Blue Book what to do in this case + arguments = self.stack[start:] interp.w_active_context = method.create_frame(receiver, arguments, self) self.pop_n(argcount + 1) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 21:56:20 2007 @@ -219,7 +219,7 @@ self.primitive = primitive def compiledin(self): - if self.w_compiledin == None: + if self.w_compiledin is None: # Last of the literals is an association with compiledin # as a class association = self.literals[-1] @@ -338,8 +338,10 @@ return self.stack[-(idx+1)] def pop_n(self, n): - res = self.stack[len(self.stack)-n:] - self.stack = self.stack[:len(self.stack)-n] + start = len(self.stack) - n + assert start >= 0 # XXX what if this fails? + res = self.stack[start:] + self.stack = self.stack[:start] return res class W_BlockContext(W_ContextPart): Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 21:56:20 2007 @@ -61,9 +61,10 @@ def decorator(wrapped): def result(args): frame = args.interp.w_active_context - if len(frame.stack) < n: - raise PrimitiveFailedError() - items = frame.stack[len(frame.stack)-n:] + start = len(frame.stack) - n + if start < 0: + raise PrimitiveFailedError() # not enough arguments + items = frame.stack[start:] res = wrapped(args, items) frame.pop_n(n) # only if no exception occurs! return res From tverwaes at codespeak.net Thu Oct 25 22:12:39 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 22:12:39 +0200 (CEST) Subject: [pypy-svn] r47994 - in pypy/dist/pypy/lang/smalltalk: . tool Message-ID: <20071025201239.4EC99813C@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 22:12:39 2007 New Revision: 47994 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: Don't save currentbytecode for longjumps Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 22:12:39 2007 @@ -282,7 +282,7 @@ self.jump(self.longJumpPosition()) def longJumpPosition(self): - return ((self.currentBytecode & 3) << 8) + self.getNextBytecode() + return ((self.currentBytecode & 3) << 8) + self.getByte() def longJumpIfTrue(self, interp): self.jumpConditional(interp.TRUE,self.longJumpPosition()) Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Thu Oct 25 22:12:39 2007 @@ -46,16 +46,26 @@ print "Going to execute %d toplevel bytecodes" % (len(w_method.bytes),) counter = 0 + from pypy.lang.smalltalk.interpreter import BYTECODE_TABLE while True: try: - interp.step() - print interp.w_active_context.stack if interp.w_active_context == w_frame: counter += 1 print "Executing toplevel bytecode nr: %d of %d" % (counter, len(w_method.bytes)) + interp.step() + if hasattr(interp.w_active_context,"currentBytecode"): + print "Executing bytecode: %s" % (BYTECODE_TABLE[interp.w_active_context.currentBytecode].__name__,) + else: + print "Jump to new stackframe" + print interp.w_active_context.stack except interpreter.ReturnFromTopLevel, e: assert e.object.value == abs(int) return + except: + if hasattr(interp.w_active_context,"currentBytecode"): + cb = interp.w_active_context.currentBytecode + print "Failing bytecode: %s %d" % (BYTECODE_TABLE[cb].__name__,cb) + raise From tverwaes at codespeak.net Thu Oct 25 22:38:34 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 22:38:34 +0200 (CEST) Subject: [pypy-svn] r47995 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025203834.9951D8146@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 22:38:34 2007 New Revision: 47995 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py Log: fixing bytecode of unconditionaljump Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Thu Oct 25 22:38:34 2007 @@ -279,7 +279,7 @@ self.jumpConditional(interp.FALSE,self.shortJumpPosition()) def longUnconditionalJump(self, interp): - self.jump(self.longJumpPosition()) + self.jump((((self.currentBytecode & 7) - 4) << 8) + self.getByte()) def longJumpPosition(self): return ((self.currentBytecode & 3) << 8) + self.getByte() From tverwaes at codespeak.net Thu Oct 25 22:57:08 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 22:57:08 +0200 (CEST) Subject: [pypy-svn] r47996 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025205708.7EB888157@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 22:57:06 2007 New Revision: 47996 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: Fixing at: and at:put: to use the absolutely uniquely correct form of 1-based indexing (and translating it to dumb-peoples 0-based indexing) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 22:57:06 2007 @@ -303,14 +303,14 @@ def common_at((w_obj, w_idx)): idx = unwrap_int(w_idx) # XXX should be idx-1, probably - assert_valid_index(idx, w_obj) - return w_obj, idx + assert_valid_index(idx-1, w_obj) + return w_obj, idx-1 def common_at_put((w_obj, w_idx, w_val)): idx = unwrap_int(w_idx) # XXX should be idx-1, probably - assert_valid_index(idx, w_obj) - return w_obj, idx, w_val + assert_valid_index(idx-1, w_obj) + return w_obj, idx-1, w_val @primitive(AT) @stack(2) From arigo at codespeak.net Thu Oct 25 23:02:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 23:02:20 +0200 (CEST) Subject: [pypy-svn] r47997 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025210220.6B0B38157@code0.codespeak.net> Author: arigo Date: Thu Oct 25 23:02:19 2007 New Revision: 47997 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/shadow.py pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Log: Some translation fixes that prevents any attribute from moving all the way up to W_Object. Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 23:02:19 2007 @@ -5,6 +5,7 @@ from pypy.tool.pairtype import extendabletype class W_Object(object): + __slots__ = () # no RPython-level instance variables allowed in W_Object def size(self): return 0 @@ -22,6 +23,8 @@ return self.getclass().as_class_get_shadow() class W_SmallInteger(W_Object): + __slots__ = ('value',) # the only allowed slot here + def __init__(self, value): self.value = value @@ -374,12 +377,13 @@ def store(self, index, value): # THIS IS ALL UNTESTED CODE and we're a bit unhappy about it - # because it crashd the translation N+3 times :-( + # because it crashd the translation N+4 times :-( if index == constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX: self.argcnt = unwrap_int(value) elif index == constants.BLKCTX_INITIAL_IP_INDEX: self.pc = unwrap_int(value) elif index == constants.BLKCTX_HOME_INDEX: + assert isinstance(value, W_MethodContext) self.w_home = value elif index >= constants.BLKCTX_TEMP_FRAME_START: stack_index = len(self.stack) - index - 1 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Oct 25 23:02:19 2007 @@ -478,7 +478,7 @@ @primitive(CLASS) @stack(1) def func(args, (w_obj,)): - return w_obj.w_class + return w_obj.getclass() @primitive(BYTES_LEFT) def func(args): @@ -495,8 +495,8 @@ @primitive(CHANGE_CLASS) @stack(2) def func(args, (w_arg, w_rcvr)): - w_arg_class = w_arg.w_class - w_rcvr_class = w_rcvr.w_class + w_arg_class = w_arg.getclass() + w_rcvr_class = w_rcvr.getclass() # We should fail if: Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Thu Oct 25 23:02:19 2007 @@ -98,6 +98,9 @@ raise ClassShadowError("bogus selector in method dict") selector = w_selector.as_string() w_compiledmethod = w_values.fetch(i) + if not isinstance(w_compiledmethod, model.W_CompiledMethod): + raise ClassShadowError("the methoddict must contain " + "CompiledMethods only for now") self.methoddict[selector] = w_compiledmethod # for the rest, we need to reset invalid to False already so # that cycles in the superclass and/or metaclass chains don't Modified: pypy/dist/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Thu Oct 25 23:02:19 2007 @@ -63,8 +63,8 @@ yield basicshape, "CompiledMeth", 0xE02, shadow.COMPILED_METHOD, True, 0 def test_methoddict(): - methods = {'foo': 'foo_method', - 'bar': 'bar_method'} + methods = {'foo': model.W_CompiledMethod(0, ""), + 'bar': model.W_CompiledMethod(1, "")} w_class = build_smalltalk_class("Demo", 0x90, methods=methods) classshadow = w_class.as_class_get_shadow() assert classshadow.methoddict == methods From arigo at codespeak.net Thu Oct 25 23:14:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 23:14:04 +0200 (CEST) Subject: [pypy-svn] r47998 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071025211404.41D758136@code0.codespeak.net> Author: arigo Date: Thu Oct 25 23:14:04 2007 New Revision: 47998 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: Fix one test. More failures in there :-( Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Thu Oct 25 23:14:04 2007 @@ -302,7 +302,7 @@ assert interp.w_active_context.pc == pc + 15 def test_longUnconditionalJump(): - interp = new_interpreter(longUnconditionalJump(0) + chr(15)) + interp = new_interpreter(longUnconditionalJump(4) + chr(15)) pc = interp.w_active_context.pc + 2 interp.step() assert interp.w_active_context.pc == pc + 15 From tverwaes at codespeak.net Thu Oct 25 23:16:08 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 25 Oct 2007 23:16:08 +0200 (CEST) Subject: [pypy-svn] r47999 - pypy/dist/pypy/lang/smalltalk/tool Message-ID: <20071025211608.C456E8150@code0.codespeak.net> Author: tverwaes Date: Thu Oct 25 23:16:08 2007 New Revision: 47999 Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: moving some things around. printing a # every 100.000 instructions executed. 0 tinyBenchmarks already running for 20 minutes (with over 100.000.000 instructions executed) Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Thu Oct 25 23:16:08 2007 @@ -6,6 +6,7 @@ from pypy.lang.smalltalk import objtable from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import interpreter +import sys mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') @@ -49,15 +50,18 @@ from pypy.lang.smalltalk.interpreter import BYTECODE_TABLE while True: try: - if interp.w_active_context == w_frame: - counter += 1 - print "Executing toplevel bytecode nr: %d of %d" % (counter, len(w_method.bytes)) + counter += 1 + #if interp.w_active_context == w_frame: + # print "Executing toplevel bytecode nr: %d of %d" % (counter, len(w_method.bytes)) interp.step() - if hasattr(interp.w_active_context,"currentBytecode"): - print "Executing bytecode: %s" % (BYTECODE_TABLE[interp.w_active_context.currentBytecode].__name__,) - else: - print "Jump to new stackframe" - print interp.w_active_context.stack + #if hasattr(interp.w_active_context,"currentBytecode"): + # print "Executing bytecode: %s" % (BYTECODE_TABLE[interp.w_active_context.currentBytecode].__name__,) + #else: + # print "Jump to new stackframe" + # print interp.w_active_context.stack + if counter == 100000: + counter = 0 + sys.stderr.write("#") except interpreter.ReturnFromTopLevel, e: assert e.object.value == abs(int) return From arigo at codespeak.net Thu Oct 25 23:17:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 23:17:53 +0200 (CEST) Subject: [pypy-svn] r48000 - pypy/dist/pypy/lang/smalltalk/tool Message-ID: <20071025211753.B799C8157@code0.codespeak.net> Author: arigo Date: Thu Oct 25 23:17:53 2007 New Revision: 48000 Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: Yikes! Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Thu Oct 25 23:17:53 2007 @@ -69,7 +69,7 @@ if hasattr(interp.w_active_context,"currentBytecode"): cb = interp.w_active_context.currentBytecode print "Failing bytecode: %s %d" % (BYTECODE_TABLE[cb].__name__,cb) - raise + raise From arigo at codespeak.net Thu Oct 25 23:41:18 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 23:41:18 +0200 (CEST) Subject: [pypy-svn] r48001 - pypy/dist/pypy/lang/smalltalk/tool Message-ID: <20071025214118.1B6188136@code0.codespeak.net> Author: arigo Date: Thu Oct 25 23:41:17 2007 New Revision: 48001 Added: pypy/dist/pypy/lang/smalltalk/tool/__init__.py (contents, props changed) Modified: pypy/dist/pypy/lang/smalltalk/tool/ (props changed) pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (props changed) Log: Make 'tool' a package. Added: pypy/dist/pypy/lang/smalltalk/tool/__init__.py ============================================================================== From arigo at codespeak.net Thu Oct 25 23:41:25 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 23:41:25 +0200 (CEST) Subject: [pypy-svn] r48002 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025214125.EA7048146@code0.codespeak.net> Author: arigo Date: Thu Oct 25 23:41:25 2007 New Revision: 48002 Modified: pypy/dist/pypy/lang/smalltalk/classtable.py (props changed) pypy/dist/pypy/lang/smalltalk/interpreter.py (props changed) pypy/dist/pypy/lang/smalltalk/objtable.py (props changed) pypy/dist/pypy/lang/smalltalk/primitives.py (props changed) pypy/dist/pypy/lang/smalltalk/squeakimage.py (props changed) pypy/dist/pypy/lang/smalltalk/test/test_classtable.py (props changed) pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (props changed) pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (props changed) pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py (props changed) Log: fixeol From arigo at codespeak.net Thu Oct 25 23:47:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 25 Oct 2007 23:47:31 +0200 (CEST) Subject: [pypy-svn] r48003 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025214731.197EC8140@code0.codespeak.net> Author: arigo Date: Thu Oct 25 23:47:30 2007 New Revision: 48003 Modified: pypy/dist/pypy/lang/smalltalk/model.py Log: Another assert, which for some unknown reason triggers in the tinyBenchmark. Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Oct 25 23:47:30 2007 @@ -341,6 +341,7 @@ return self.stack[-(idx+1)] def pop_n(self, n): + assert n >= 0 start = len(self.stack) - n assert start >= 0 # XXX what if this fails? res = self.stack[start:] From arigo at codespeak.net Fri Oct 26 00:08:55 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 26 Oct 2007 00:08:55 +0200 (CEST) Subject: [pypy-svn] r48005 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025220855.E12E08141@code0.codespeak.net> Author: arigo Date: Fri Oct 26 00:08:55 2007 New Revision: 48005 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: This looked wrong. Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 00:08:55 2007 @@ -678,7 +678,7 @@ # Initialize the block stack from the contents of the stack: # Don't bother to copy the 'self' argument - block_args = frame.pop_n(exp_arg_cnt - 1) + block_args = frame.pop_n(exp_arg_cnt) w_block_ctx.push_all(block_args) # Set some fields From rxe at codespeak.net Fri Oct 26 00:12:18 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 26 Oct 2007 00:12:18 +0200 (CEST) Subject: [pypy-svn] r48006 - pypy/dist/pypy/rpython/raisingops Message-ID: <20071025221218.ED4CA8124@code0.codespeak.net> Author: rxe Date: Fri Oct 26 00:12:18 2007 New Revision: 48006 Modified: pypy/dist/pypy/rpython/raisingops/raisingops.py Log: int_add_nonneg_ovf operation seems to be needed these days Modified: pypy/dist/pypy/rpython/raisingops/raisingops.py ============================================================================== --- pypy/dist/pypy/rpython/raisingops/raisingops.py (original) +++ pypy/dist/pypy/rpython/raisingops/raisingops.py Fri Oct 26 00:12:18 2007 @@ -71,6 +71,18 @@ else: raise OverflowError("integer addition") +def int_add_nonneg_ovf(x, y): + ''' + OP_INT_ADD(x,y,r); \ + if (r >= (x)); \ + else FAIL_OVF("integer addition") + ''' + r = x + y + if r >= x: + return r + else: + raise OverflowError("integer addition") + def int_sub_ovf(x, y): '''#define OP_INT_SUB_OVF(x,y,r,err) \ OP_INT_SUB(x,y,r,err); \ From akuhn at codespeak.net Fri Oct 26 00:34:25 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Fri, 26 Oct 2007 00:34:25 +0200 (CEST) Subject: [pypy-svn] r48007 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025223425.683408136@code0.codespeak.net> Author: akuhn Date: Fri Oct 26 00:34:25 2007 New Revision: 48007 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (akuhn, tverwaes) fixed all those mind numbing off by one errors in the test We strongly suggest to use n1 and n0 as identifier namesto discriminate 1 and 0 based indices Also we would like to recommend to use *speaking* names whenever possible, a method named common_at_put that just asserst the range but does not set anything is certainly NOT speaking (lying if you want) Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 00:34:25 2007 @@ -116,15 +116,16 @@ def _sendSelfSelector(self, selector, argcount, interp): receiver = self.peek(argcount) self._sendSelector(selector, argcount, interp, - receiver, receiver.shadow_of_my_class()) + receiver, receiver.shadow_of_my_class(), 1) def _sendSuperSelector(self, selector, argcount, interp): s_compiledin = self.w_method().compiledin().as_class_get_shadow() self._sendSelector(selector, argcount, interp, self.w_receiver, - s_compiledin.s_superclass) + s_compiledin.s_superclass, 0) def _sendSelector(self, selector, argcount, interp, - receiver, receiverclassshadow): + receiver, receiverclassshadow, popreceiver): + assert argcount >= 0 method = receiverclassshadow.lookup(selector) # XXX catch MethodNotFound here and send doesNotUnderstand: if method.primitive: @@ -140,8 +141,8 @@ start = len(self.stack) - argcount assert start >= 0 # XXX check in the Blue Book what to do in this case arguments = self.stack[start:] - interp.w_active_context = method.create_frame(receiver, arguments, self) - self.pop_n(argcount + 1) + interp.w_active_context = method.create_frame(receiver, arguments, self) + self.pop_n(argcount + popreceiver) def _return(self, object, interp): if self.w_sender is None: # for tests, when returning from the top-level context Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Fri Oct 26 00:34:25 2007 @@ -116,13 +116,13 @@ W_AbstractObjectWithClassReference.__init__(self, w_class) self._vars = [None] * size - def fetch(self, index): - return self._vars[index] + def fetch(self, n0): + return self._vars[n0] - def store(self, index, w_value): + def store(self, n0, w_value): if self._shadow is not None: self._shadow.invalidate() - self._vars[index] = w_value + self._vars[n0] = w_value def size(self): return len(self._vars) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 00:34:25 2007 @@ -25,12 +25,12 @@ return objtable.wrap_int(w_obj.getbyte(idx)) raise PrimitiveFailedError() -def assert_bounds(idx, minb, maxb): - if idx < minb or idx >= maxb: +def assert_bounds(n0, minimum, maximum): + if not minimum <= n0 < maximum: raise PrimitiveFailedError() -def assert_valid_index(idx, w_obj): - assert_bounds(idx, 0, w_obj.size()) +def assert_valid_index(n0, w_obj): + assert_bounds(n0, 0, w_obj.size()) # ___________________________________________________________________________ # Primitive table: it is filled in at initialization time with the @@ -300,15 +300,13 @@ STRING_AT = 63 STRING_AT_PUT = 64 -def common_at((w_obj, w_idx)): - idx = unwrap_int(w_idx) - # XXX should be idx-1, probably - assert_valid_index(idx-1, w_obj) - return w_obj, idx-1 +def common_at((w_obj, w_index1)): + index1 = unwrap_int(w_index1) + assert_valid_index(index1-1, w_obj) + return w_obj, index1-1 def common_at_put((w_obj, w_idx, w_val)): idx = unwrap_int(w_idx) - # XXX should be idx-1, probably assert_valid_index(idx-1, w_obj) return w_obj, idx-1, w_val @@ -366,19 +364,17 @@ @primitive(OBJECT_AT) @stack(2) -def func(args, (w_rcvr, w_idx)): - idx = unwrap_int(w_idx) - # XXX should be idx-1, probably - assert_bounds(idx, 0, w_rcvr.shadow_of_my_class().instance_size) - return w_rcvr.fetch(idx) +def func(args, (w_rcvr, w_n1)): + n0 = unwrap_int(w_n1) - 1 + assert_bounds(n0, 0, w_rcvr.shadow_of_my_class().instance_size) + return w_rcvr.fetch(n0) @primitive(OBJECT_AT_PUT) @stack(3) -def func(args, (w_rcvr, w_idx, w_val)): - idx = unwrap_int(w_idx) - # XXX should be idx-1, probably - assert_bounds(idx, 0, w_rcvr.shadow_of_my_class().instance_size) - w_rcvr.store(idx, w_val) +def func(args, (w_rcvr, w_n1, w_val)): + n0 = unwrap_int(w_n1) - 1 + assert_bounds(n0, 0, w_rcvr.shadow_of_my_class().instance_size) + w_rcvr.store(n0, w_val) return w_val @primitive(NEW) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 00:34:25 2007 @@ -169,53 +169,53 @@ def test_at(): w_obj = mockclass(0, varsized=True).as_class_get_shadow().new(1) w_obj.store(0, "foo") - assert prim(p.AT, [w_obj, 0]) == "foo" + assert prim(p.AT, [w_obj, 1]) == "foo" def test_invalid_at(): w_obj = mockclass(0).as_class_get_shadow().new() - prim_fails(p.AT, [w_obj, 0]) + prim_fails(p.AT, [w_obj, 1]) def test_at_put(): w_obj = mockclass(0, varsized=1).as_class_get_shadow().new(1) - assert prim(p.AT_PUT, [w_obj, 0, 22]).value == 22 - assert prim(p.AT, [w_obj, 0]).value == 22 + assert prim(p.AT_PUT, [w_obj, 1, 22]).value == 22 + assert prim(p.AT, [w_obj, 1]).value == 22 def test_invalid_at_put(): w_obj = mockclass(0).as_class_get_shadow().new() - prim_fails(p.AT_PUT, [w_obj, 0, 22]) + prim_fails(p.AT_PUT, [w_obj, 1, 22]) def test_string_at(): - assert prim(p.STRING_AT, ["foobar", 3]) == wrap("b") + assert prim(p.STRING_AT, ["foobar", 4]) == wrap("b") def test_string_at_put(): test_str = wrap("foobar") - assert prim(p.STRING_AT_PUT, [test_str, 3, "c"]) == wrap("c") + assert prim(p.STRING_AT_PUT, [test_str, 4, "c"]) == wrap("c") exp = "foocar" for i in range(len(exp)): assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i]) def test_object_at(): - w_v = prim(p.OBJECT_AT, ["q", objtable.CHARACTER_VALUE_INDEX]) + w_v = prim(p.OBJECT_AT, ["q", objtable.CHARACTER_VALUE_INDEX+1]) assert w_v.value == ord("q") def test_invalid_object_at(): - prim_fails(p.OBJECT_AT, ["q", objtable.CHARACTER_VALUE_INDEX+1]) + prim_fails(p.OBJECT_AT, ["q", objtable.CHARACTER_VALUE_INDEX+2]) def test_object_at_put(): w_obj = mockclass(1).as_class_get_shadow().new() - assert prim(p.OBJECT_AT_PUT, [w_obj, 0, "q"]) is wrap("q") - assert prim(p.OBJECT_AT, [w_obj, 0]) is wrap("q") + assert prim(p.OBJECT_AT_PUT, [w_obj, 1, "q"]) is wrap("q") + assert prim(p.OBJECT_AT, [w_obj, 1]) is wrap("q") def test_invalid_object_at_put(): w_obj = mockclass(1).as_class_get_shadow().new() - prim_fails(p.OBJECT_AT, [w_obj, 1, 1]) + prim_fails(p.OBJECT_AT, [w_obj, 2, 42]) def test_string_at_put(): test_str = wrap("foobar") - assert prim(p.STRING_AT_PUT, [test_str, 3, "c"]) == wrap("c") + assert prim(p.STRING_AT_PUT, [test_str, 4, "c"]) == wrap("c") exp = "foocar" - for i in range(len(exp)): - assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i]) + for i in range(1,len(exp)+1): + assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i-1]) def test_new(): w_Object = classtable.classtable['w_Object'] @@ -243,6 +243,7 @@ assert w_v.value == ord("b") def test_as_oop(): + py.test.skip("not yet clear what AS_OOP returns: hash or header?") w_obj = mockclass(0).as_class_get_shadow().new() w_obj.w_hash = wrap(22) assert prim(p.AS_OOP, [w_obj]).value == 22 From akuhn at codespeak.net Fri Oct 26 00:38:31 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Fri, 26 Oct 2007 00:38:31 +0200 (CEST) Subject: [pypy-svn] r48008 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025223831.2AE388136@code0.codespeak.net> Author: akuhn Date: Fri Oct 26 00:38:30 2007 New Revision: 48008 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: just added one comment Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 00:38:30 2007 @@ -17,6 +17,7 @@ raise PrimitiveFailedError() def subscript(idx, w_obj): + # XXX what does this do? explain if isinstance(w_obj, model.W_PointersObject): return w_obj.fetch(idx) elif isinstance(w_obj, model.W_WordsObject): From cfbolz at codespeak.net Fri Oct 26 00:49:24 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 26 Oct 2007 00:49:24 +0200 (CEST) Subject: [pypy-svn] r48009 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025224924.528538134@code0.codespeak.net> Author: cfbolz Date: Fri Oct 26 00:49:24 2007 New Revision: 48009 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (arigo, cfbolz): puh. refactor the way primitives are registered to reduce the amount of boilerplate, have more security checks and various fixes to primitives along the way. annoying but worthwhile. note: please write more tests, half of this it is not tested. I am looking at you, primitive people. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 00:49:24 2007 @@ -131,7 +131,8 @@ if method.primitive: func = primitives.prim_table[method.primitive] try: - w_result = func(primitives.Args(interp, argcount)) + # add +1 to account for the receiver + w_result = func(interp, argcount + 1) except primitives.PrimitiveFailedError: pass # ignore this error and fall back to the Smalltalk version else: @@ -294,8 +295,8 @@ def callPrimitiveAndPush(self, primitive, selector, argcount, interp): try: - args = primitives.Args(interp, argcount) - self.push(primitives.prim_table[primitive](args)) + # add one to the argcount to account for the self + self.push(primitives.prim_table[primitive](interp, argcount + 1)) except primitives.PrimitiveFailedError: self._sendSelfSelector(selector, argcount, interp) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 00:49:24 2007 @@ -1,5 +1,6 @@ -import operator +import inspect import math +import operator from pypy.lang.smalltalk import model, shadow from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import objtable @@ -35,41 +36,54 @@ # ___________________________________________________________________________ # Primitive table: it is filled in at initialization time with the -# primitive functions. Each primitive function takes a single -# argument, an instance of the Args class below; the function either +# primitive functions. Each primitive function takes two +# arguments, an interp and an argument_count # completes, and returns a result, or throws a PrimitiveFailedError. -def raise_failing_default(args): - raise PrimitiveFailedError +def raise_failing_default(interp, argument_count): + raise PrimitiveFailedError # Squeak has primitives all the way up to 575 # So all optional primitives will default to the bytecode implementation prim_table = [raise_failing_default] * 576 -class Args: - def __init__(self, interp, argument_count): - self.interp = interp - self.argument_count = argument_count - -def primitive(code): +def expose_primitive(code, unwrap_spec=None): + from pypy.rlib.unroll import unrolling_iterable def decorator(func): assert code not in prim_table - prim_table[code] = func - return func - return decorator - -def stack(n): - def decorator(wrapped): - def result(args): - frame = args.interp.w_active_context - start = len(frame.stack) - n - if start < 0: - raise PrimitiveFailedError() # not enough arguments - items = frame.stack[start:] - res = wrapped(args, items) - frame.pop_n(n) # only if no exception occurs! + if unwrap_spec is None: + prim_table[code] = func + return func + for spec in unwrap_spec: + assert spec in (int, float, object) + len_unwrap_spec = len(unwrap_spec) + assert (len_unwrap_spec == len(inspect.getargspec(func)[0]) + 1, + "wrong number of arguments") + unrolling_unwrap_spec = unrolling_iterable(enumerate(unwrap_spec)) + def wrapped(interp, argument_count): + frame = interp.w_active_context + assert argument_count == len_unwrap_spec + if len(frame.stack) < len_unwrap_spec: + raise PrimitiveFailedError() + args = () + for i, spec in unrolling_unwrap_spec: + index = -len_unwrap_spec + i + arg = frame.stack[index] + if spec is int: + args += (unwrap_int(arg), ) + elif spec is float: + args += (unwrap_float(arg), ) + elif spec is object: + args += (arg, ) + else: + assert 0, "this should never happen" + res = func(interp, *args) + frame.pop_n(len_unwrap_spec) # only if no exception occurs! return res - return result + + wrapped.func_name = func.func_name + prim_table[code] = wrapped + return func return decorator # ___________________________________________________________________________ @@ -106,11 +120,8 @@ } for (code,op) in math_ops.items(): def make_func(op): - @primitive(code) - @stack(2) - def func(args, (w_receiver, w_argument)): - receiver = unwrap_int(w_receiver) - argument = unwrap_int(w_argument) + @expose_primitive(code, unwrap_spec=[int, int]) + def func(interp, receiver, argument): try: res = rarithmetic.ovfcheck(op(receiver, argument)) except OverflowError: @@ -125,21 +136,15 @@ } for (code,op) in bitwise_binary_ops.items(): def make_func(op): - @primitive(code) - @stack(2) - def func(args, (w_receiver, w_argument)): - receiver = unwrap_int(w_receiver) - argument = unwrap_int(w_argument) + @expose_primitive(code, unwrap_spec=[int, int]) + def func(interp, receiver, argument): res = op(receiver, argument) return wrap_int(res) make_func(op) # #/ -- return the result of a division, only succeed if the division is exact - at primitive(DIVIDE) - at stack(2) -def func(args, (w_receiver, w_argument)): - receiver = unwrap_int(w_receiver) - argument = unwrap_int(w_argument) + at expose_primitive(DIVIDE, unwrap_spec=[int, int]) +def func(interp, receiver, argument): if argument == 0: raise PrimitiveFailedError() if receiver % argument != 0: @@ -147,45 +152,29 @@ return wrap_int(receiver // argument) # #\\ -- return the remainder of a division - at primitive(MOD) - at stack(2) -def func(args, (w_receiver, w_argument)): - receiver = unwrap_int(w_receiver) - argument = unwrap_int(w_argument) + at expose_primitive(MOD, unwrap_spec=[int, int]) +def func(interp, receiver, argument): if argument == 0: raise PrimitiveFailedError() return wrap_int(receiver % argument) # #// -- return the result of a division, rounded towards negative zero - at primitive(DIV) - at stack(2) -def func(args, (w_receiver, w_argument)): - receiver = unwrap_int(w_receiver) - argument = unwrap_int(w_argument) + at expose_primitive(DIV, unwrap_spec=[int, int]) +def func(interp, receiver, argument): if argument == 0: raise PrimitiveFailedError() return wrap_int(receiver // argument) # #// -- return the result of a division, rounded towards negative infinity - at primitive(QUO) - at stack(2) -def func(args, (w_receiver, w_argument)): - receiver = unwrap_int(w_receiver) - argument = unwrap_int(w_argument) + at expose_primitive(QUO, unwrap_spec=[int, int]) +def func(interp, receiver, argument): if argument == 0: raise PrimitiveFailedError() return wrap_int(receiver // argument) # #bitShift: -- return the shifted value - at primitive(BIT_SHIFT) - at stack(2) -def func(args, (w_receiver, w_argument)): - receiver = unwrap_int(w_receiver) - argument = unwrap_int(w_argument) - - # heh, no shifting at all - if argument == 0: - return w_receiver + at expose_primitive(BIT_SHIFT, unwrap_spec=[int, int]) +def func(interp, receiver, argument): # left shift, must fail if we loose bits beyond 32 if argument > 0: @@ -225,57 +214,41 @@ } for (code,op) in math_ops.items(): def make_func(op): - @primitive(code) - @stack(2) - def func(args, (w_v1, w_v2)): - v1 = unwrap_float(w_v1) - v2 = unwrap_float(w_v2) + @expose_primitive(code, unwrap_spec=[float, float]) + def func(interp, v1, v2): w_res = objtable.wrap_float(op(v1, v2)) return w_res make_func(op) - at primitive(FLOAT_TRUNCATED) - at stack(1) -def func(args, (w_float,)): - f = unwrap_float(w_float) + at expose_primitive(FLOAT_TRUNCATED, unwrap_spec=[float]) +def func(interp, f): w_res = objtable.wrap_int(int(f)) return w_res - at primitive(FLOAT_TIMES_TWO_POWER) - at stack(2) -def func(args, (w_rcvr,w_arg,)): - rcvr = unwrap_float(w_rcvr) - arg = unwrap_int(w_arg) + at expose_primitive(FLOAT_TIMES_TWO_POWER, unwrap_spec=[float, float]) +def func(interp, rcvr, arg): w_res = objtable.wrap_float(math.ldexp(rcvr,arg)) return w_res - at primitive(FLOAT_SQUARE_ROOT) - at stack(1) -def func(args, (w_float,)): - f = unwrap_float(w_float) + at expose_primitive(FLOAT_SQUARE_ROOT, unwrap_spec=[float]) +def func(interp, f): if f < 0.0: raise PrimitiveFailedError w_res = objtable.wrap_float(math.sqrt(f)) return w_res - at primitive(FLOAT_SIN) - at stack(1) -def func(args, (w_float,)): - f = unwrap_float(w_float) + at expose_primitive(FLOAT_SIN, unwrap_spec=[float]) +def func(interp, f): w_res = objtable.wrap_float(math.sin(f)) return w_res - at primitive(FLOAT_ARCTAN) - at stack(1) -def func(args, (w_float,)): - f = unwrap_float(w_float) + at expose_primitive(FLOAT_ARCTAN, unwrap_spec=[float]) +def func(interp, f): w_res = objtable.wrap_float(math.atan(f)) return w_res - at primitive(FLOAT_LOG_N) - at stack(1) -def func(args, (w_float,)): - f = unwrap_float(w_float) + at expose_primitive(FLOAT_LOG_N, unwrap_spec=[float]) +def func(interp, f): if f == 0: res = -rarithmetic.INFINITY elif f < 0: @@ -284,10 +257,8 @@ res = math.log(f) return objtable.wrap_float(res) - at primitive(FLOAT_EXP) - at stack(1) -def func(args, (w_float,)): - f = unwrap_float(w_float) + at expose_primitive(FLOAT_EXP, unwrap_spec=[float]) +def func(interp, f): w_res = objtable.wrap_float(math.exp(f)) return w_res @@ -301,47 +272,42 @@ STRING_AT = 63 STRING_AT_PUT = 64 -def common_at((w_obj, w_index1)): +def common_at(w_obj, w_index1): index1 = unwrap_int(w_index1) assert_valid_index(index1-1, w_obj) return w_obj, index1-1 -def common_at_put((w_obj, w_idx, w_val)): +def common_at_put(w_obj, w_idx, w_val): idx = unwrap_int(w_idx) assert_valid_index(idx-1, w_obj) return w_obj, idx-1, w_val - at primitive(AT) - at stack(2) -def func(args, stack): - [w_obj, idx] = common_at(stack) + at expose_primitive(AT, unwrap_spec=[object, object]) +def func(interp, w_obj, w_idx): + [w_obj, idx] = common_at(w_obj, w_idx) return w_obj.fetch(idx) - at primitive(AT_PUT) - at stack(3) -def func(args, stack): - [w_obj, idx, w_val] = common_at_put(stack) + at expose_primitive(AT_PUT, unwrap_spec=[object, object, object]) +def func(interp, w_obj, w_idx, w_val): + [w_obj, idx, w_val] = common_at_put(w_obj, w_idx, w_val) w_obj.store(idx, w_val) return w_val - at primitive(SIZE) - at stack(1) -def func(args, (w_obj,)): + at expose_primitive(SIZE, unwrap_spec=[object]) +def func(interp, w_obj): if not w_obj.shadow_of_my_class().isvariable(): raise PrimitiveFailedError() return wrap_int(w_obj.size()) - at primitive(STRING_AT) - at stack(2) -def func(args, stack): - w_obj, idx = common_at(stack) + at expose_primitive(STRING_AT, unwrap_spec=[object, object]) +def func(interp, w_obj, w_idx): + w_obj, idx = common_at(w_obj, w_idx) byte = w_obj.getbyte(idx) return objtable.CharacterTable[byte] - at primitive(STRING_AT_PUT) - at stack(3) -def func(args, stack): - w_obj, idx, w_val = common_at_put(stack) + at expose_primitive(STRING_AT_PUT, unwrap_spec=[object, object, object]) +def func(interp, w_obj, w_idx, w_val): + w_obj, idx, w_val = common_at_put(w_obj, w_idx, w_val) if w_val.getclass() is not classtable.w_Character: raise PrimitiveFailedError() w_obj.setbyte(idx, objtable.ord_w_char(w_val)) @@ -363,50 +329,44 @@ NEXT_INSTANCE = 78 NEW_METHOD = 79 - at primitive(OBJECT_AT) - at stack(2) -def func(args, (w_rcvr, w_n1)): - n0 = unwrap_int(w_n1) - 1 + at expose_primitive(OBJECT_AT, unwrap_spec=[object, int]) +def func(interp, w_rcvr, n1): + n0 = n1 - 1 assert_bounds(n0, 0, w_rcvr.shadow_of_my_class().instance_size) return w_rcvr.fetch(n0) - at primitive(OBJECT_AT_PUT) - at stack(3) -def func(args, (w_rcvr, w_n1, w_val)): - n0 = unwrap_int(w_n1) - 1 + at expose_primitive(OBJECT_AT_PUT, unwrap_spec=[object, int, object]) +def func(interp, w_rcvr, n1, w_val): + n0 = n1 - 1 assert_bounds(n0, 0, w_rcvr.shadow_of_my_class().instance_size) w_rcvr.store(n0, w_val) return w_val - at primitive(NEW) - at stack(1) -def func(args, (w_cls,)): + at expose_primitive(NEW, unwrap_spec=[object]) +def func(interp, w_cls): shadow = w_cls.as_class_get_shadow() if shadow.isvariable(): raise PrimitiveFailedError() return shadow.new() - at primitive(NEW_WITH_ARG) - at stack(2) -def func(args, (w_cls, w_size)): + at expose_primitive(NEW_WITH_ARG, unwrap_spec=[object, object]) +def func(interp, w_cls, w_size): shadow = w_cls.as_class_get_shadow() if not shadow.isvariable(): raise PrimitiveFailedError() size = unwrap_int(w_size) return shadow.new(size) - at primitive(ARRAY_BECOME_ONE_WAY) -def func(args): + at expose_primitive(ARRAY_BECOME_ONE_WAY, unwrap_spec=[]) +def func(interp): raise PrimitiveNotYetWrittenError - at primitive(INST_VAR_AT) - at stack(2) -def func(args, (w_rcvr, w_idx)): + at expose_primitive(INST_VAR_AT, unwrap_spec=[object, int]) +def func(interp, w_rcvr, idx): # I *think* this is the correct behavior, but I'm not quite sure. # Might be restricted to fixed length fields? # XXX this doesn't look correct. Our guess is that INST_VAR_AT # is used to access *only* the fixed length fields. - idx = unwrap_int(w_idx) shadow = w_rcvr.shadow_of_my_class() if idx < 0: raise PrimitiveFailedError() @@ -417,41 +377,37 @@ return subscript(idx, w_rcvr) raise PrimitiveFailedError() - at primitive(INST_VAR_AT_PUT) -def func(args): + at expose_primitive(INST_VAR_AT_PUT, unwrap_spec=[]) +def func(interp): raise PrimitiveNotYetWrittenError() - at primitive(AS_OOP) - at stack(1) -def func(args, (w_rcvr,)): + at expose_primitive(AS_OOP, unwrap_spec=[object]) +def func(interp, w_rcvr): if isinstance(w_rcvr, model.W_SmallInteger): raise PrimitiveFailedError() return wrap_int(w_rcvr.gethash()) - at primitive(STORE_STACKP) - at stack(2) -def func(args, stack): + at expose_primitive(STORE_STACKP, unwrap_spec=[object, object]) +def func(interp, w_obj1, w_obj2): # This primitive seems to resize the stack. I don't think this is # really relevant in our implementation. raise PrimitiveNotYetWrittenError() - at primitive(SOME_INSTANCE) - at stack(1) -def func(args, (w_class,)): + at expose_primitive(SOME_INSTANCE, unwrap_spec=[object]) +def func(interp, w_class): # This primitive returns some instance of the class on the stack. # Not sure quite how to do this; maintain a weak list of all # existing instances or something? raise PrimitiveNotYetWrittenError() - at primitive(NEXT_INSTANCE) - at stack(1) -def func(args, (w_obj,)): + at expose_primitive(NEXT_INSTANCE, unwrap_spec=[object]) +def func(interp, w_obj): # This primitive is used to iterate through all instances of a class: # it returns the "next" instance after w_obj. raise PrimitiveNotYetWrittenError() - at primitive(NEW_METHOD) -def func(args): + at expose_primitive(NEW_METHOD, unwrap_spec=[]) +def func(interp): raise PrimitiveNotYetWrittenError() # ___________________________________________________________________________ @@ -464,40 +420,38 @@ EXIT_TO_DEBUGGER = 114 CHANGE_CLASS = 115 # Blue Book: primitiveOopsLeft - at primitive(EQUIVALENT) - at stack(2) -def func(args, (w_arg, w_rcvr)): + at expose_primitive(EQUIVALENT, unwrap_spec=[object, object]) +def func(interp, w_arg, w_rcvr): # XXX this is bogus in the presence of (our implementation of) become, # as we might plan to implement become by copying all fields from one # object to the other return objtable.wrap_bool(w_arg is w_rcvr) - at primitive(CLASS) - at stack(1) -def func(args, (w_obj,)): + at expose_primitive(CLASS, unwrap_spec=[object]) +def func(interp, w_obj): return w_obj.getclass() - at primitive(BYTES_LEFT) -def func(args): + at expose_primitive(BYTES_LEFT, unwrap_spec=[]) +def func(interp): raise PrimitiveNotYetWrittenError() - at primitive(QUIT) -def func(args): + at expose_primitive(QUIT, unwrap_spec=[]) +def func(interp): raise PrimitiveNotYetWrittenError() - at primitive(EXIT_TO_DEBUGGER) -def func(args): + at expose_primitive(EXIT_TO_DEBUGGER, unwrap_spec=[]) +def func(interp): raise PrimitiveNotYetWrittenError() - at primitive(CHANGE_CLASS) - at stack(2) -def func(args, (w_arg, w_rcvr)): + at expose_primitive(CHANGE_CLASS, unwrap_spec=[object, object]) +def func(interp, w_arg, w_rcvr): w_arg_class = w_arg.getclass() w_rcvr_class = w_rcvr.getclass() # We should fail if: # 1. Rcvr or arg are SmallIntegers + # XXX this is wrong too if (w_arg_class == classtable.w_SmallInteger or w_rcvr_class == classtable.w_SmallInteger): raise PrimitiveFailedError() @@ -526,10 +480,9 @@ def fake_bytes_left(): return wrap_int(2**20) # XXX we don't know how to do this :-( - at primitive(INC_GC) # XXX the same for now - at primitive(FULL_GC) - at stack(1) # Squeak pops the arg and ignores it ... go figure -def func(args, (w_arg,)): + at expose_primitive(INC_GC, unwrap_spec=[object]) + at expose_primitive(FULL_GC, unwrap_spec=[object]) +def func(interp, w_arg): # Squeak pops the arg and ignores it ... go figure from pypy.rlib import rgc rgc.collect() return fake_bytes_left() @@ -561,11 +514,8 @@ } for (code,op) in bool_ops.items(): def make_func(op): - @primitive(code) - @stack(2) - def func(args, (w_v1, w_v2)): - v1 = unwrap_int(w_v1) - v2 = unwrap_int(w_v2) + @expose_primitive(code, unwrap_spec=[int, int]) + def func(interp, v1, v2): res = op(v1, v2) w_res = objtable.wrap_bool(res) return w_res @@ -573,11 +523,8 @@ for (code,op) in bool_ops.items(): def make_func(op): - @primitive(code+_FLOAT_OFFSET) - @stack(2) - def func(args, (w_v1, w_v2)): - v1 = unwrap_float(w_v1) - v2 = unwrap_float(w_v2) + @expose_primitive(code+_FLOAT_OFFSET, unwrap_spec=[float, float]) + def func(interp, v1, v2): res = op(v1, v2) w_res = objtable.wrap_bool(res) return w_res @@ -595,29 +542,25 @@ PUSH_ONE = 262 PUSH_TWO = 263 - at primitive(PUSH_SELF) - at stack(1) -def func(args, stack): - [w_self] = stack + at expose_primitive(PUSH_SELF, unwrap_spec=[object]) +def func(interp, w_self): + # no-op really return w_self -def define_const_primitives(): - for (code, const) in [ - (PUSH_TRUE, objtable.w_true), - (PUSH_FALSE, objtable.w_false), - (PUSH_NIL, objtable.w_nil), - (PUSH_MINUS_ONE, objtable.w_mone), - (PUSH_ZERO, objtable.w_zero), - (PUSH_ONE, objtable.w_one), - (PUSH_TWO, objtable.w_two), - ]: - def make_func(const): - @primitive(code) - @stack(1) - def func(args, stack): - return const - make_func(const) -define_const_primitives() +for (code, const) in [ + (PUSH_TRUE, objtable.w_true), + (PUSH_FALSE, objtable.w_false), + (PUSH_NIL, objtable.w_nil), + (PUSH_MINUS_ONE, objtable.w_mone), + (PUSH_ZERO, objtable.w_zero), + (PUSH_ONE, objtable.w_one), + (PUSH_TWO, objtable.w_two), + ]: + def make_func(const): + @expose_primitive(code, unwrap_spec=[object]) + def func(interp, w_ignored): + return const + make_func(const) # ___________________________________________________________________________ # Control Primitives @@ -633,10 +576,9 @@ PRIMITIVE_SUSPEND = 88 PRIMITIVE_FLUSH_CACHE = 89 - at primitive(PRIMITIVE_BLOCK_COPY) - at stack(2) -def func(args, (w_context, w_argcnt)): - frame = args.interp.w_active_context + at expose_primitive(PRIMITIVE_BLOCK_COPY, unwrap_spec=[object, object]) +def func(interp, w_context, w_argcnt): + frame = interp.w_active_context argcnt = unwrap_int(w_argcnt) # From B.B.: If receiver is a MethodContext, then it becomes @@ -654,23 +596,23 @@ w_method_context, objtable.w_nil, argcnt, initialip) return w_new_context - at primitive(PRIMITIVE_VALUE) -def func(args): - - # If nargs == 4, stack looks like: + at expose_primitive(PRIMITIVE_VALUE) +def func(interp, argument_count): + # XXX XXX XXX XXX test me + # If argument_count == 4, stack looks like: # 3 2 1 0 # Rcvr | Arg 0 | Arg1 | Arg 2 # - frame = args.interp.w_active_context + frame = interp.w_active_context # Validate that we have a block on the stack and that it received # the proper number of arguments: - w_block_ctx = frame.peek(args.argument_count-1) + w_block_ctx = frame.peek(argument_count-1) if not isinstance(w_block_ctx, model.W_BlockContext): raise PrimitiveFailedError() exp_arg_cnt = w_block_ctx.expected_argument_count() - if args.argument_count != exp_arg_cnt: + if argument_count != exp_arg_cnt + 1: # exp_arg_cnt doesn't count self raise PrimitiveFailedError() # Initialize the block stack from the contents of the stack: @@ -681,45 +623,38 @@ # Set some fields w_block_ctx.pc = w_block_ctx.initialip w_block_ctx.w_sender = frame - args.interp.w_active_context = w_block_ctx + interp.w_active_context = w_block_ctx - at primitive(PRIMITIVE_VALUE_WITH_ARGS) - at stack(2) -def func(args, (w_rcvr, w_args)): + at expose_primitive(PRIMITIVE_VALUE_WITH_ARGS, unwrap_spec=[object, object]) +def func(interp, w_rcvr, w_args): raise PrimitiveNotYetWrittenError() - at primitive(PRIMITIVE_PERFORM) - at stack(2) -def func(args, (w_rcvr, w_sel)): + at expose_primitive(PRIMITIVE_PERFORM, unwrap_spec=[object, object]) +def func(interp, w_rcvr, w_sel): # XXX we can implement this when lookup on shadow class is done raise PrimitiveNotYetWrittenError() - at primitive(PRIMITIVE_PERFORM_WITH_ARGS) - at stack(3) -def func(args, (w_rcvr, w_sel, w_args)): + at expose_primitive(PRIMITIVE_PERFORM_WITH_ARGS, + unwrap_spec=[object, object, object]) +def func(interp, w_rcvr, w_sel, w_args): raise PrimitiveNotYetWrittenError() - at primitive(PRIMITIVE_SIGNAL) - at stack(1) -def func(args, (w_rcvr,)): + at expose_primitive(PRIMITIVE_SIGNAL, unwrap_spec=[object]) +def func(interp, w_rcvr): raise PrimitiveNotYetWrittenError() - at primitive(PRIMITIVE_WAIT) - at stack(1) -def func(args, (w_rcvr,)): + at expose_primitive(PRIMITIVE_WAIT, unwrap_spec=[object]) +def func(interp, w_rcvr): raise PrimitiveNotYetWrittenError() - at primitive(PRIMITIVE_RESUME) - at stack(1) -def func(args, (w_rcvr,)): + at expose_primitive(PRIMITIVE_RESUME, unwrap_spec=[object]) +def func(interp, w_rcvr,): raise PrimitiveNotYetWrittenError() - at primitive(PRIMITIVE_SUSPEND) - at stack(1) -def func(args, (w_rcvr,)): + at expose_primitive(PRIMITIVE_SUSPEND, unwrap_spec=[object]) +def func(interp, w_rcvr): raise PrimitiveNotYetWrittenError() - at primitive(PRIMITIVE_FLUSH_CACHE) - at stack(1) -def func(args, (w_rcvr,)): + at expose_primitive(PRIMITIVE_FLUSH_CACHE, unwrap_spec=[object]) +def func(interp, w_rcvr): raise PrimitiveNotYetWrittenError() Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 00:49:24 2007 @@ -8,7 +8,7 @@ from pypy.rlib.rarithmetic import INFINITY, NAN, isinf, isnan # Violates the guideline, but we use it A LOT to reference the primitive codes: -import pypy.lang.smalltalk.primitives as p +from pypy.lang.smalltalk import primitives mockclass = classtable.bootstrap_class @@ -29,257 +29,257 @@ frame = MockFrame(mapped_stack) interp = interpreter.Interpreter() interp.w_active_context = frame - return p.Args(interp, len(stack)) + return (interp, len(stack)) def prim(code, stack): - args = mock(stack) - res = prim_table[code](args) - assert not len(args.interp.w_active_context.stack) # check args are consumed + interp, argument_count = mock(stack) + res = prim_table[code](interp, argument_count) + assert not len(interp.w_active_context.stack) # check args are consumed return res def prim_fails(code, stack): - args = mock(stack) - orig_stack = list(args.interp.w_active_context.stack) + interp, argument_count = mock(stack) + orig_stack = list(interp.w_active_context.stack) try: - prim_table[code](args) + prim_table[code](interp, argument_count) py.test.fail("Expected PrimitiveFailedError") except PrimitiveFailedError: - assert args.interp.w_active_context.stack == orig_stack + assert interp.w_active_context.stack == orig_stack # smallinteger tests def test_small_int_add(): - assert prim(p.ADD, [1,2]).value == 3 - assert prim(p.ADD, [3,4]).value == 7 + assert prim(primitives.ADD, [1,2]).value == 3 + assert prim(primitives.ADD, [3,4]).value == 7 def test_small_int_add_fail(): - prim_fails(p.ADD, [1073741823,2]) + prim_fails(primitives.ADD, [1073741823,2]) def test_small_int_minus(): - assert prim(p.SUBTRACT, [5,9]).value == -4 + assert prim(primitives.SUBTRACT, [5,9]).value == -4 def test_small_int_minus_fail(): - prim_fails(p.SUBTRACT, [-1073741823,2]) + prim_fails(primitives.SUBTRACT, [-1073741823,2]) def test_small_int_divide(): - assert prim(p.DIVIDE, [6,3]).value == 2 + assert prim(primitives.DIVIDE, [6,3]).value == 2 def test_small_int_divide_fail(): - prim_fails(p.DIVIDE, [12, 0]) - prim_fails(p.DIVIDE, [12, 7]) + prim_fails(primitives.DIVIDE, [12, 0]) + prim_fails(primitives.DIVIDE, [12, 7]) def test_small_int_mod(): - assert prim(p.MOD, [12,7]).value == 5 + assert prim(primitives.MOD, [12,7]).value == 5 def test_small_int_mod_fail(): - prim_fails(p.MOD, [12, 0]) + prim_fails(primitives.MOD, [12, 0]) def test_small_int_div(): - assert prim(p.DIV, [12,3]).value == 4 - assert prim(p.DIV, [12,7]).value == 1 + assert prim(primitives.DIV, [12,3]).value == 4 + assert prim(primitives.DIV, [12,7]).value == 1 def test_small_int_div_fail(): - prim_fails(p.DIV, [12, 0]) + prim_fails(primitives.DIV, [12, 0]) def test_small_int_quo(): - assert prim(p.QUO, [12,3]).value == 4 - assert prim(p.QUO, [12,7]).value == 1 + assert prim(primitives.QUO, [12,3]).value == 4 + assert prim(primitives.QUO, [12,7]).value == 1 def test_small_int_quo_fail(): - prim_fails(p.QUO, [12, 0]) + prim_fails(primitives.QUO, [12, 0]) def test_small_int_bit_and(): - assert prim(p.BIT_AND, [2, 4]).value == 0 - assert prim(p.BIT_AND, [2, 3]).value == 2 - assert prim(p.BIT_AND, [3, 4]).value == 0 - assert prim(p.BIT_AND, [4, 4]).value == 4 + assert prim(primitives.BIT_AND, [2, 4]).value == 0 + assert prim(primitives.BIT_AND, [2, 3]).value == 2 + assert prim(primitives.BIT_AND, [3, 4]).value == 0 + assert prim(primitives.BIT_AND, [4, 4]).value == 4 def test_small_int_bit_or(): - assert prim(p.BIT_OR, [2, 4]).value == 6 - assert prim(p.BIT_OR, [2, 3]).value == 3 - assert prim(p.BIT_OR, [3, 4]).value == 7 - assert prim(p.BIT_OR, [4, 4]).value == 4 + assert prim(primitives.BIT_OR, [2, 4]).value == 6 + assert prim(primitives.BIT_OR, [2, 3]).value == 3 + assert prim(primitives.BIT_OR, [3, 4]).value == 7 + assert prim(primitives.BIT_OR, [4, 4]).value == 4 def test_small_int_bit_xor(): - assert prim(p.BIT_XOR, [2, 4]).value == 6 - assert prim(p.BIT_XOR, [2, 3]).value == 1 - assert prim(p.BIT_XOR, [3, 4]).value == 7 - assert prim(p.BIT_XOR, [4, 4]).value == 0 + assert prim(primitives.BIT_XOR, [2, 4]).value == 6 + assert prim(primitives.BIT_XOR, [2, 3]).value == 1 + assert prim(primitives.BIT_XOR, [3, 4]).value == 7 + assert prim(primitives.BIT_XOR, [4, 4]).value == 0 def test_small_int_bit_shift(): - assert prim(p.BIT_SHIFT, [0, -3]).value == 0 - assert prim(p.BIT_SHIFT, [0, -2]).value == 0 - assert prim(p.BIT_SHIFT, [0, -1]).value == 0 - assert prim(p.BIT_SHIFT, [0, 0]).value == 0 - assert prim(p.BIT_SHIFT, [0, 1]).value == 0 - assert prim(p.BIT_SHIFT, [0, 2]).value == 0 - assert prim(p.BIT_SHIFT, [0, 3]).value == 0 + assert prim(primitives.BIT_SHIFT, [0, -3]).value == 0 + assert prim(primitives.BIT_SHIFT, [0, -2]).value == 0 + assert prim(primitives.BIT_SHIFT, [0, -1]).value == 0 + assert prim(primitives.BIT_SHIFT, [0, 0]).value == 0 + assert prim(primitives.BIT_SHIFT, [0, 1]).value == 0 + assert prim(primitives.BIT_SHIFT, [0, 2]).value == 0 + assert prim(primitives.BIT_SHIFT, [0, 3]).value == 0 def test_small_int_bit_shift_positive(): - assert prim(p.BIT_SHIFT, [4, -3]).value == 0 - assert prim(p.BIT_SHIFT, [4, -2]).value == 1 - assert prim(p.BIT_SHIFT, [4, -1]).value == 2 - assert prim(p.BIT_SHIFT, [4, 0]).value == 4 - assert prim(p.BIT_SHIFT, [4, 1]).value == 8 - assert prim(p.BIT_SHIFT, [4, 2]).value == 16 - assert prim(p.BIT_SHIFT, [4, 3]).value == 32 - assert prim(p.BIT_SHIFT, [4, 27]).value == 536870912 + assert prim(primitives.BIT_SHIFT, [4, -3]).value == 0 + assert prim(primitives.BIT_SHIFT, [4, -2]).value == 1 + assert prim(primitives.BIT_SHIFT, [4, -1]).value == 2 + assert prim(primitives.BIT_SHIFT, [4, 0]).value == 4 + assert prim(primitives.BIT_SHIFT, [4, 1]).value == 8 + assert prim(primitives.BIT_SHIFT, [4, 2]).value == 16 + assert prim(primitives.BIT_SHIFT, [4, 3]).value == 32 + assert prim(primitives.BIT_SHIFT, [4, 27]).value == 536870912 def test_small_int_bit_shift_negative(): - assert prim(p.BIT_SHIFT, [-4, -3]).value == -1 - assert prim(p.BIT_SHIFT, [-4, -2]).value == -1 - assert prim(p.BIT_SHIFT, [-4, -1]).value == -2 - assert prim(p.BIT_SHIFT, [-4, 0]).value == -4 - assert prim(p.BIT_SHIFT, [-4, 1]).value == -8 - assert prim(p.BIT_SHIFT, [-4, 2]).value == -16 - assert prim(p.BIT_SHIFT, [-4, 3]).value == -32 - assert prim(p.BIT_SHIFT, [-4, 27]).value == -536870912 + assert prim(primitives.BIT_SHIFT, [-4, -3]).value == -1 + assert prim(primitives.BIT_SHIFT, [-4, -2]).value == -1 + assert prim(primitives.BIT_SHIFT, [-4, -1]).value == -2 + assert prim(primitives.BIT_SHIFT, [-4, 0]).value == -4 + assert prim(primitives.BIT_SHIFT, [-4, 1]).value == -8 + assert prim(primitives.BIT_SHIFT, [-4, 2]).value == -16 + assert prim(primitives.BIT_SHIFT, [-4, 3]).value == -32 + assert prim(primitives.BIT_SHIFT, [-4, 27]).value == -536870912 def test_small_int_bit_shift_fail(): - prim_fails(p.BIT_SHIFT, [4, 32]) - prim_fails(p.BIT_SHIFT, [4, 31]) - prim_fails(p.BIT_SHIFT, [4, 30]) - prim_fails(p.BIT_SHIFT, [4, 29]) - prim_fails(p.BIT_SHIFT, [4, 28]) + prim_fails(primitives.BIT_SHIFT, [4, 32]) + prim_fails(primitives.BIT_SHIFT, [4, 31]) + prim_fails(primitives.BIT_SHIFT, [4, 30]) + prim_fails(primitives.BIT_SHIFT, [4, 29]) + prim_fails(primitives.BIT_SHIFT, [4, 28]) def test_float_add(): - assert prim(p.FLOAT_ADD, [1.0,2.0]).value == 3.0 - assert prim(p.FLOAT_ADD, [3.0,4.5]).value == 7.5 + assert prim(primitives.FLOAT_ADD, [1.0,2.0]).value == 3.0 + assert prim(primitives.FLOAT_ADD, [3.0,4.5]).value == 7.5 def test_float_subtract(): - assert prim(p.FLOAT_SUBTRACT, [1.0,2.0]).value == -1.0 - assert prim(p.FLOAT_SUBTRACT, [15.0,4.5]).value == 10.5 + assert prim(primitives.FLOAT_SUBTRACT, [1.0,2.0]).value == -1.0 + assert prim(primitives.FLOAT_SUBTRACT, [15.0,4.5]).value == 10.5 def test_float_multiply(): - assert prim(p.FLOAT_MULTIPLY, [10.0,2.0]).value == 20.0 - assert prim(p.FLOAT_MULTIPLY, [3.0,4.5]).value == 13.5 + assert prim(primitives.FLOAT_MULTIPLY, [10.0,2.0]).value == 20.0 + assert prim(primitives.FLOAT_MULTIPLY, [3.0,4.5]).value == 13.5 def test_float_divide(): - assert prim(p.FLOAT_DIVIDE, [1.0,2.0]).value == 0.5 - assert prim(p.FLOAT_DIVIDE, [3.5,4.0]).value == 0.875 + assert prim(primitives.FLOAT_DIVIDE, [1.0,2.0]).value == 0.5 + assert prim(primitives.FLOAT_DIVIDE, [3.5,4.0]).value == 0.875 def test_float_truncate(): - assert prim(p.FLOAT_TRUNCATED, [-4.6]).value == -4 - assert prim(p.FLOAT_TRUNCATED, [-4.5]).value == -4 - assert prim(p.FLOAT_TRUNCATED, [-4.4]).value == -4 - assert prim(p.FLOAT_TRUNCATED, [4.4]).value == 4 - assert prim(p.FLOAT_TRUNCATED, [4.5]).value == 4 - assert prim(p.FLOAT_TRUNCATED, [4.6]).value == 4 + assert prim(primitives.FLOAT_TRUNCATED, [-4.6]).value == -4 + assert prim(primitives.FLOAT_TRUNCATED, [-4.5]).value == -4 + assert prim(primitives.FLOAT_TRUNCATED, [-4.4]).value == -4 + assert prim(primitives.FLOAT_TRUNCATED, [4.4]).value == 4 + assert prim(primitives.FLOAT_TRUNCATED, [4.5]).value == 4 + assert prim(primitives.FLOAT_TRUNCATED, [4.6]).value == 4 def test_at(): w_obj = mockclass(0, varsized=True).as_class_get_shadow().new(1) w_obj.store(0, "foo") - assert prim(p.AT, [w_obj, 1]) == "foo" + assert prim(primitives.AT, [w_obj, 1]) == "foo" def test_invalid_at(): w_obj = mockclass(0).as_class_get_shadow().new() - prim_fails(p.AT, [w_obj, 1]) + prim_fails(primitives.AT, [w_obj, 1]) def test_at_put(): w_obj = mockclass(0, varsized=1).as_class_get_shadow().new(1) - assert prim(p.AT_PUT, [w_obj, 1, 22]).value == 22 - assert prim(p.AT, [w_obj, 1]).value == 22 + assert prim(primitives.AT_PUT, [w_obj, 1, 22]).value == 22 + assert prim(primitives.AT, [w_obj, 1]).value == 22 def test_invalid_at_put(): w_obj = mockclass(0).as_class_get_shadow().new() - prim_fails(p.AT_PUT, [w_obj, 1, 22]) + prim_fails(primitives.AT_PUT, [w_obj, 1, 22]) def test_string_at(): - assert prim(p.STRING_AT, ["foobar", 4]) == wrap("b") + assert prim(primitives.STRING_AT, ["foobar", 4]) == wrap("b") def test_string_at_put(): test_str = wrap("foobar") - assert prim(p.STRING_AT_PUT, [test_str, 4, "c"]) == wrap("c") + assert prim(primitives.STRING_AT_PUT, [test_str, 4, "c"]) == wrap("c") exp = "foocar" for i in range(len(exp)): - assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i]) + assert prim(primitives.STRING_AT, [test_str, i]) == wrap(exp[i]) def test_object_at(): - w_v = prim(p.OBJECT_AT, ["q", objtable.CHARACTER_VALUE_INDEX+1]) + w_v = prim(primitives.OBJECT_AT, ["q", objtable.CHARACTER_VALUE_INDEX+1]) assert w_v.value == ord("q") def test_invalid_object_at(): - prim_fails(p.OBJECT_AT, ["q", objtable.CHARACTER_VALUE_INDEX+2]) + prim_fails(primitives.OBJECT_AT, ["q", objtable.CHARACTER_VALUE_INDEX+2]) def test_object_at_put(): w_obj = mockclass(1).as_class_get_shadow().new() - assert prim(p.OBJECT_AT_PUT, [w_obj, 1, "q"]) is wrap("q") - assert prim(p.OBJECT_AT, [w_obj, 1]) is wrap("q") + assert prim(primitives.OBJECT_AT_PUT, [w_obj, 1, "q"]) is wrap("q") + assert prim(primitives.OBJECT_AT, [w_obj, 1]) is wrap("q") def test_invalid_object_at_put(): w_obj = mockclass(1).as_class_get_shadow().new() - prim_fails(p.OBJECT_AT, [w_obj, 2, 42]) + prim_fails(primitives.OBJECT_AT_PUT, [w_obj, 2, 42]) def test_string_at_put(): test_str = wrap("foobar") - assert prim(p.STRING_AT_PUT, [test_str, 4, "c"]) == wrap("c") + assert prim(primitives.STRING_AT_PUT, [test_str, 4, "c"]) == wrap("c") exp = "foocar" for i in range(1,len(exp)+1): - assert prim(p.STRING_AT, [test_str, i]) == wrap(exp[i-1]) + assert prim(primitives.STRING_AT, [test_str, i]) == wrap(exp[i-1]) def test_new(): w_Object = classtable.classtable['w_Object'] - w_res = prim(p.NEW, [w_Object]) + w_res = prim(primitives.NEW, [w_Object]) assert w_res.getclass() is w_Object def test_invalid_new(): - prim_fails(p.NEW, [classtable.w_String]) + prim_fails(primitives.NEW, [classtable.w_String]) def test_new_with_arg(): - w_res = prim(p.NEW_WITH_ARG, [classtable.w_String, 20]) + w_res = prim(primitives.NEW_WITH_ARG, [classtable.w_String, 20]) assert w_res.getclass() == classtable.w_String assert w_res.size() == 20 def test_invalid_new_with_arg(): w_Object = classtable.classtable['w_Object'] - prim_fails(p.NEW_WITH_ARG, [w_Object, 20]) + prim_fails(primitives.NEW_WITH_ARG, [w_Object, 20]) def test_inst_var_at(): # I am not entirely sure if this is what this primitive is # supposed to do, so the test may be bogus: - w_v = prim(p.INST_VAR_AT, ["q", objtable.CHARACTER_VALUE_INDEX]) + w_v = prim(primitives.INST_VAR_AT, ["q", objtable.CHARACTER_VALUE_INDEX]) assert w_v.value == ord("q") - w_v = prim(p.INST_VAR_AT, ["abc", 1]) + w_v = prim(primitives.INST_VAR_AT, ["abc", 1]) assert w_v.value == ord("b") def test_as_oop(): py.test.skip("not yet clear what AS_OOP returns: hash or header?") w_obj = mockclass(0).as_class_get_shadow().new() w_obj.w_hash = wrap(22) - assert prim(p.AS_OOP, [w_obj]).value == 22 + assert prim(primitives.AS_OOP, [w_obj]).value == 22 def test_as_oop_not_applicable_to_int(): - prim_fails(p.AS_OOP, [22]) + prim_fails(primitives.AS_OOP, [22]) def test_const_primitives(): for (code, const) in [ - (p.PUSH_TRUE, objtable.w_true), - (p.PUSH_FALSE, objtable.w_false), - (p.PUSH_NIL, objtable.w_nil), - (p.PUSH_MINUS_ONE, objtable.w_mone), - (p.PUSH_ZERO, objtable.w_zero), - (p.PUSH_ONE, objtable.w_one), - (p.PUSH_TWO, objtable.w_two), + (primitives.PUSH_TRUE, objtable.w_true), + (primitives.PUSH_FALSE, objtable.w_false), + (primitives.PUSH_NIL, objtable.w_nil), + (primitives.PUSH_MINUS_ONE, objtable.w_mone), + (primitives.PUSH_ZERO, objtable.w_zero), + (primitives.PUSH_ONE, objtable.w_one), + (primitives.PUSH_TWO, objtable.w_two), ]: assert prim(code, [objtable.w_nil]) is const - assert prim(p.PUSH_SELF, [objtable.w_nil]) is objtable.w_nil - assert prim(p.PUSH_SELF, ["a"]) is wrap("a") + assert prim(primitives.PUSH_SELF, [objtable.w_nil]) is objtable.w_nil + assert prim(primitives.PUSH_SELF, ["a"]) is wrap("a") def test_boolean(): - assert prim(p.LESSTHAN, [1,2]) == objtable.w_true - assert prim(p.GREATERTHAN, [3,4]) == objtable.w_false - assert prim(p.LESSOREQUAL, [1,2]) == objtable.w_true - assert prim(p.GREATEROREQUAL, [3,4]) == objtable.w_false - assert prim(p.EQUAL, [2,2]) == objtable.w_true - assert prim(p.NOTEQUAL, [2,2]) == objtable.w_false + assert prim(primitives.LESSTHAN, [1,2]) == objtable.w_true + assert prim(primitives.GREATERTHAN, [3,4]) == objtable.w_false + assert prim(primitives.LESSOREQUAL, [1,2]) == objtable.w_true + assert prim(primitives.GREATEROREQUAL, [3,4]) == objtable.w_false + assert prim(primitives.EQUAL, [2,2]) == objtable.w_true + assert prim(primitives.NOTEQUAL, [2,2]) == objtable.w_false def test_float_boolean(): - assert prim(p.FLOAT_LESSTHAN, [1.0,2.0]) == objtable.w_true - assert prim(p.FLOAT_GREATERTHAN, [3.0,4.0]) == objtable.w_false - assert prim(p.FLOAT_LESSOREQUAL, [1.3,2.6]) == objtable.w_true - assert prim(p.FLOAT_GREATEROREQUAL, [3.5,4.9]) == objtable.w_false - assert prim(p.FLOAT_EQUAL, [2.2,2.2]) == objtable.w_true - assert prim(p.FLOAT_NOTEQUAL, [2.2,2.2]) == objtable.w_false + assert prim(primitives.FLOAT_LESSTHAN, [1.0,2.0]) == objtable.w_true + assert prim(primitives.FLOAT_GREATERTHAN, [3.0,4.0]) == objtable.w_false + assert prim(primitives.FLOAT_LESSOREQUAL, [1.3,2.6]) == objtable.w_true + assert prim(primitives.FLOAT_GREATEROREQUAL, [3.5,4.9]) == objtable.w_false + assert prim(primitives.FLOAT_EQUAL, [2.2,2.2]) == objtable.w_true + assert prim(primitives.FLOAT_NOTEQUAL, [2.2,2.2]) == objtable.w_false def test_block_copy_and_value(): # see test_interpreter for tests of these opcodes @@ -291,35 +291,35 @@ return round(w_f.value,ROUNDING_DIGITS) == round(f,ROUNDING_DIGITS) def test_primitive_square_root(): - assert prim(p.FLOAT_SQUARE_ROOT, [4.0]).value == 2.0 - assert float_equals(prim(p.FLOAT_SQUARE_ROOT, [2.0]), math.sqrt(2)) - prim_fails(p.FLOAT_SQUARE_ROOT, [-2.0]) + assert prim(primitives.FLOAT_SQUARE_ROOT, [4.0]).value == 2.0 + assert float_equals(prim(primitives.FLOAT_SQUARE_ROOT, [2.0]), math.sqrt(2)) + prim_fails(primitives.FLOAT_SQUARE_ROOT, [-2.0]) def test_primitive_sin(): - assert prim(p.FLOAT_SIN, [0.0]).value == 0.0 - assert float_equals(prim(p.FLOAT_SIN, [math.pi]), 0.0) - assert float_equals(prim(p.FLOAT_SIN, [math.pi/2]), 1.0) + assert prim(primitives.FLOAT_SIN, [0.0]).value == 0.0 + assert float_equals(prim(primitives.FLOAT_SIN, [math.pi]), 0.0) + assert float_equals(prim(primitives.FLOAT_SIN, [math.pi/2]), 1.0) def test_primitive_arctan(): - assert prim(p.FLOAT_ARCTAN, [0.0]).value == 0.0 - assert float_equals(prim(p.FLOAT_ARCTAN, [1]), math.pi/4) - assert float_equals(prim(p.FLOAT_ARCTAN, [1e99]), math.pi/2) + assert prim(primitives.FLOAT_ARCTAN, [0.0]).value == 0.0 + assert float_equals(prim(primitives.FLOAT_ARCTAN, [1]), math.pi/4) + assert float_equals(prim(primitives.FLOAT_ARCTAN, [1e99]), math.pi/2) def test_primitive_log_n(): - assert prim(p.FLOAT_LOG_N, [1.0]).value == 0.0 - assert prim(p.FLOAT_LOG_N, [math.e]).value == 1.0 - assert float_equals(prim(p.FLOAT_LOG_N, [10.0]), math.log(10)) - assert isinf(prim(p.FLOAT_LOG_N, [0.0]).value) # works also for negative infinity - assert isnan(prim(p.FLOAT_LOG_N, [-1.0]).value) + assert prim(primitives.FLOAT_LOG_N, [1.0]).value == 0.0 + assert prim(primitives.FLOAT_LOG_N, [math.e]).value == 1.0 + assert float_equals(prim(primitives.FLOAT_LOG_N, [10.0]), math.log(10)) + assert isinf(prim(primitives.FLOAT_LOG_N, [0.0]).value) # works also for negative infinity + assert isnan(prim(primitives.FLOAT_LOG_N, [-1.0]).value) def test_primitive_exp(): - assert float_equals(prim(p.FLOAT_EXP, [-1.0]), 1/math.e) - assert prim(p.FLOAT_EXP, [0]).value == 1 - assert float_equals(prim(p.FLOAT_EXP, [1]), math.e) - assert float_equals(prim(p.FLOAT_EXP, [math.log(10)]), 10) + assert float_equals(prim(primitives.FLOAT_EXP, [-1.0]), 1/math.e) + assert prim(primitives.FLOAT_EXP, [0]).value == 1 + assert float_equals(prim(primitives.FLOAT_EXP, [1]), math.e) + assert float_equals(prim(primitives.FLOAT_EXP, [math.log(10)]), 10) def equals_ttp(rcvr,arg,res): - return float_equals(prim(p.FLOAT_TIMES_TWO_POWER, [rcvr,arg]), res) + return float_equals(prim(primitives.FLOAT_TIMES_TWO_POWER, [rcvr,arg]), res) def test_times_two_power(): assert equals_ttp(1,1,2) @@ -332,9 +332,9 @@ def test_inc_gc(): # Should not fail :-) - prim(p.INC_GC, [42]) # Dummy arg + prim(primitives.INC_GC, [42]) # Dummy arg def test_full_gc(): # Should not fail :-) - prim(p.FULL_GC, [42]) # Dummy arg + prim(primitives.FULL_GC, [42]) # Dummy arg From cfbolz at codespeak.net Fri Oct 26 00:51:51 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 26 Oct 2007 00:51:51 +0200 (CEST) Subject: [pypy-svn] r48010 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025225151.4DF538134@code0.codespeak.net> Author: cfbolz Date: Fri Oct 26 00:51:51 2007 New Revision: 48010 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: a comment and a problem found by translation Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 00:51:51 2007 @@ -48,6 +48,7 @@ prim_table = [raise_failing_default] * 576 def expose_primitive(code, unwrap_spec=None): + # some serious magic, don't look from pypy.rlib.unroll import unrolling_iterable def decorator(func): assert code not in prim_table @@ -225,9 +226,9 @@ w_res = objtable.wrap_int(int(f)) return w_res - at expose_primitive(FLOAT_TIMES_TWO_POWER, unwrap_spec=[float, float]) + at expose_primitive(FLOAT_TIMES_TWO_POWER, unwrap_spec=[float, int]) def func(interp, rcvr, arg): - w_res = objtable.wrap_float(math.ldexp(rcvr,arg)) + w_res = objtable.wrap_float(math.ldexp(rcvr, arg)) return w_res @expose_primitive(FLOAT_SQUARE_ROOT, unwrap_spec=[float]) From arigo at codespeak.net Fri Oct 26 00:59:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 26 Oct 2007 00:59:59 +0200 (CEST) Subject: [pypy-svn] r48011 - pypy/dist/pypy/translator/goal Message-ID: <20071025225959.CA9268134@code0.codespeak.net> Author: arigo Date: Fri Oct 26 00:59:58 2007 New Revision: 48011 Added: pypy/dist/pypy/translator/goal/targettinybenchsmalltalk.py (contents, props changed) Log: Another smalltalk target. Added: pypy/dist/pypy/translator/goal/targettinybenchsmalltalk.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/goal/targettinybenchsmalltalk.py Fri Oct 26 00:59:58 2007 @@ -0,0 +1,66 @@ +import os +from pypy.lang.smalltalk import model, interpreter, primitives, shadow +from pypy.lang.smalltalk import objtable +from pypy.lang.smalltalk.objtable import wrap_int +from pypy.lang.smalltalk import classtable +from pypy.lang.smalltalk.tool.analyseimage import * + +# This loads the whole mini.image in advance. At run-time, +# it executes the tinyBenchmark. In this way we get an RPython +# "image" frozen into the executable, mmap'ed by the OS from +# there and loaded lazily when needed :-) + + +# XXX this only compiles if sys.recursionlimit is high enough! +# On non-Linux platforms I don't know if there is enough stack to +# compile... +sys.setrecursionlimit(100000) + + +def tinyBenchmarks(): + image = create_squeakimage() + interp = interpreter.Interpreter() + + w_object = model.W_SmallInteger(0) + + # Should get this from w_object + w_smallint_class = image.special(constants.SO_SMALLINTEGER_CLASS) + s_class = w_object.shadow_of_my_class() + w_method = s_class.lookup("tinyBenchmarks") + + assert w_method + w_frame = w_method.create_frame(w_object, []) + interp.w_active_context = w_frame + + print w_method + print "Going to execute %d toplevel bytecodes" % (len(w_method.bytes),) + counter = 0 + + from pypy.lang.smalltalk.interpreter import BYTECODE_TABLE + return interp + + +interp = tinyBenchmarks() + + +def entry_point(argv): + counter = 0 + try: + while True: + counter += 1 + interp.step() + if counter == 100000: + counter = 0 + os.write(2, '#') + except interpreter.ReturnFromTopLevel, e: + w_result = e.object + + assert isinstance(w_result, model.W_BytesObject) + print w_result.as_string() + return 0 + + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None From cfbolz at codespeak.net Fri Oct 26 01:00:08 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 26 Oct 2007 01:00:08 +0200 (CEST) Subject: [pypy-svn] r48012 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025230008.5AF368143@code0.codespeak.net> Author: cfbolz Date: Fri Oct 26 01:00:07 2007 New Revision: 48012 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: small heuristic to assign nicer names to functions to make it easier to debug Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 01:00:07 2007 @@ -50,8 +50,19 @@ def expose_primitive(code, unwrap_spec=None): # some serious magic, don't look from pypy.rlib.unroll import unrolling_iterable + # heuristics to give it a nice name + name = None + for key, value in globals().iteritems(): + if isinstance(value, int) and value == code and key == key.upper(): + if name is not None: + # refusing to guess + name = "unknown" + else: + name = key + def decorator(func): assert code not in prim_table + func.func_name = "prim_" + name if unwrap_spec is None: prim_table[code] = func return func @@ -82,7 +93,7 @@ frame.pop_n(len_unwrap_spec) # only if no exception occurs! return res - wrapped.func_name = func.func_name + wrapped.func_name = "wrap_prim_" + name prim_table[code] = wrapped return func return decorator From arigo at codespeak.net Fri Oct 26 01:09:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 26 Oct 2007 01:09:11 +0200 (CEST) Subject: [pypy-svn] r48013 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025230911.AE3838134@code0.codespeak.net> Author: arigo Date: Fri Oct 26 01:09:11 2007 New Revision: 48013 Modified: pypy/dist/pypy/lang/smalltalk/shadow.py Log: Attach information to MethodNotFound just for debugging. Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Fri Oct 26 01:09:11 2007 @@ -179,7 +179,8 @@ while selector not in look_in_shadow.methoddict: look_in_shadow = look_in_shadow.s_superclass if look_in_shadow is None: - raise MethodNotFound + # attach information on the exception, for debugging. + raise MethodNotFound(self, selector) return look_in_shadow.methoddict[selector] def installmethod(self, selector, method): From akuhn at codespeak.net Fri Oct 26 01:13:28 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Fri, 26 Oct 2007 01:13:28 +0200 (CEST) Subject: [pypy-svn] r48014 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071025231328.EDEAA8134@code0.codespeak.net> Author: akuhn Date: Fri Oct 26 01:13:28 2007 New Revision: 48014 Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: (akuhn, tverwaes) trying to perform(w(10).getclass(), "compile:classified:notifying:", w(sourcecode), w('pypy'), w(None)) Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Fri Oct 26 01:13:28 2007 @@ -224,3 +224,45 @@ assert w_true is objtable.w_true w_false = image.special(constants.SO_FALSE) assert w_false is objtable.w_false + +def test_compile_method(): + py.test.skip("Todo, make compiling of methods in mini.image work.") + sourcecode = """fib + ^self < 2 + ifTrue: [ 1 ] + ifFalse: [ (self - 1) fib + (self - 2) fib ]""" + perform(w(10).getclass(), "compile:classified:notifying:", w(sourcecode), w('pypy'), w(None)) + assert perform(w(10), "fib") == w(89) + +def w(any): + if any is None: + return objtable.w_nil + if isinstance(any, str): + # assume never have strings of length 1 + if len(any) == 1: + return objtable.wrap_chr(any) + else: + return objtable.wrap_string(any) + if isinstance(any, int): + return objtable.wrap_int(any) + if isinstance(any, bool): + return objtable.wrap_bool(any) + if isinstance(any, float): + return objtable.wrap_float(any) + else: + raise Exception + +def perform(w_receiver, selector, *arguments_w): + interp = interpreter.Interpreter() + s_class = w_receiver.shadow_of_my_class() + w_method = s_class.lookup(selector) + assert w_method + w_frame = w_method.create_frame(w_receiver, list(arguments_w)) + interp.w_active_context = w_frame + while True: + try: + interp.step() + #print interp.w_active_context.stack + except interpreter.ReturnFromTopLevel, e: + return e.object.value + \ No newline at end of file From akuhn at codespeak.net Fri Oct 26 01:19:36 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Fri, 26 Oct 2007 01:19:36 +0200 (CEST) Subject: [pypy-svn] r48015 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071025231936.8D7778136@code0.codespeak.net> Author: akuhn Date: Fri Oct 26 01:19:36 2007 New Revision: 48015 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: started to using speaking names and correctly tagged index names, like n0 and n1 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 01:19:36 2007 @@ -294,10 +294,11 @@ assert_valid_index(idx-1, w_obj) return w_obj, idx-1, w_val - at expose_primitive(AT, unwrap_spec=[object, object]) -def func(interp, w_obj, w_idx): - [w_obj, idx] = common_at(w_obj, w_idx) - return w_obj.fetch(idx) + at expose_primitive(AT, unwrap_spec=[object, int]) +def func(interp, w_obj, n1): + n0 = n1 - 1 + assert_valid_index(n0, w_obj) + return w_obj.fetch(n0) @expose_primitive(AT_PUT, unwrap_spec=[object, object, object]) def func(interp, w_obj, w_idx, w_val): From tverwaes at codespeak.net Fri Oct 26 01:24:25 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 26 Oct 2007 01:24:25 +0200 (CEST) Subject: [pypy-svn] r48016 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071025232425.973FB8136@code0.codespeak.net> Author: tverwaes Date: Fri Oct 26 01:24:25 2007 New Revision: 48016 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: link byte->prim for getclass Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 01:24:25 2007 @@ -370,7 +370,8 @@ raise MissingBytecode def bytecodePrimClass(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush( + primitives.CLASS, "class", 0, interp) def bytecodePrimBlockCopy(self, interp): self.callPrimitiveAndPush( Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Fri Oct 26 01:24:25 2007 @@ -265,4 +265,4 @@ #print interp.w_active_context.stack except interpreter.ReturnFromTopLevel, e: return e.object.value - \ No newline at end of file + From rxe at codespeak.net Fri Oct 26 04:13:19 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 26 Oct 2007 04:13:19 +0200 (CEST) Subject: [pypy-svn] r48017 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071026021319.E156480E0@code0.codespeak.net> Author: rxe Date: Fri Oct 26 04:13:18 2007 New Revision: 48017 Modified: pypy/dist/pypy/translator/llvm/test/test_lladdresses.py Log: fix up some imports Modified: pypy/dist/pypy/translator/llvm/test/test_lladdresses.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lladdresses.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lladdresses.py Fri Oct 26 04:13:18 2007 @@ -1,5 +1,5 @@ import py -from pypy.rpython.lltypesystem.llmemory import * +from pypy.rpython.lltypesystem import llmemory, lltype from pypy.annotation.model import SomeAddress, SomeChar from pypy.rlib.objectmodel import free_non_gc_object @@ -8,15 +8,15 @@ def test_null(): def f(): - return NULL - NULL + return llmemory.NULL - llmemory.NULL fc = compile_function(f, []) def test_convert_to_bool(): def convert_to_bool(x): if x: - return bool(NULL) + return bool(llmemory.NULL) else: - return bool(NULL + 1) + return bool(llmemory.NULL + 1) fc = compile_function(convert_to_bool, [int]) res = fc(1) assert isinstance(res, int) and not res @@ -25,10 +25,10 @@ def test_memory_access(): def f(value): - addr = raw_malloc(16) + addr = llmemory.raw_malloc(16) addr.signed[0] = value res = addr.signed[0] - raw_free(addr) + llmemory.raw_free(addr) return res fc = compile_function(f, [int]) res = fc(42) @@ -39,11 +39,11 @@ def test_pointer_arithmetic(): def f(offset, char): char = chr(char) - addr = raw_malloc(10000) + addr = llmemory.raw_malloc(10000) same_offset = (addr + 2 * offset - offset) - addr addr.char[offset] = char result = (addr + same_offset).char[0] - raw_free(addr) + llmemory.raw_free(addr) return ord(result) fc = compile_function(f, [int, int]) res = fc(10, ord("c")) @@ -54,7 +54,7 @@ def test_pointer_arithmetic_inplace(): def f(offset, char): char = chr(char) - addr = raw_malloc(10000) + addr = llmemory.raw_malloc(10000) addr += offset addr.char[-offset] = char addr -= offset @@ -65,17 +65,17 @@ def test_raw_memcopy(): def f(): - addr = raw_malloc(100) + addr = llmemory.raw_malloc(100) addr.signed[0] = 12 (addr + 10).signed[0] = 42 (addr + 20).char[0] = "a" - addr1 = raw_malloc(100) - raw_memcopy(addr, addr1, 100) + addr1 = llmemory.raw_malloc(100) + llmemory.raw_memcopy(addr, addr1, 100) result = addr1.signed[0] == 12 result = result and (addr1 + 10).signed[0] == 42 result = result and (addr1 + 20).char[0] == "a" - raw_free(addr) - raw_free(addr1) + llmemory.raw_free(addr) + llmemory.raw_free(addr1) return result fc = compile_function(f, []) res = fc() @@ -84,7 +84,7 @@ def test_pointer_comparison(): def f(): result = 0 - for addr1 in [raw_malloc(1), NULL]: + for addr1 in [llmemory.raw_malloc(1), llmemory.NULL]: addr2 = addr1 + 1 result = result * 2 + int(addr1 == addr2) result = result * 2 + int(addr1 != addr2) From rxe at codespeak.net Fri Oct 26 04:19:59 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 26 Oct 2007 04:19:59 +0200 (CEST) Subject: [pypy-svn] r48018 - pypy/dist/pypy/translator/llvm Message-ID: <20071026021959.70A4E80E0@code0.codespeak.net> Author: rxe Date: Fri Oct 26 04:19:59 2007 New Revision: 48018 Modified: pypy/dist/pypy/translator/llvm/opwriter.py Log: add some approximation of interior pointer operations that were making lots of llvm tests fail. Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Fri Oct 26 04:19:59 2007 @@ -319,6 +319,38 @@ self.codewriter.call(opr.retref, 'sbyte*', '%pypy_malloc', [word], [opr.argrefs[0]]) + def _interiorhelper(self, TYPE, args): + indices = [] + # XXX this mess is because an FixedSizeArray can sometimes be an Array and sometimes a Struct + for arg in args: + name = None + if arg.concretetype is lltype.Void: + name = arg.value + assert name in list(TYPE._names) + fieldnames = TYPE._names_without_voids() + indexref = fieldnames.index(name) + else: + indexref = self.db.repr_arg(arg) + + if isinstance(TYPE, lltype.FixedSizeArray): + indices.append(("int", indexref)) + TYPE = TYPE.OF + + elif isinstance(TYPE, lltype.Array): + indices.append(("uint", 1)) + indices.append(("int", indexref)) + TYPE = TYPE.OF + + elif isinstance(TYPE, lltype.Struct): + assert name is not None + TYPE = getattr(TYPE, name) + indices.append(("uint", indexref)) + + else: + raise Exception("unsupported type: %s" % TYPE) + + return TYPE, indices + def getfield(self, opr): op = opr.op if opr.rettype != "void": @@ -332,6 +364,42 @@ else: self._skipped(opr) + def getinteriorfield(self, opr): + if opr.rettype != "void": + op = opr.op + _, indices = self._interiorhelper(op.args[0].concretetype.TO, op.args[1:]) + tmpvar = self._tmp() + self.codewriter.getelementptr(tmpvar, opr.argtypes[0], opr.argrefs[0], indices) + self.codewriter.load(opr.retref, opr.rettype, tmpvar) + else: + self._skipped(opr) + + def setinteriorfield(self, opr): + op = opr.op + if opr.argtypes[-1] != "void": + print op.args, op.args[1:-1] + _, indices = self._interiorhelper(op.args[0].concretetype.TO, op.args[1:-1]) + tmpvar = self._tmp() + self.codewriter.getelementptr(tmpvar, opr.argtypes[0], opr.argrefs[0], indices) + self.codewriter.store(opr.argtypes[-1], opr.argrefs[-1], tmpvar) + else: + self._skipped(opr) + bare_setinteriorfield = setinteriorfield + + def getinteriorarraysize(self, opr): + op = opr.op + TYPE, indices = self._interiorhelper(op.args[0].concretetype.TO, op.args[1:]) + if isinstance(TYPE, lltype.Array): + # gets the length + indices.append(("uint", 0)) + lengthref = self._tmp() + self.codewriter.getelementptr(lengthref, opr.argtypes[0], opr.argrefs[0], indices) + else: + assert isinstance(TYPE, lltype.FixedSizeArray) + lengthref = TYPE.length + XXX # test + self.codewriter.load(opr.retref, opr.rettype, lengthref) + def direct_fieldptr(self, opr): op = opr.op assert opr.rettype != "void" From lukas at codespeak.net Fri Oct 26 10:33:48 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Fri, 26 Oct 2007 10:33:48 +0200 (CEST) Subject: [pypy-svn] r48019 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026083348.A48DA8134@code0.codespeak.net> Author: lukas Date: Fri Oct 26 10:33:47 2007 New Revision: 48019 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: (cfbolz,lr) made it possible to have failing primitives print something Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 10:33:47 2007 @@ -39,13 +39,15 @@ # primitive functions. Each primitive function takes two # arguments, an interp and an argument_count # completes, and returns a result, or throws a PrimitiveFailedError. - -def raise_failing_default(interp, argument_count): - raise PrimitiveFailedError +def make_failing(code): + def raise_failing_default(interp, argument_count): +# print "Primitive failed", code + raise PrimitiveFailedError + return raise_failing_default # Squeak has primitives all the way up to 575 # So all optional primitives will default to the bytecode implementation -prim_table = [raise_failing_default] * 576 +prim_table = [make_failing(i) for i in range(576)] def expose_primitive(code, unwrap_spec=None): # some serious magic, don't look @@ -642,8 +644,8 @@ def func(interp, w_rcvr, w_args): raise PrimitiveNotYetWrittenError() - at expose_primitive(PRIMITIVE_PERFORM, unwrap_spec=[object, object]) -def func(interp, w_rcvr, w_sel): + at expose_primitive(PRIMITIVE_PERFORM) +def func(interp, argument_count): # XXX we can implement this when lookup on shadow class is done raise PrimitiveNotYetWrittenError() From tverwaes at codespeak.net Fri Oct 26 11:22:57 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 26 Oct 2007 11:22:57 +0200 (CEST) Subject: [pypy-svn] r48021 - in pypy/dist/pypy/lang/smalltalk: . test tool Message-ID: <20071026092257.5677B8130@code0.codespeak.net> Author: tverwaes Date: Fri Oct 26 11:22:56 2007 New Revision: 48021 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: reverting change for extendedSuperSend (popping receiver, fixed test) Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 11:22:56 2007 @@ -116,15 +116,15 @@ def _sendSelfSelector(self, selector, argcount, interp): receiver = self.peek(argcount) self._sendSelector(selector, argcount, interp, - receiver, receiver.shadow_of_my_class(), 1) + receiver, receiver.shadow_of_my_class()) def _sendSuperSelector(self, selector, argcount, interp): s_compiledin = self.w_method().compiledin().as_class_get_shadow() self._sendSelector(selector, argcount, interp, self.w_receiver, - s_compiledin.s_superclass, 0) + s_compiledin.s_superclass) def _sendSelector(self, selector, argcount, interp, - receiver, receiverclassshadow, popreceiver): + receiver, receiverclassshadow): assert argcount >= 0 method = receiverclassshadow.lookup(selector) # XXX catch MethodNotFound here and send doesNotUnderstand: @@ -143,7 +143,7 @@ assert start >= 0 # XXX check in the Blue Book what to do in this case arguments = self.stack[start:] interp.w_active_context = method.create_frame(receiver, arguments, self) - self.pop_n(argcount + popreceiver) + self.pop_n(argcount + 1) def _return(self, object, interp): if self.w_sender is None: # for tests, when returning from the top-level context Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 11:22:56 2007 @@ -410,10 +410,10 @@ # first call method installed in w_class bytecodes = singleExtendedSendBytecode + chr(0) # which does a call to its super - meth1 = model.W_CompiledMethod(0, bytecode) + meth1 = model.W_CompiledMethod(0, pushReceiverBytecode + bytecode) w_class.as_class_get_shadow().installmethod("foo", meth1) # and that one again to its super - meth2 = model.W_CompiledMethod(0, bytecode) + meth2 = model.W_CompiledMethod(0, pushReceiverBytecode + bytecode) w_super.as_class_get_shadow().installmethod("foo", meth2) meth3 = model.W_CompiledMethod(0, "") w_supersuper.as_class_get_shadow().installmethod("foo", meth3) @@ -422,9 +422,11 @@ interp = new_interpreter(bytecodes) interp.w_active_context.w_method().literals = fakeliterals("foo") interp.w_active_context.push(w_object) - for w_specificclass in [w_class, w_super, w_supersuper]: + interp.step() + for w_specificclass in [w_super, w_supersuper]: callerContext = interp.w_active_context interp.step() + interp.step() assert interp.w_active_context.w_sender == callerContext assert interp.w_active_context.stack == [] assert interp.w_active_context.w_receiver == w_object Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Fri Oct 26 11:22:56 2007 @@ -51,8 +51,10 @@ while True: try: counter += 1 - #if interp.w_active_context == w_frame: - # print "Executing toplevel bytecode nr: %d of %d" % (counter, len(w_method.bytes)) + if interp.w_active_context == w_frame: + print "Executing toplevel bytecode nr: %d of %d" % (interp.w_active_context.pc+1, len(w_method.bytes)) + cb = ord(interp.w_active_context.w_method().bytes[interp.w_active_context.pc]) + print "= bytecode: %s %d" % (BYTECODE_TABLE[cb].__name__,cb) interp.step() #if hasattr(interp.w_active_context,"currentBytecode"): # print "Executing bytecode: %s" % (BYTECODE_TABLE[interp.w_active_context.currentBytecode].__name__,) From akuhn at codespeak.net Fri Oct 26 11:23:10 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Fri, 26 Oct 2007 11:23:10 +0200 (CEST) Subject: [pypy-svn] r48022 - pypy/extradoc/sprintinfo/bern2007 Message-ID: <20071026092310.32E2E813B@code0.codespeak.net> Author: akuhn Date: Fri Oct 26 11:23:09 2007 New Revision: 48022 Modified: pypy/extradoc/sprintinfo/bern2007/planning.txt Log: (akuhn,cfbolz,arigo,etc...)) planning session done Modified: pypy/extradoc/sprintinfo/bern2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/bern2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/bern2007/planning.txt Fri Oct 26 11:23:09 2007 @@ -8,31 +8,30 @@ Niko Oscar Lukas -Tudor +Adrian L Task ideas ---------- -- give everybody an account (arigo) DONE +- presenting the new way of registering primitives (cfbolz) -- present coding conventions (cfbolz) DONE +- new coding convention for indices (akuhn) -- present the "shadow" (cfbolz, arigo) DONE +- demoing the working translation (cfbolz) Afterwards: ----------- -- Implement loading the image (Adrian K, Carl Friedrich) - MOSTLY DONE, the mapping between the loaded object and - the ones expected by the rest of the code is incomplete - (cfbolz, akuhn) - +- Implement loading the image (cfbolz, akuhn) + MOSTLY DONE, floats missing + - interpreter loop and bytecode dispatch (Armin, Adrian L, Lukas, Toon) DONE - implementing numbered primitives MORE PROGRESS (lukas, - cfbolz, niko) - - still more to go + cfbolz, niko, oscar, etc) + +- test for all primitives (niko, lukas) - implement all the bytecodes (Armin, Toon, Lukas, Adrian L) MOSTLY DONE @@ -40,11 +39,12 @@ - define a common interface for real image and mock image (Niko, Carl Friedrich, Adrian K) DONE +- make image loading RPython (oscar, cfbolz) + - try to generate the fib example and fight with RPython :) - (Armin, together to teach the smalltalkers) + (Armin, together to teach the smalltalkers) DONE! -- write the report for yesterday (Carl Friedrich, Adrian K, - ...) +- write the report for yesterday (Carl Friedrich, Adrian K) DONE okay, done. From fijal at codespeak.net Fri Oct 26 11:39:02 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 26 Oct 2007 11:39:02 +0200 (CEST) Subject: [pypy-svn] r48026 - pypy/dist/pypy/module/zipimport/test Message-ID: <20071026093902.0A221813B@code0.codespeak.net> Author: fijal Date: Fri Oct 26 11:39:01 2007 New Revision: 48026 Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py Log: * Cleanup sys.modules in tests * Check also for different compression. This is probably being too paranoid, but I'm copying from CPython tests Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/test/test_zipimport.py Fri Oct 26 11:39:01 2007 @@ -14,6 +14,8 @@ """ A bit structurized tests stolen and adapted from cpy's regression tests """ + compression = ZIP_STORED + def make_pyc(cls, space, co, mtime): data = marshal.dumps(co) if type(mtime) is type(0.0): @@ -45,7 +47,7 @@ cls.w_now = space.wrap(now) test_pyc = cls.make_pyc(space, co, now) cls.w_test_pyc = space.wrap(test_pyc) - cls.w_compression = space.wrap(ZIP_STORED) + cls.w_compression = space.wrap(cls.compression) #ziptestmodule = tmpdir.ensure('ziptestmodule.zip').write( ziptestmodule = tmpdir.join("somezip.zip") cls.w_tmpzip = space.wrap(str(ziptestmodule)) @@ -74,7 +76,7 @@ def setup_method(self, meth): space = self.space - name = "test_%s.zip" % meth.__name__ + name = "test_%s_%s.zip" % (self.__class__.__name__, meth.__name__) self.w_zipfile = space.wrap(str(self.tmpdir.join(name))) space.appexec([space.wrap(self)], """(self): self.write_files = [] @@ -107,6 +109,7 @@ import sys, os self.writefile(self, "uuu.py", "def f(x): return x") mod = __import__('uuu', globals(), locals(), []) + print mod assert mod.f(3) == 3 expected = { '__doc__' : None, @@ -153,6 +156,7 @@ self.writefile(self, "uu.py", "def f(x): return x") mod = __import__("uu", globals(), locals(), []) assert mod.f(3) == 3 + del sys.modules['uu'] def test_sys_modules(self): m0 = ord(self.test_pyc[0]) @@ -165,6 +169,7 @@ sys.modules['uuu'] = lambda x : x + 1 mod = z.load_module('uuu') assert mod(3) == 4 + del sys.modules['uuu'] def test_package(self): import os @@ -173,6 +178,7 @@ mod = __import__("xx", globals(), locals(), ['yy']) assert mod.__path__ assert mod.yy.f(3) == 3 + del sys.modules['xx'] def test_functions(self): import os @@ -193,3 +199,6 @@ assert z.get_code('xx') assert z.get_source('xx') == "5" assert z.archive == self.zipfile + +class AppTestZipimportDeflated(AppTestZipimport): + compression = ZIP_DEFLATED From niko at codespeak.net Fri Oct 26 11:39:06 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 11:39:06 +0200 (CEST) Subject: [pypy-svn] r48027 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026093906.5595E8145@code0.codespeak.net> Author: niko Date: Fri Oct 26 11:39:06 2007 New Revision: 48027 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (niko, lukas) adjust argument_count convention for primitives: should not include rcvr Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 11:39:06 2007 @@ -131,8 +131,8 @@ if method.primitive: func = primitives.prim_table[method.primitive] try: - # add +1 to account for the receiver - w_result = func(interp, argcount + 1) + # note: argcount does not include rcvr + w_result = func(interp, argcount) except primitives.PrimitiveFailedError: pass # ignore this error and fall back to the Smalltalk version else: @@ -295,8 +295,8 @@ def callPrimitiveAndPush(self, primitive, selector, argcount, interp): try: - # add one to the argcount to account for the self - self.push(primitives.prim_table[primitive](interp, argcount + 1)) + # note that argcount does not include self + self.push(primitives.prim_table[primitive](interp, argcount)) except primitives.PrimitiveFailedError: self._sendSelfSelector(selector, argcount, interp) @@ -382,7 +382,8 @@ primitives.PRIMITIVE_VALUE, "value", 0, interp) def bytecodePrimValueWithArg(self, interp): - raise MissingBytecode + self.callPrimitiveAndPush( + primitives.PRIMITIVE_VALUE, "value", 1, interp) def bytecodePrimDo(self, interp): self._sendSelfSelector("do:", 1, interp) @@ -477,18 +478,21 @@ def initialize_bytecode_table(): result = [None] * 256 for entry in BYTECODE_RANGES: - #def dump_func(f): - # def wrapped(*args): - # print "Bytecode: %s" % (f.__name__) - # return f(*args) - # return wrapped + def dump_func(f): + def wrapped(self, interp): + print "Bytecode: %s" % (f.__name__,) + res = f(self, interp) + print " stack after: %s" % ( + interp.w_active_context.stack) + return res + return wrapped if len(entry) == 2: positions = [entry[0]] else: positions = range(entry[0], entry[1]+1) for pos in positions: - #result[pos] = dump_func(entry[-1]) - result[pos] = entry[-1] + result[pos] = dump_func(entry[-1]) + #result[pos] = entry[-1] assert None not in result return result Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 11:39:06 2007 @@ -74,7 +74,8 @@ assert (len_unwrap_spec == len(inspect.getargspec(func)[0]) + 1, "wrong number of arguments") unrolling_unwrap_spec = unrolling_iterable(enumerate(unwrap_spec)) - def wrapped(interp, argument_count): + def wrapped(interp, argument_count_m1): + argument_count = argument_count_m1 + 1 # to account for the rcvr frame = interp.w_active_context assert argument_count == len_unwrap_spec if len(frame.stack) < len_unwrap_spec: @@ -614,8 +615,9 @@ @expose_primitive(PRIMITIVE_VALUE) def func(interp, argument_count): # XXX XXX XXX XXX test me - # If argument_count == 4, stack looks like: - # 3 2 1 0 + # argument_count does NOT include the receiver. + # This means that for argument_count == 3 the stack looks like: + # 3 2 1 Top # Rcvr | Arg 0 | Arg1 | Arg 2 # @@ -623,17 +625,18 @@ # Validate that we have a block on the stack and that it received # the proper number of arguments: - w_block_ctx = frame.peek(argument_count-1) + w_block_ctx = frame.peek(argument_count) if not isinstance(w_block_ctx, model.W_BlockContext): raise PrimitiveFailedError() exp_arg_cnt = w_block_ctx.expected_argument_count() - if argument_count != exp_arg_cnt + 1: # exp_arg_cnt doesn't count self + if argument_count != exp_arg_cnt: # exp_arg_cnt doesn't count self raise PrimitiveFailedError() - # Initialize the block stack from the contents of the stack: - # Don't bother to copy the 'self' argument + # Initialize the block stack with the arguments that were + # pushed. Also pop the receiver. block_args = frame.pop_n(exp_arg_cnt) w_block_ctx.push_all(block_args) + frame.pop() # Set some fields w_block_ctx.pc = w_block_ctx.initialip Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 11:39:06 2007 @@ -463,18 +463,27 @@ storeAssociation(doubleExtendedDoAnythingBytecode + chr(7<<5) + chr(0)) -def test_block_copy_and_value(): +def interp_bc_and_check_result_is_7(bcodes): + bcode = "".join([chr(x) for x in bcodes]) + interp = new_interpreter(bcode) + interp.w_active_context.w_method().literals = \ + fakeliterals(wrap_int(3), + wrap_int(4)) + res = interp.interpret() + assert res.value == 7 - bc_3_plus_4 = [ 137, 117, 200, 164, 4, 32, 33, 176, 125, 201, 124] - #bc_x_plus_x_plus_1 = [ 137, 118, 200, 164, 7, 104, 16, 16, 176, 118, 176, 125, 32, 202, 124 ] - #bc_x_plus_y = [ 137, 119, 200, 164, 6, 105, 104, 16, 17, 176, 125, 33, 34, 240, 124 ] +def test_bc_3_plus_4(): + interp_bc_and_check_result_is_7( + [ 137, 117, 200, 164, 4, 32, 33, 176, 125, 201, 124]) - for bcodes in [ bc_3_plus_4 ]: #, bc_x_plus_x_plus_1, bc_x_plus_y ]: - bcode = "".join([chr(x) for x in bcodes]) - interp = new_interpreter(bcode) - interp.w_active_context.w_method().literals = \ - fakeliterals(wrap_int(3), - wrap_int(4)) - res = interp.interpret() - assert res.value == 7 +def test_bc_x_plus_x_plus_1(): + interp_bc_and_check_result_is_7( + [ 137, 118, 200, 164, 7, 104, 16, 16, + 176, 118, 176, 125, 32, 202, 124 ]) + +def test_bc_x_plus_y(): + py.test.skip("not yet working") + interp_bc_and_check_result_is_7( + [ 137, 119, 200, 164, 6, 105, 104, 16, 17, + 176, 125, 33, 34, 240, 124 ]) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 11:39:06 2007 @@ -33,7 +33,7 @@ def prim(code, stack): interp, argument_count = mock(stack) - res = prim_table[code](interp, argument_count) + res = prim_table[code](interp, argument_count-1) assert not len(interp.w_active_context.stack) # check args are consumed return res @@ -41,7 +41,7 @@ interp, argument_count = mock(stack) orig_stack = list(interp.w_active_context.stack) try: - prim_table[code](interp, argument_count) + prim_table[code](interp, argument_count-1) py.test.fail("Expected PrimitiveFailedError") except PrimitiveFailedError: assert interp.w_active_context.stack == orig_stack From fijal at codespeak.net Fri Oct 26 11:40:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 26 Oct 2007 11:40:04 +0200 (CEST) Subject: [pypy-svn] r48028 - in pypy/dist/pypy/rpython/module: . test Message-ID: <20071026094004.3C754813B@code0.codespeak.net> Author: fijal Date: Fri Oct 26 11:40:03 2007 New Revision: 48028 Modified: pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/module/test/test_posix.py Log: Add os.getgid, rpython level Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Fri Oct 26 11:40:03 2007 @@ -348,6 +348,10 @@ def register_os_getpid(self): return self.extdef_for_os_function_returning_int('getpid') + @registering_if(os, 'getgid') + def register_os_getgid(self): + return self.extdef_for_os_function_returning_int('getgid') + @registering(os.open) def register_os_open(self): os_open = self.llexternal(underscore_on_windows+'open', Modified: pypy/dist/pypy/rpython/module/test/test_posix.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_posix.py (original) +++ pypy/dist/pypy/rpython/module/test/test_posix.py Fri Oct 26 11:40:03 2007 @@ -116,6 +116,12 @@ return os.getuid() assert self.interpret(f, []) == f() + if hasattr(os, 'getgid'): + def test_getgid(self): + def f(): + return os.getgid() + assert self.interpret(f, []) == f() + def test_os_wstar(self): from pypy.rpython.module.ll_os import RegisterOs for name in RegisterOs.w_star: From oscar at codespeak.net Fri Oct 26 11:42:45 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Fri, 26 Oct 2007 11:42:45 +0200 (CEST) Subject: [pypy-svn] r48029 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071026094245.6232A813B@code0.codespeak.net> Author: oscar Date: Fri Oct 26 11:42:45 2007 New Revision: 48029 Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (cfbolz, oscar) template for test_become Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 11:42:45 2007 @@ -338,3 +338,25 @@ # Should not fail :-) prim(primitives.FULL_GC, [42]) # Dummy arg +def test_become(): + py.test.skip("implement me!") + """ + testBecome + | p1 p2 a | + p1 := 1 at 2. + p2 := #(3 4 5). + a := p1 -> p2. + self assert: 1 at 2 = a key. + self assert: #(3 4 5) = a value. + self assert: p1 -> p2 = a. + self assert: p1 == a key. + self assert: p2 == a value. + p1 become: p2. + self assert: 1 at 2 = a value. + self assert: #(3 4 5) = a key. + self assert: p1 -> p2 = a. + self assert: p1 == a key. + self assert: p2 == a value. + + self should: [1 become: 2] raise: Error. + """ From fijal at codespeak.net Fri Oct 26 11:43:45 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 26 Oct 2007 11:43:45 +0200 (CEST) Subject: [pypy-svn] r48030 - in pypy/dist/pypy/module/posix: . test Message-ID: <20071026094345.D5E04813B@code0.codespeak.net> Author: fijal Date: Fri Oct 26 11:43:45 2007 New Revision: 48030 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/posix/test/test_posix2.py Log: Add os.getgid Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Fri Oct 26 11:43:45 2007 @@ -86,6 +86,8 @@ if hasattr(os, 'getuid'): interpleveldefs['getuid'] = 'interp_posix.getuid' interpleveldefs['geteuid'] = 'interp_posix.geteuid' + if hasattr(os, 'getgid'): + interpleveldefs['getgid'] = 'interp_posix.getgid' for name in RegisterOs.w_star: if hasattr(os, name): Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Fri Oct 26 11:43:45 2007 @@ -541,6 +541,14 @@ return space.wrap(os.getuid()) getuid.unwrap_spec = [ObjSpace] +def getgid(space): + """ getgid() -> gid + + Return the current process's group id. + """ + return space.wrap(os.getgid()) +getgid.unwrap_spec = [ObjSpace] + def geteuid(space): """ geteuid() -> euid Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Fri Oct 26 11:43:45 2007 @@ -29,6 +29,8 @@ if hasattr(os, 'getuid'): cls.w_getuid = space.wrap(os.getuid()) cls.w_geteuid = space.wrap(os.geteuid()) + if hasattr(os, 'getgid'): + cls.w_getgid = space.wrap(os.getgid()) def test_posix_is_pypy_s(self): assert self.posix.__file__ @@ -276,6 +278,11 @@ assert os.getuid() == self.getuid assert os.geteuid() == self.geteuid + if hasattr(os, 'getgid'): + def test_os_getgid(self): + os = self.posix + assert os.getgid() == self.getgid + def test_largefile(self): os = self.posix import sys From niko at codespeak.net Fri Oct 26 11:52:43 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 11:52:43 +0200 (CEST) Subject: [pypy-svn] r48031 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026095243.CB540813B@code0.codespeak.net> Author: niko Date: Fri Oct 26 11:52:43 2007 New Revision: 48031 Added: pypy/dist/pypy/lang/smalltalk/conftest.py Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py Log: (niko, arigo) make a py.test option --bc-trace to enable bytecode tracing Added: pypy/dist/pypy/lang/smalltalk/conftest.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/conftest.py Fri Oct 26 11:52:43 2007 @@ -0,0 +1,10 @@ +import py + +Option = py.test.config.Option +option = py.test.config.addoptions("smalltalk options", + Option('--bc-trace', + action="store_true", + dest="bc_trace", + default=False, + help="print bytecodes and stack during execution"), + ) Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 11:52:43 2007 @@ -2,6 +2,7 @@ from pypy.lang.smalltalk import model, constants, primitives from pypy.lang.smalltalk import objtable from pypy.lang.smalltalk.model import W_ContextPart +from pypy.lang.smalltalk.conftest import option class MissingBytecode(NotImplementedError): @@ -34,6 +35,9 @@ def step(self): next = self.w_active_context.getNextBytecode() bytecodeimpl = BYTECODE_TABLE[next] + if option.bc_trace: + print "About to execute bytecode %s:" % (bytecodeimpl.__name__,) + print " Stack=%s" % (repr(self.w_active_context.stack),) bytecodeimpl(self.w_active_context, self) class ReturnFromTopLevel(Exception): @@ -478,21 +482,12 @@ def initialize_bytecode_table(): result = [None] * 256 for entry in BYTECODE_RANGES: - def dump_func(f): - def wrapped(self, interp): - print "Bytecode: %s" % (f.__name__,) - res = f(self, interp) - print " stack after: %s" % ( - interp.w_active_context.stack) - return res - return wrapped if len(entry) == 2: positions = [entry[0]] else: positions = range(entry[0], entry[1]+1) for pos in positions: - result[pos] = dump_func(entry[-1]) - #result[pos] = entry[-1] + result[pos] = entry[-1] assert None not in result return result From niko at codespeak.net Fri Oct 26 11:53:40 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 11:53:40 +0200 (CEST) Subject: [pypy-svn] r48032 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071026095340.990708140@code0.codespeak.net> Author: niko Date: Fri Oct 26 11:53:38 2007 New Revision: 48032 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (niko, lukas) refactor bc tests into separate calls to help with debugging Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 11:53:38 2007 @@ -463,27 +463,32 @@ storeAssociation(doubleExtendedDoAnythingBytecode + chr(7<<5) + chr(0)) -def interp_bc_and_check_result_is_7(bcodes): +def interp_bc_and_check_result_is_7(bcodes, literals): bcode = "".join([chr(x) for x in bcodes]) interp = new_interpreter(bcode) - interp.w_active_context.w_method().literals = \ - fakeliterals(wrap_int(3), - wrap_int(4)) + interp.w_active_context.w_method().literals = literals res = interp.interpret() assert res.value == 7 def test_bc_3_plus_4(): + # [ 3+4 ] value interp_bc_and_check_result_is_7( - [ 137, 117, 200, 164, 4, 32, 33, 176, 125, 201, 124]) + [ 137, 117, 200, 164, 4, 32, 33, 176, 125, 201, 124], + fakeliterals(wrap_int(3), wrap_int(4))) + def test_bc_x_plus_x_plus_1(): + # [ :x | x+x+1 ] value: 3 interp_bc_and_check_result_is_7( [ 137, 118, 200, 164, 7, 104, 16, 16, - 176, 118, 176, 125, 32, 202, 124 ]) + 176, 118, 176, 125, 32, 202, 124 ], + fakeliterals(wrap_int(3))) def test_bc_x_plus_y(): - py.test.skip("not yet working") + # [ :x x+y ] value: 3 value: 4 + py.test.skip("FIX") interp_bc_and_check_result_is_7( [ 137, 119, 200, 164, 6, 105, 104, 16, 17, - 176, 125, 33, 34, 240, 124 ]) + 176, 125, 33, 34, 240, 124 ], + fakeliterals(wrap_int(3), wrap_int(4))) From niko at codespeak.net Fri Oct 26 11:57:27 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 11:57:27 +0200 (CEST) Subject: [pypy-svn] r48033 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026095727.574318140@code0.codespeak.net> Author: niko Date: Fri Oct 26 11:57:25 2007 New Revision: 48033 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: (niko, arigo) make unwrap_spec an error Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 11:57:25 2007 @@ -62,6 +62,9 @@ else: name = key + # Because methods always have a receiver, an unwrap_spec of [] is a bug + assert unwrap_spec is None or unwrap_spec + def decorator(func): assert code not in prim_table func.func_name = "prim_" + name @@ -373,8 +376,8 @@ size = unwrap_int(w_size) return shadow.new(size) - at expose_primitive(ARRAY_BECOME_ONE_WAY, unwrap_spec=[]) -def func(interp): + at expose_primitive(ARRAY_BECOME_ONE_WAY, unwrap_spec=[object, object]) +def func(interp, w_obj1, w_obj2): raise PrimitiveNotYetWrittenError @expose_primitive(INST_VAR_AT, unwrap_spec=[object, int]) @@ -393,8 +396,8 @@ return subscript(idx, w_rcvr) raise PrimitiveFailedError() - at expose_primitive(INST_VAR_AT_PUT, unwrap_spec=[]) -def func(interp): + at expose_primitive(INST_VAR_AT_PUT, unwrap_spec=[object]) +def func(interp, w_rcvr): raise PrimitiveNotYetWrittenError() @expose_primitive(AS_OOP, unwrap_spec=[object]) @@ -422,8 +425,8 @@ # it returns the "next" instance after w_obj. raise PrimitiveNotYetWrittenError() - at expose_primitive(NEW_METHOD, unwrap_spec=[]) -def func(interp): + at expose_primitive(NEW_METHOD, unwrap_spec=[object]) +def func(interp, w_mthd): raise PrimitiveNotYetWrittenError() # ___________________________________________________________________________ @@ -447,16 +450,16 @@ def func(interp, w_obj): return w_obj.getclass() - at expose_primitive(BYTES_LEFT, unwrap_spec=[]) -def func(interp): + at expose_primitive(BYTES_LEFT, unwrap_spec=[object]) +def func(interp, w_rcvr): raise PrimitiveNotYetWrittenError() - at expose_primitive(QUIT, unwrap_spec=[]) -def func(interp): + at expose_primitive(QUIT, unwrap_spec=[object]) +def func(interp, w_rcvr): raise PrimitiveNotYetWrittenError() - at expose_primitive(EXIT_TO_DEBUGGER, unwrap_spec=[]) -def func(interp): + at expose_primitive(EXIT_TO_DEBUGGER, unwrap_spec=[object]) +def func(interp, w_rcvr): raise PrimitiveNotYetWrittenError() @expose_primitive(CHANGE_CLASS, unwrap_spec=[object, object]) From tverwaes at codespeak.net Fri Oct 26 12:01:14 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 26 Oct 2007 12:01:14 +0200 (CEST) Subject: [pypy-svn] r48034 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026100114.D354F8140@code0.codespeak.net> Author: tverwaes Date: Fri Oct 26 12:01:14 2007 New Revision: 48034 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: adding milliseconds primitive Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 12:01:14 2007 @@ -506,6 +506,16 @@ rgc.collect() return fake_bytes_left() +#____________________________________________________________________________ +# Time Primitives +MILLISECOND_CLOCK = 135 + + at expose_primitive(MILLISECOND_CLOCK, unwrap_spec=[object]) +def func(interp, w_arg): + import time + import math + return wrap_int(math.fmod(time.time()*1000,1073741823/2)) + # ___________________________________________________________________________ # Boolean Primitives From lukas at codespeak.net Fri Oct 26 12:01:55 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Fri, 26 Oct 2007 12:01:55 +0200 (CEST) Subject: [pypy-svn] r48035 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071026100155.02D708140@code0.codespeak.net> Author: lukas Date: Fri Oct 26 12:01:55 2007 New Revision: 48035 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: added comments to the value: tests Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 12:01:55 2007 @@ -471,24 +471,37 @@ assert res.value == 7 def test_bc_3_plus_4(): - # [ 3+4 ] value + # value0 + # " (self >> #value0) byteCode " + # " (self >> #value0) literals " + # + # ^ [ 3 + 4 ] value interp_bc_and_check_result_is_7( [ 137, 117, 200, 164, 4, 32, 33, 176, 125, 201, 124], fakeliterals(wrap_int(3), wrap_int(4))) def test_bc_x_plus_x_plus_1(): - # [ :x | x+x+1 ] value: 3 + # value1 + # " (self >> #value1) byteCode " + # " (self >> #value1) literals " + # + # ^ [ :x | x + x + 1 ] value: 3 interp_bc_and_check_result_is_7( [ 137, 118, 200, 164, 7, 104, 16, 16, 176, 118, 176, 125, 32, 202, 124 ], fakeliterals(wrap_int(3))) -def test_bc_x_plus_y(): - # [ :x x+y ] value: 3 value: 4 - py.test.skip("FIX") - interp_bc_and_check_result_is_7( - [ 137, 119, 200, 164, 6, 105, 104, 16, 17, - 176, 125, 33, 34, 240, 124 ], - fakeliterals(wrap_int(3), wrap_int(4))) - +# we skip this test, because it requires to lookup the method #value:value: +# in the BlockContext class, which we don't have at the moment +# +# def test_bc_x_plus_y(): +# # value2 +# # " (self >> #value2) byteCode " +# # " (self >> #value2) literals " +# # +# # ^ [ :x :y | x + y ] value: 3 value: 4 +# interp_bc_and_check_result_is_7( +# [ 137, 119, 200, 164, 6, 105, 104, 16, 17, +# 176, 125, 33, 34, 240, 124 ], +# fakeliterals("value:value:", wrap_int(3), wrap_int(4))) \ No newline at end of file From niko at codespeak.net Fri Oct 26 12:07:46 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 12:07:46 +0200 (CEST) Subject: [pypy-svn] r48036 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026100746.461F280C5@code0.codespeak.net> Author: niko Date: Fri Oct 26 12:07:45 2007 New Revision: 48036 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (niko, arigo, lukas) introduce hack to give a value:value: method to the BlockContext during the test bc_x_plus_y() Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 12:07:45 2007 @@ -627,7 +627,6 @@ @expose_primitive(PRIMITIVE_VALUE) def func(interp, argument_count): - # XXX XXX XXX XXX test me # argument_count does NOT include the receiver. # This means that for argument_count == 3 the stack looks like: # 3 2 1 Top Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 12:07:45 2007 @@ -492,16 +492,21 @@ 176, 118, 176, 125, 32, 202, 124 ], fakeliterals(wrap_int(3))) -# we skip this test, because it requires to lookup the method #value:value: -# in the BlockContext class, which we don't have at the moment -# -# def test_bc_x_plus_y(): -# # value2 -# # " (self >> #value2) byteCode " -# # " (self >> #value2) literals " -# # -# # ^ [ :x :y | x + y ] value: 3 value: 4 -# interp_bc_and_check_result_is_7( -# [ 137, 119, 200, 164, 6, 105, 104, 16, 17, -# 176, 125, 33, 34, 240, 124 ], -# fakeliterals("value:value:", wrap_int(3), wrap_int(4))) \ No newline at end of file +def test_bc_x_plus_y(): + # value2 + # " (self >> #value2) byteCode " + # " (self >> #value2) literals " + # + # ^ [ :x :y | x + y ] value: 3 value: 4 + s_BlockContext = ct.w_BlockContext.as_class_get_shadow() + prim_meth = model.W_CompiledMethod( + 0, "", argsize=2, primitive=primitives.PRIMITIVE_VALUE) + s_BlockContext.installmethod("value:value:", prim_meth) + try: + interp_bc_and_check_result_is_7( + [ 137, 119, 200, 164, 6, 105, 104, 16, 17, + 176, 125, 33, 34, 240, 124 ], + fakeliterals("value:value:", wrap_int(3), wrap_int(4))) + finally: + del s_BlockContext.methoddict['value:value:'] # clean up after you + From tverwaes at codespeak.net Fri Oct 26 12:14:48 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 26 Oct 2007 12:14:48 +0200 (CEST) Subject: [pypy-svn] r48037 - pypy/dist/pypy/lang/smalltalk/tool Message-ID: <20071026101448.1A71B814A@code0.codespeak.net> Author: tverwaes Date: Fri Oct 26 12:14:47 2007 New Revision: 48037 Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Log: set back to tinybenchmarks Modified: pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/analyseimage.py Fri Oct 26 12:14:47 2007 @@ -37,6 +37,7 @@ # Should get this from w_object w_smallint_class = image.special(constants.SO_SMALLINTEGER_CLASS) s_class = w_object.shadow_of_my_class() + #w_method = s_class.lookup("benchFib") w_method = s_class.lookup("tinyBenchmarks") assert w_method @@ -65,7 +66,7 @@ counter = 0 sys.stderr.write("#") except interpreter.ReturnFromTopLevel, e: - assert e.object.value == abs(int) + print e.object return except: if hasattr(interp.w_active_context,"currentBytecode"): From oscar at codespeak.net Fri Oct 26 12:16:27 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Fri, 26 Oct 2007 12:16:27 +0200 (CEST) Subject: [pypy-svn] r48038 - pypy/dist/pypy/translator/goal Message-ID: <20071026101627.6A19F814A@code0.codespeak.net> Author: oscar Date: Fri Oct 26 12:16:26 2007 New Revision: 48038 Added: pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py Log: (cfbolz, oscar) added target for image loading Added: pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py Fri Oct 26 12:16:26 2007 @@ -0,0 +1,57 @@ +import autopath +import sys +from pypy.lang.smalltalk import model, interpreter, primitives, shadow +from pypy.lang.smalltalk import objtable +from pypy.lang.smalltalk.objtable import wrap_int +from pypy.lang.smalltalk import classtable +# from pypy.lang.smalltalk.test.test_interpreter import * +from pypy.lang.smalltalk import squeakimage + +mockclass = classtable.bootstrap_class + +def new_interpreter(bytes): + assert isinstance(bytes, str) + w_method = model.W_CompiledMethod(0, bytes=bytes) + w_frame = w_method.create_frame(objtable.w_nil, []) + interp = interpreter.Interpreter() + interp.w_active_context = w_frame + return interp + +def entry_point(argv): + if len(argv) > 1: + filename = argv[1] + else: + print "usage:", argv[0], "" + return -1 + reader = squeakimage.ImageReader(squeakimage.Stream(DummyFile(filename))) + reader.initialize() + image = squeakimage.SqueakImage() + image.from_reader(reader) + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None + +class DummyFile: + def __init__(self,filename): + import os + fd = os.open(filename, os.O_RDONLY, 0777) + try: + content = [] + while 1: + s = os.read(fd, 4096) + if not s: + break + content.append(s) + self.content = "".join(content) + finally: + os.close(fd) + def read(self): + return self.content + def close(self): + pass + +if __name__ == "__main__": + entry_point(sys.argv) From niko at codespeak.net Fri Oct 26 13:29:57 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 13:29:57 +0200 (CEST) Subject: [pypy-svn] r48039 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026112957.103F18122@code0.codespeak.net> Author: niko Date: Fri Oct 26 13:29:56 2007 New Revision: 48039 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (niko, toon, lukas) replace all references to w_receiver with .receiver() so that it works in block contexts too Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 13:29:56 2007 @@ -53,7 +53,7 @@ # push bytecodes def pushReceiverVariableBytecode(self, interp): index = self.currentBytecode & 15 - self.push(self.w_receiver.fetch(index)) + self.push(self.receiver().fetch(index)) def pushTemporaryVariableBytecode(self, interp): index = self.currentBytecode & 15 @@ -74,7 +74,7 @@ def storeAndPopReceiverVariableBytecode(self, interp): index = self.currentBytecode & 7 - self.w_receiver.store(index, self.pop()) + self.receiver().store(index, self.pop()) def storeAndPopTemporaryVariableBytecode(self, interp): index = self.currentBytecode & 7 @@ -82,7 +82,7 @@ # push bytecodes def pushReceiverBytecode(self, interp): - self.push(self.w_receiver) + self.push(self.receiver()) def pushConstantTrueBytecode(self, interp): self.push(interp.TRUE) @@ -124,7 +124,7 @@ def _sendSuperSelector(self, selector, argcount, interp): s_compiledin = self.w_method().compiledin().as_class_get_shadow() - self._sendSelector(selector, argcount, interp, self.w_receiver, + self._sendSelector(selector, argcount, interp, self.receiver(), s_compiledin.s_superclass) def _sendSelector(self, selector, argcount, interp, @@ -149,29 +149,30 @@ interp.w_active_context = method.create_frame(receiver, arguments, self) self.pop_n(argcount + 1) - def _return(self, object, interp): - if self.w_sender is None: # for tests, when returning from the top-level context + def _return(self, object, interp, w_return_to): + # for tests, when returning from the top-level context + if w_return_to is None: raise ReturnFromTopLevel(object) - self.w_sender.push(object) - interp.w_active_context = self.w_sender + w_return_to.push(object) + interp.w_active_context = w_return_to def returnReceiver(self, interp): - self._return(self.w_receiver, interp) + self._return(self.receiver(), interp, self.w_sender) def returnTrue(self, interp): - self._return(interp.TRUE, interp) + self._return(interp.TRUE, interp, self.w_sender) def returnFalse(self, interp): - self._return(interp.FALSE, interp) + self._return(interp.FALSE, interp, self.w_sender) def returnNil(self, interp): - self._return(interp.NIL, interp) + self._return(interp.NIL, interp, self.w_sender) def returnTopFromMethod(self, interp): - self._return(self.top(), interp) + self._return(self.top(), interp, self.w_home.w_sender) def returnTopFromBlock(self, interp): - self._return(self.top(), interp) + self._return(self.top(), interp, self.w_sender) def unknownBytecode(self, interp): raise MissingBytecode @@ -183,7 +184,7 @@ def extendedPushBytecode(self, interp): variableType, variableIndex = self.extendedVariableTypeAndIndex() if variableType == 0: - self.push(self.w_receiver.fetch(variableIndex)) + self.push(self.receiver().fetch(variableIndex)) elif variableType == 1: self.push(self.gettemp(variableIndex)) elif variableType == 2: @@ -196,7 +197,7 @@ def extendedStoreBytecode(self, interp): variableType, variableIndex = self.extendedVariableTypeAndIndex() if variableType == 0: - self.w_receiver.store(variableIndex, self.top()) + self.receiver().store(variableIndex, self.top()) elif variableType == 1: self.settemp(variableIndex, self.top()) elif variableType == 2: @@ -233,7 +234,7 @@ second & 31, interp) elif opType == 2: # pushReceiver - self.push(self.w_receiver.fetch(third)) + self.push(self.receiver().fetch(third)) elif opType == 3: # pushLiteralConstant self.push(self.w_method().getliteral(third)) @@ -243,9 +244,9 @@ assert isinstance(association, model.W_PointersObject) self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) elif opType == 5: - self.w_receiver.store(third, self.top()) + self.receiver().store(third, self.top()) elif opType == 6: - self.w_receiver.store(third, self.pop()) + self.receiver().store(third, self.pop()) elif opType == 7: association = self.w_method().getliteral(third) assert isinstance(association, model.W_PointersObject) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Fri Oct 26 13:29:56 2007 @@ -124,6 +124,10 @@ self._shadow.invalidate() self._vars[n0] = w_value + def fetchvarpointer(self, idx): + instsize = self.getclass().as_class_get_shadow().instsize() + return self._vars[idx+instsize] + def size(self): return len(self._vars) @@ -276,6 +280,10 @@ self.w_home = w_home self.w_sender = w_sender + def receiver(self): + " Return self of the method, or the method that contains the block " + return self.w_home.w_receiver + # ______________________________________________________________________ # Imitate the primitive accessors @@ -296,7 +304,7 @@ # ______________________________________________________________________ # Method that contains the bytecode for this method/block context - + def w_method(self): return self.w_home._w_method Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 13:29:56 2007 @@ -624,6 +624,12 @@ w_new_context = model.W_BlockContext( w_method_context, objtable.w_nil, argcnt, initialip) return w_new_context + +def finalize_block_ctx(interp, w_block_ctx, frame): + # Set some fields + w_block_ctx.pc = w_block_ctx.initialip + w_block_ctx.w_sender = frame + interp.w_active_context = w_block_ctx @expose_primitive(PRIMITIVE_VALUE) def func(interp, argument_count): @@ -650,14 +656,25 @@ w_block_ctx.push_all(block_args) frame.pop() - # Set some fields - w_block_ctx.pc = w_block_ctx.initialip - w_block_ctx.w_sender = frame - interp.w_active_context = w_block_ctx + finalize_block_ctx(interp, w_block_ctx, frame) @expose_primitive(PRIMITIVE_VALUE_WITH_ARGS, unwrap_spec=[object, object]) -def func(interp, w_rcvr, w_args): - raise PrimitiveNotYetWrittenError() +def func(interp, w_block_ctx, w_args): + if not isinstance(w_block_ctx, model.W_BlockContext): + raise PrimitiveFailedError() + exp_arg_cnt = w_block_ctx.expected_argument_count() + + # Check that our arguments have pointers format and the right size: + if not isinstance(w_args, model.W_PointersObject): + raise PrimitiveFailedError() + if w_args.size() != exp_arg_cnt: + raise PrimitiveFailedError() + + # Push all the items from the array + for i in range(exp_arg_cnt): + w_block_ctx.push(w_args.fetchvarpointer(i)) + + finalize_block_ctx(interp, w_block_ctx, interp.w_active_context) @expose_primitive(PRIMITIVE_PERFORM) def func(interp, argument_count): Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 13:29:56 2007 @@ -463,12 +463,11 @@ storeAssociation(doubleExtendedDoAnythingBytecode + chr(7<<5) + chr(0)) -def interp_bc_and_check_result_is_7(bcodes, literals): +def interpret_bc(bcodes, literals): bcode = "".join([chr(x) for x in bcodes]) interp = new_interpreter(bcode) interp.w_active_context.w_method().literals = literals - res = interp.interpret() - assert res.value == 7 + return interp.interpret() def test_bc_3_plus_4(): # value0 @@ -476,9 +475,9 @@ # " (self >> #value0) literals " # # ^ [ 3 + 4 ] value - interp_bc_and_check_result_is_7( + assert interpret_bc( [ 137, 117, 200, 164, 4, 32, 33, 176, 125, 201, 124], - fakeliterals(wrap_int(3), wrap_int(4))) + fakeliterals(wrap_int(3), wrap_int(4))).value == 7 def test_bc_x_plus_x_plus_1(): @@ -487,10 +486,10 @@ # " (self >> #value1) literals " # # ^ [ :x | x + x + 1 ] value: 3 - interp_bc_and_check_result_is_7( + assert interpret_bc( [ 137, 118, 200, 164, 7, 104, 16, 16, 176, 118, 176, 125, 32, 202, 124 ], - fakeliterals(wrap_int(3))) + fakeliterals(wrap_int(3))).value == 7 def test_bc_x_plus_y(): # value2 @@ -498,15 +497,28 @@ # " (self >> #value2) literals " # # ^ [ :x :y | x + y ] value: 3 value: 4 + + # Temporarily introduce a primitive method value:value: that would + # normally be in the loaded image: s_BlockContext = ct.w_BlockContext.as_class_get_shadow() prim_meth = model.W_CompiledMethod( 0, "", argsize=2, primitive=primitives.PRIMITIVE_VALUE) s_BlockContext.installmethod("value:value:", prim_meth) try: - interp_bc_and_check_result_is_7( + assert interpret_bc( [ 137, 119, 200, 164, 6, 105, 104, 16, 17, 176, 125, 33, 34, 240, 124 ], - fakeliterals("value:value:", wrap_int(3), wrap_int(4))) + fakeliterals("value:value:", wrap_int(3), wrap_int(4))).value == 7 finally: del s_BlockContext.methoddict['value:value:'] # clean up after you +def test_bc_push_rcvr_in_block(): + # value1 + # " (self >> #value1) byteCode " + # " (self >> #value1) literals " + # + # ^ [ self ] value + assert interpret_bc( + [ 137, 117, 200, 164, 2, 112, 125, 201, 124 ], + fakeliterals(wrap_int(3))) is objtable.w_nil + From oscar at codespeak.net Fri Oct 26 13:36:50 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Fri, 26 Oct 2007 13:36:50 +0200 (CEST) Subject: [pypy-svn] r48040 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026113650.290308130@code0.codespeak.net> Author: oscar Date: Fri Oct 26 13:36:49 2007 New Revision: 48040 Modified: pypy/dist/pypy/lang/smalltalk/objtable.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (arigo, oscar) translation fixes to objtable and squeakimage Modified: pypy/dist/pypy/lang/smalltalk/objtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/objtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/objtable.py Fri Oct 26 13:36:49 2007 @@ -1,5 +1,5 @@ import pypy.lang.smalltalk.classtable as ct -from pypy.lang.smalltalk.constants import CHARACTER_VALUE_INDEX +from pypy.lang.smalltalk import constants from pypy.lang.smalltalk import model # ___________________________________________________________________________ @@ -24,7 +24,7 @@ def ord_w_char(w_c): assert w_c.getclass() is ct.w_Character - w_ord = w_c.fetch(CHARACTER_VALUE_INDEX) + w_ord = w_c.fetch(constants.CHARACTER_VALUE_INDEX) assert w_ord.getclass() is ct.w_SmallInteger assert isinstance(w_ord, model.W_SmallInteger) return w_ord.value @@ -42,7 +42,7 @@ global CharacterTable def bld_char(i): w_cinst = ct.w_Character.as_class_get_shadow().new() - w_cinst.store(CHARACTER_VALUE_INDEX, wrap_int(i)) + w_cinst.store(constants.CHARACTER_VALUE_INDEX, wrap_int(i)) return w_cinst CharacterTable = [bld_char(i) for i in range(256)] wrap_char_table() @@ -54,3 +54,8 @@ w_zero = wrap_int(0) w_one = wrap_int(1) w_two = wrap_int(2) + +objtable = {} + +for name in constants.objects_in_special_object_table: + objtable["w_" + name] = globals()["w_" + name] Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Fri Oct 26 13:36:49 2007 @@ -1,4 +1,5 @@ import py +import os from pypy.lang.smalltalk import model from pypy.lang.smalltalk import objtable from pypy.rlib import objectmodel @@ -19,11 +20,13 @@ def splitbits(integer, lengths): #XXX we can later let the tool chain mask and unroll this - assert sum(lengths) <= 32 result = [] + sum = 0 for length in lengths: + sum += length result.append(integer & (2**length - 1)) integer = integer >> length + assert sum <= 32 return result @@ -95,7 +98,7 @@ self.stream.swap = True version = self.stream.peek() if version != 0x1966: - raise CorrupImageError + raise CorruptImageError version = self.stream.next() #------ headersize = self.stream.next() @@ -113,7 +116,7 @@ self.stream.reset_count() while self.stream.count < self.endofmemory: chunk, pos = self.read_object() - if len(self.chunklist) % 1000 == 0: sys.stderr.write('#') + if len(self.chunklist) % 1000 == 0: os.write(2,'#') self.chunklist.append(chunk) self.chunks[pos + self.oldbaseaddress] = chunk self.stream.close() @@ -134,11 +137,13 @@ from pypy.lang.smalltalk import classtable, constants, objtable # assign w_objects for objects that are already in classtable for name, so_index in constants.classes_in_special_object_table.items(): - w_object = getattr(classtable, "w_" + name) + # w_object = getattr(classtable, "w_" + name) + w_object = classtable.classtable["w_" + name] self.special_object(so_index).w_object = w_object # assign w_objects for objects that are already in objtable for name, so_index in constants.objects_in_special_object_table.items(): - w_object = getattr(objtable, "w_" + name) + # w_object = getattr(objtable, "w_" + name) + w_object = objtable.objtable["w_" + name] self.special_object(so_index).w_object = w_object def special_object(self, index): Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 13:36:49 2007 @@ -5,6 +5,7 @@ from pypy.lang.smalltalk import interpreter from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import objtable +from pypy.lang.smalltalk import constants from pypy.rlib.rarithmetic import INFINITY, NAN, isinf, isnan # Violates the guideline, but we use it A LOT to reference the primitive codes: @@ -195,11 +196,11 @@ assert prim(primitives.STRING_AT, [test_str, i]) == wrap(exp[i]) def test_object_at(): - w_v = prim(primitives.OBJECT_AT, ["q", objtable.CHARACTER_VALUE_INDEX+1]) + w_v = prim(primitives.OBJECT_AT, ["q", constants.CHARACTER_VALUE_INDEX+1]) assert w_v.value == ord("q") def test_invalid_object_at(): - prim_fails(primitives.OBJECT_AT, ["q", objtable.CHARACTER_VALUE_INDEX+2]) + prim_fails(primitives.OBJECT_AT, ["q", constants.CHARACTER_VALUE_INDEX+2]) def test_object_at_put(): w_obj = mockclass(1).as_class_get_shadow().new() @@ -237,7 +238,7 @@ def test_inst_var_at(): # I am not entirely sure if this is what this primitive is # supposed to do, so the test may be bogus: - w_v = prim(primitives.INST_VAR_AT, ["q", objtable.CHARACTER_VALUE_INDEX]) + w_v = prim(primitives.INST_VAR_AT, ["q", constants.CHARACTER_VALUE_INDEX]) assert w_v.value == ord("q") w_v = prim(primitives.INST_VAR_AT, ["abc", 1]) assert w_v.value == ord("b") From oscar at codespeak.net Fri Oct 26 13:39:10 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Fri, 26 Oct 2007 13:39:10 +0200 (CEST) Subject: [pypy-svn] r48041 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026113910.7C8078130@code0.codespeak.net> Author: oscar Date: Fri Oct 26 13:39:10 2007 New Revision: 48041 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: (arigo, oscar) removed dynamic lambnda filter from assertion Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Fri Oct 26 13:39:10 2007 @@ -258,7 +258,8 @@ if not self.ispointers(): return self.pointers = [self.decode_pointer(pointer) for pointer in chunk.data] - assert len(filter(lambda x: x is None, self.pointers)) == 0 + # assert len(filter(lambda x: x is None, self.pointers)) == 0 + assert None not in self.pointers def decode_pointer(self, pointer): if (pointer & 1) == 1: From niko at codespeak.net Fri Oct 26 13:41:36 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 13:41:36 +0200 (CEST) Subject: [pypy-svn] r48042 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026114136.D41438134@code0.codespeak.net> Author: niko Date: Fri Oct 26 13:41:36 2007 New Revision: 48042 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (niko, lukas) make returnTopFromMethod return to the caller of the home current, not the caller of the CURRENT context Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 13:41:36 2007 @@ -157,16 +157,16 @@ interp.w_active_context = w_return_to def returnReceiver(self, interp): - self._return(self.receiver(), interp, self.w_sender) + self._return(self.receiver(), interp, self.w_home.w_sender) def returnTrue(self, interp): - self._return(interp.TRUE, interp, self.w_sender) + self._return(interp.TRUE, interp, self.w_home.w_sender) def returnFalse(self, interp): - self._return(interp.FALSE, interp, self.w_sender) + self._return(interp.FALSE, interp, self.w_home.w_sender) def returnNil(self, interp): - self._return(interp.NIL, interp, self.w_sender) + self._return(interp.NIL, interp, self.w_home.w_sender) def returnTopFromMethod(self, interp): self._return(self.top(), interp, self.w_home.w_sender) Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 13:41:36 2007 @@ -522,3 +522,13 @@ [ 137, 117, 200, 164, 2, 112, 125, 201, 124 ], fakeliterals(wrap_int(3))) is objtable.w_nil +def test_bc_value_return(): + # valueReturn + # " (self >> #value1) byteCode " + # " (self >> #value1) literals " + # + # [ ^ 1 ] value. ^ 2 + assert interpret_bc( + [ 137, 117, 200, 164, 2, 118, 124, 201, 135, 119, 124 ], + fakeliterals()).value == 1 + From oscar at codespeak.net Fri Oct 26 13:43:55 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Fri, 26 Oct 2007 13:43:55 +0200 (CEST) Subject: [pypy-svn] r48043 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026114355.27F668134@code0.codespeak.net> Author: oscar Date: Fri Oct 26 13:43:54 2007 New Revision: 48043 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: (arigo, oscar) fixed type-checking idiom Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Fri Oct 26 13:43:54 2007 @@ -258,7 +258,6 @@ if not self.ispointers(): return self.pointers = [self.decode_pointer(pointer) for pointer in chunk.data] - # assert len(filter(lambda x: x is None, self.pointers)) == 0 assert None not in self.pointers def decode_pointer(self, pointer): @@ -316,14 +315,13 @@ # below we are using an RPython idiom to 'cast' self.w_object # and pass the casted reference to the fillin_* methods casted = self.w_object - case = casted.__class__ - if case == model.W_PointersObject: + if isinstance(casted, model.W_PointersObject): self.fillin_pointersobject(casted) - elif case == model.W_WordsObject: + elif isinstance(casted, model.W_WordsObject): self.fillin_wordsobject(casted) - elif case == model.W_BytesObject: + elif isinstance(casted, model.W_BytesObject): self.fillin_bytesobject(casted) - elif case == model.W_CompiledMethod: + elif isinstance(casted, model.W_CompiledMethod): self.fillin_compiledmethod(casted) else: assert 0 From oscar at codespeak.net Fri Oct 26 13:49:13 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Fri, 26 Oct 2007 13:49:13 +0200 (CEST) Subject: [pypy-svn] r48044 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026114913.1CF2680FA@code0.codespeak.net> Author: oscar Date: Fri Oct 26 13:49:12 2007 New Revision: 48044 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: (arigo, oscar) don't compile hasattr in invariant Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Fri Oct 26 13:49:12 2007 @@ -325,7 +325,8 @@ self.fillin_compiledmethod(casted) else: assert 0 - assert casted.invariant() + if not objectmodel.we_are_translated(): + assert casted.invariant() def fillin_pointersobject(self, w_pointersobject): assert self.pointers is not None From oscar at codespeak.net Fri Oct 26 13:52:56 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Fri, 26 Oct 2007 13:52:56 +0200 (CEST) Subject: [pypy-svn] r48045 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026115256.6331D80FA@code0.codespeak.net> Author: oscar Date: Fri Oct 26 13:52:56 2007 New Revision: 48045 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: (arigo, oscar) assert positive slice indices Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Fri Oct 26 13:52:56 2007 @@ -362,7 +362,9 @@ bytes.append(chr((each >> 0) & 0xff)) #strange, for example range(4)[:0] returns [] instead of [0,1,2,3]! #hence what we have to write list[:-odd] as list[:len(list)-odd] instead :( - return bytes[:len(bytes)-(self.format & 3)] # omit odd bytes + stop = len(bytes)-(self.format & 3) + assert stop >= 0 + return bytes[:stop] # omit odd bytes def fillin_compiledmethod(self, w_compiledmethod): From niko at codespeak.net Fri Oct 26 13:55:02 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 13:55:02 +0200 (CEST) Subject: [pypy-svn] r48046 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071026115502.8A3D980FA@code0.codespeak.net> Author: niko Date: Fri Oct 26 13:55:01 2007 New Revision: 48046 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (niko, lukas) refactor the faked primitive methods into a helper routine Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 13:55:01 2007 @@ -25,6 +25,22 @@ globals()[name] = make_getter(entry) setup() +def run_with_faked_methods(methods, func): + + # Install faked compiled methods that just invoke the primitive: + for (w_class, primnum, argsize, methname) in methods: + s_class = w_class.as_class_get_shadow() + prim_meth = model.W_CompiledMethod( + 0, "", argsize=argsize, primitive=primnum) + s_class.installmethod(methname, prim_meth) + + try: + func() + finally: + # Uninstall those methods: + for (w_class, _, _, methname) in methods: + s_class = w_class.as_class_get_shadow() + del s_class.methoddict[methname] def fakesymbol(s, _cache={}): try: @@ -260,11 +276,8 @@ assert primitives.unwrap_int(result) == 34 def test_send_to_primitive(): - s_smallintclass = ct.w_SmallInteger.as_class_get_shadow() - prim_meth = model.W_CompiledMethod(0, "", argsize=1, - primitive=primitives.SUBTRACT) - s_smallintclass.installmethod("sub", prim_meth) - try: + + def test(): interp = new_interpreter(sendLiteralSelectorBytecode(1 + 16)) interp.w_active_context.w_method().literals = fakeliterals("foo", "sub") interp.w_active_context.push(wrap_int(50)) @@ -275,8 +288,11 @@ assert len(interp.w_active_context.stack) == 1 w_result = interp.w_active_context.pop() assert primitives.unwrap_int(w_result) == 42 - finally: - del s_smallintclass.methoddict['sub'] # clean up after you + + run_with_faked_methods( + [[ct.w_SmallInteger, primitives.SUBTRACT, + 1, "sub"]], + test) def test_longJumpIfTrue(): interp = new_interpreter(longJumpIfTrue(0) + chr(15) + longJumpIfTrue(0) + chr(15)) @@ -498,19 +514,15 @@ # # ^ [ :x :y | x + y ] value: 3 value: 4 - # Temporarily introduce a primitive method value:value: that would - # normally be in the loaded image: - s_BlockContext = ct.w_BlockContext.as_class_get_shadow() - prim_meth = model.W_CompiledMethod( - 0, "", argsize=2, primitive=primitives.PRIMITIVE_VALUE) - s_BlockContext.installmethod("value:value:", prim_meth) - try: + def test(): assert interpret_bc( [ 137, 119, 200, 164, 6, 105, 104, 16, 17, 176, 125, 33, 34, 240, 124 ], fakeliterals("value:value:", wrap_int(3), wrap_int(4))).value == 7 - finally: - del s_BlockContext.methoddict['value:value:'] # clean up after you + run_with_faked_methods( + [[ct.w_BlockContext, primitives.PRIMITIVE_VALUE, + 2, "value:value:"]], + test) def test_bc_push_rcvr_in_block(): # value1 @@ -532,3 +544,20 @@ [ 137, 117, 200, 164, 2, 118, 124, 201, 135, 119, 124 ], fakeliterals()).value == 1 +def test_bc_value_with_args(): + # valueWithArgs + # " (self >> #value1) byteCode " + # " (self >> #value1) literals " + # + # [ :a :b | a - b ] valueWithArguments: #(3 2) + py.test.skip("in progress") + def test(): + assert interpret_bc( + [ 137, 119, 200, 164, 6, 105, 104, 16, 17, 177, 125, 33, 224, 124 ], + fakeliterals("valueWithArguments:", + [3, 2])).value == 1 + run_with_faked_methods( + [[ct.w_BlockContext, primitives.PRIMITIVE_VALUE_WITH_ARGS, + 1, "valueWithArguments:"]], + test) + From tverwaes at codespeak.net Fri Oct 26 13:57:22 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 26 Oct 2007 13:57:22 +0200 (CEST) Subject: [pypy-svn] r48047 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026115722.EE64D80FA@code0.codespeak.net> Author: tverwaes Date: Fri Oct 26 13:57:22 2007 New Revision: 48047 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: make sure we have ints in MILLISECOND Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 13:57:22 2007 @@ -514,7 +514,7 @@ def func(interp, w_arg): import time import math - return wrap_int(math.fmod(time.time()*1000,1073741823/2)) + return wrap_int(int(math.fmod(time.time()*1000,1073741823/2))) # ___________________________________________________________________________ # Boolean Primitives From oscar at codespeak.net Fri Oct 26 14:01:08 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Fri, 26 Oct 2007 14:01:08 +0200 (CEST) Subject: [pypy-svn] r48048 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026120108.9391C8141@code0.codespeak.net> Author: oscar Date: Fri Oct 26 14:01:07 2007 New Revision: 48048 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: (arigo, oscar) fixed another slice assertion Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Fri Oct 26 14:01:07 2007 @@ -24,12 +24,13 @@ sum = 0 for length in lengths: sum += length - result.append(integer & (2**length - 1)) + n = integer & (2**length - 1) + assert n >= 0 + result.append(n) integer = integer >> length assert sum <= 32 return result - # ____________________________________________________________ # # Reads an image file and creates all model objects @@ -121,7 +122,7 @@ self.chunks[pos + self.oldbaseaddress] = chunk self.stream.close() self.swap = self.stream.swap #save for later - del self.stream + self.stream = None return self.chunklist # return for testing def init_g_objects(self): From oscar at codespeak.net Fri Oct 26 14:04:12 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Fri, 26 Oct 2007 14:04:12 +0200 (CEST) Subject: [pypy-svn] r48049 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026120412.8BB1A80B3@code0.codespeak.net> Author: oscar Date: Fri Oct 26 14:04:12 2007 New Revision: 48049 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: (cbolz, oscar) replaced 2**n by 1<= 0 result.append(n) integer = integer >> length From niko at codespeak.net Fri Oct 26 14:15:56 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 14:15:56 +0200 (CEST) Subject: [pypy-svn] r48050 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026121556.B482880B3@code0.codespeak.net> Author: niko Date: Fri Oct 26 14:15:56 2007 New Revision: 48050 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (niko, lukas) 1. improved fake_literal to handle arrays 2. make value_with_args test pass Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 14:15:56 2007 @@ -36,7 +36,9 @@ next = self.w_active_context.getNextBytecode() bytecodeimpl = BYTECODE_TABLE[next] if option.bc_trace: - print "About to execute bytecode %s:" % (bytecodeimpl.__name__,) + print "About to execute bytecode at %d (%d:%s):" % ( + self.w_active_context.pc, + next, bytecodeimpl.__name__,) print " Stack=%s" % (repr(self.w_active_context.stack),) bytecodeimpl(self.w_active_context, self) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Fri Oct 26 14:15:56 2007 @@ -128,6 +128,10 @@ instsize = self.getclass().as_class_get_shadow().instsize() return self._vars[idx+instsize] + def storevarpointer(self, idx, value): + instsize = self.getclass().as_class_get_shadow().instsize() + self._vars[idx+instsize] = value + def size(self): return len(self._vars) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 14:15:56 2007 @@ -665,7 +665,7 @@ exp_arg_cnt = w_block_ctx.expected_argument_count() # Check that our arguments have pointers format and the right size: - if not isinstance(w_args, model.W_PointersObject): + if w_args.getclass() != classtable.w_Array: raise PrimitiveFailedError() if w_args.size() != exp_arg_cnt: raise PrimitiveFailedError() Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 14:15:56 2007 @@ -50,11 +50,20 @@ return result def fakeliterals(*literals): - lst = ["methodheader"] + list(literals) - for i in range(len(lst)): - if isinstance(lst[i], str): - lst[i] = fakesymbol(lst[i]) - return lst + def fakeliteral(lit): + if isinstance(lit, str): + return fakesymbol(lit) + elif isinstance(lit, int): + return wrap_int(lit) + elif isinstance(lit, list): + lstlen = len(lit) + res = ct.w_Array.as_class_get_shadow().new(lstlen) + for i in range(lstlen): + res.storevarpointer(i, fakeliteral(lit[i])) + return res + return lit + + return ["methodheader"] + [fakeliteral(lit) for lit in literals] def new_interpreter(bytes, receiver=objtable.w_nil): assert isinstance(bytes, str) @@ -550,10 +559,11 @@ # " (self >> #value1) literals " # # [ :a :b | a - b ] valueWithArguments: #(3 2) - py.test.skip("in progress") def test(): assert interpret_bc( - [ 137, 119, 200, 164, 6, 105, 104, 16, 17, 177, 125, 33, 224, 124 ], + [ 137, 119, 200, 164, 6, + 105, 104, 16, 17, 177, + 125, 33, 224, 124 ], fakeliterals("valueWithArguments:", [3, 2])).value == 1 run_with_faked_methods( From niko at codespeak.net Fri Oct 26 14:20:09 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 14:20:09 +0200 (CEST) Subject: [pypy-svn] r48051 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026122009.840F880B3@code0.codespeak.net> Author: niko Date: Fri Oct 26 14:20:09 2007 New Revision: 48051 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: add a brief docstring to "subscript", rename to "w_subscript" to reflect that it returns a wrapped object, and swap the order of the argument to match smalltalk's defn Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 14:20:09 2007 @@ -17,8 +17,12 @@ elif isinstance(w_v, model.W_SmallInteger): return float(w_v.value) raise PrimitiveFailedError() -def subscript(idx, w_obj): - # XXX what does this do? explain +def w_subscript(w_obj, idx): + """ + Rather cryptically named function which retrieves an indexed field + from the object, wrapping as necessary depending on the format of + the object so that the result can be returned. + """ if isinstance(w_obj, model.W_PointersObject): return w_obj.fetch(idx) elif isinstance(w_obj, model.W_WordsObject): @@ -393,7 +397,7 @@ return w_rcvr.fetch(idx) idx -= shadow.instsize() if idx < w_rcvr.size(): - return subscript(idx, w_rcvr) + return w_subscript(w_rcvr, idx) raise PrimitiveFailedError() @expose_primitive(INST_VAR_AT_PUT, unwrap_spec=[object]) From niko at codespeak.net Fri Oct 26 14:21:41 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 14:21:41 +0200 (CEST) Subject: [pypy-svn] r48052 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071026122141.91EDB80BC@code0.codespeak.net> Author: niko Date: Fri Oct 26 14:21:41 2007 New Revision: 48052 Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: remove out of date comment line Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 14:21:41 2007 @@ -7,8 +7,6 @@ from pypy.lang.smalltalk import objtable from pypy.lang.smalltalk import constants from pypy.rlib.rarithmetic import INFINITY, NAN, isinf, isnan - -# Violates the guideline, but we use it A LOT to reference the primitive codes: from pypy.lang.smalltalk import primitives mockclass = classtable.bootstrap_class From cfbolz at codespeak.net Fri Oct 26 14:27:09 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 26 Oct 2007 14:27:09 +0200 (CEST) Subject: [pypy-svn] r48053 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026122709.4A71580BE@code0.codespeak.net> Author: cfbolz Date: Fri Oct 26 14:27:09 2007 New Revision: 48053 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: turn the bytecode loop into a C switch (instead of a list lookup and an indirect call). should help perfomance a bit. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 14:27:09 2007 @@ -3,6 +3,7 @@ from pypy.lang.smalltalk import objtable from pypy.lang.smalltalk.model import W_ContextPart from pypy.lang.smalltalk.conftest import option +from pypy.rlib import objectmodel, unroll class MissingBytecode(NotImplementedError): @@ -24,7 +25,6 @@ def __init__(self): self.w_active_context = None - def interpret(self): try: while True: @@ -34,13 +34,20 @@ def step(self): next = self.w_active_context.getNextBytecode() - bytecodeimpl = BYTECODE_TABLE[next] - if option.bc_trace: - print "About to execute bytecode at %d (%d:%s):" % ( - self.w_active_context.pc, - next, bytecodeimpl.__name__,) - print " Stack=%s" % (repr(self.w_active_context.stack),) - bytecodeimpl(self.w_active_context, self) + if not objectmodel.we_are_translated(): + bytecodeimpl = BYTECODE_TABLE[next] + if option.bc_trace: + print "About to execute bytecode at %d (%d:%s):" % ( + self.w_active_context.pc, + next, bytecodeimpl.__name__,) + print " Stack=%s" % (repr(self.w_active_context.stack),) + bytecodeimpl(self.w_active_context, self) + else: + for code, bytecodeimpl in unrolling_bytecode_table: + if code == next: + bytecodeimpl(self.w_active_context, self) + break + class ReturnFromTopLevel(Exception): def __init__(self, object): @@ -482,6 +489,7 @@ (208, 255, W_ContextPart.sendLiteralSelectorBytecode), ] + def initialize_bytecode_table(): result = [None] * 256 for entry in BYTECODE_RANGES: @@ -495,3 +503,4 @@ return result BYTECODE_TABLE = initialize_bytecode_table() +unrolling_bytecode_table = unroll.unrolling_iterable(enumerate(BYTECODE_TABLE)) Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Fri Oct 26 14:27:09 2007 @@ -18,18 +18,22 @@ first = first - 0x100 return first << 24 | ord(b[2]) << 16 | ord(b[1]) << 8 | ord(b[0]) -def splitbits(integer, lengths): - #XXX we can later let the tool chain mask and unroll this - result = [] - sum = 0 - for length in lengths: - sum += length - n = integer & ((1<= 0 - result.append(n) - integer = integer >> length - assert sum <= 32 - return result +def make_bit_splitter(lengths): + from pypy.rlib import unroll + iterator = unroll.unrolling_iterable(lengths) + def splitbits(integer): + result = [] + sum = 0 + for length in iterator: + sum += length + n = integer & ((1<= 0 + result.append(n) + integer = integer >> length + assert sum <= 32 + return result + splitbits.func_name = "split_bits_" + "_".join([str(i) for str in lengths]) + return splitbits # ____________________________________________________________ # @@ -180,19 +184,21 @@ for _ in range(size - 1)] #size-1, excluding header return chunk, pos + split_header = make_bit_splitter([2,6,4,5,12]) def read_1wordobjectheader(self): kind, size, format, classid, idhash = ( - splitbits(self.stream.next(), [2,6,4,5,12])) + self.split_header(self.stream.next())) assert kind == 3 return ImageChunk(size, format, classid, idhash), self.stream.count - 4 def read_2wordobjectheader(self): assert self.stream.peek() & 3 == 1 #kind classid = self.stream.next() - 01 # remove headertype to get pointer - kind, size, format, _, idhash = splitbits(self.stream.next(), [2,6,4,5,12]) + kind, size, format, _, idhash = self.split_header(f.stream.next(), [2,6,4,5,12]) assert kind == 1 return ImageChunk(size, format, classid, idhash), self.stream.count - 4 + split_2_30 = make_bit_splitter([2, 30]) def read_3wordobjectheader(self): kind, size = splitbits(self.stream.next(), [2,30]) assert kind == 0 From cfbolz at codespeak.net Fri Oct 26 14:32:24 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 26 Oct 2007 14:32:24 +0200 (CEST) Subject: [pypy-svn] r48054 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026123224.7C9A7813A@code0.codespeak.net> Author: cfbolz Date: Fri Oct 26 14:32:24 2007 New Revision: 48054 Added: pypy/dist/pypy/lang/smalltalk/squeakimage.py - copied unchanged from r48052, pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: revert accidental checkin From oscar at codespeak.net Fri Oct 26 14:36:18 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Fri, 26 Oct 2007 14:36:18 +0200 (CEST) Subject: [pypy-svn] r48055 - pypy/dist/pypy/translator/goal Message-ID: <20071026123618.2AB91813A@code0.codespeak.net> Author: oscar Date: Fri Oct 26 14:36:17 2007 New Revision: 48055 Modified: pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py Log: (arigo, oscar)image loading with benchmarks Modified: pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py (original) +++ pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py Fri Oct 26 14:36:17 2007 @@ -1,11 +1,13 @@ import autopath import sys +import os from pypy.lang.smalltalk import model, interpreter, primitives, shadow from pypy.lang.smalltalk import objtable from pypy.lang.smalltalk.objtable import wrap_int from pypy.lang.smalltalk import classtable # from pypy.lang.smalltalk.test.test_interpreter import * from pypy.lang.smalltalk import squeakimage +from pypy.lang.smalltalk import constants mockclass = classtable.bootstrap_class @@ -17,6 +19,46 @@ interp.w_active_context = w_frame return interp + + +def tinyBenchmarks(image): + interp = interpreter.Interpreter() + + w_object = model.W_SmallInteger(0) + + # Should get this from w_object + w_smallint_class = image.special(constants.SO_SMALLINTEGER_CLASS) + s_class = w_object.shadow_of_my_class() + w_method = s_class.lookup("tinyBenchmarks") + + assert w_method + w_frame = w_method.create_frame(w_object, []) + interp.w_active_context = w_frame + + print w_method + print "Going to execute %d toplevel bytecodes" % (len(w_method.bytes),) + counter = 0 + + from pypy.lang.smalltalk.interpreter import BYTECODE_TABLE + return interp + + +def run_benchmarks(interp): + counter = 0 + try: + while True: + counter += 1 + interp.step() + if counter == 100000: + counter = 0 + os.write(2, '#') + except interpreter.ReturnFromTopLevel, e: + w_result = e.object + + assert isinstance(w_result, model.W_BytesObject) + print w_result.as_string() + return 0 + def entry_point(argv): if len(argv) > 1: filename = argv[1] @@ -27,6 +69,8 @@ reader.initialize() image = squeakimage.SqueakImage() image.from_reader(reader) + interp = tinyBenchmarks(image) + run_benchmarks(interp) return 0 # _____ Define and setup target ___ From arigo at codespeak.net Fri Oct 26 14:49:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 26 Oct 2007 14:49:15 +0200 (CEST) Subject: [pypy-svn] r48056 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026124915.AE25B8139@code0.codespeak.net> Author: arigo Date: Fri Oct 26 14:49:15 2007 New Revision: 48056 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py Log: Print which bytecode is missing. Useful in translated versions. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 14:49:15 2007 @@ -6,8 +6,11 @@ from pypy.rlib import objectmodel, unroll -class MissingBytecode(NotImplementedError): +class MissingBytecode(Exception): """Bytecode not implemented yet.""" + def __init__(self, bytecodename): + self.bytecodename = bytecodename + print "MissingBytecode:", bytecodename # hack for debugging class IllegalStoreError(Exception): """Illegal Store.""" @@ -184,7 +187,7 @@ self._return(self.top(), interp, self.w_sender) def unknownBytecode(self, interp): - raise MissingBytecode + raise MissingBytecode("unknownBytecode") def extendedVariableTypeAndIndex(self): descriptor = self.getByte() @@ -275,7 +278,7 @@ self.pop() def experimentalBytecode(self, interp): - raise MissingBytecode + raise MissingBytecode("experimentalBytecode") def jump(self,offset): self.pc = self.pc + offset @@ -348,7 +351,7 @@ self.callPrimitiveAndPush(primitives.MOD, "\\", 1, interp) def bytecodePrimMakePoint(self, interp): - raise MissingBytecode + raise MissingBytecode("bytecodePrimMakePoint") def bytecodePrimBitShift(self, interp): self.callPrimitiveAndPush(primitives.BIT_SHIFT, "bitShift:", 1, interp) @@ -372,16 +375,16 @@ self.callPrimitiveAndPush(primitives.SIZE, "size", 0, interp) def bytecodePrimNext(self, interp): - raise MissingBytecode + raise MissingBytecode("bytecodePrimNext") def bytecodePrimNextPut(self, interp): - raise MissingBytecode + raise MissingBytecode("bytecodePrimNextPut") def bytecodePrimAtEnd(self, interp): - raise MissingBytecode + raise MissingBytecode("bytecodePrimAtEnd") def bytecodePrimEquivalent(self, interp): - raise MissingBytecode + raise MissingBytecode("bytecodePrimEquivalent") def bytecodePrimClass(self, interp): self.callPrimitiveAndPush( From niko at codespeak.net Fri Oct 26 14:49:20 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 14:49:20 +0200 (CEST) Subject: [pypy-svn] r48057 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026124920.53E678140@code0.codespeak.net> Author: niko Date: Fri Oct 26 14:49:20 2007 New Revision: 48057 Added: pypy/dist/pypy/lang/smalltalk/find_untested_prims.sh (contents, props changed) Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (niko, lukas) implement tests for CLASS and MULTIPLY primitives Added: pypy/dist/pypy/lang/smalltalk/find_untested_prims.sh ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/find_untested_prims.sh Fri Oct 26 14:49:20 2007 @@ -0,0 +1,5 @@ +#!/bin/sh +grep '^[A-Z][A-Z_]' primitives.py | awk '{print $1;}' | sort | uniq > pcodes.1 +grep 'primitives.[A-Z][A-Z_]' test/test_primitives.py | sed 's/.*primitives.\([A-Z_]*\).*/\1/g' | sort | uniq > pcodes.2 +echo Unimplemented primitive codes: +diff pcodes.1 pcodes.2 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 14:49:20 2007 @@ -62,7 +62,6 @@ res.storevarpointer(i, fakeliteral(lit[i])) return res return lit - return ["methodheader"] + [fakeliteral(lit) for lit in literals] def new_interpreter(bytes, receiver=objtable.w_nil): Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 14:49:20 2007 @@ -59,6 +59,15 @@ def test_small_int_minus_fail(): prim_fails(primitives.SUBTRACT, [-1073741823,2]) +def test_small_int_multiply(): + assert prim(primitives.MULTIPLY, [6,3]).value == 18 + +def test_small_int_multiply_overflow(): + prim_fails(primitives.MULTIPLY, [1073741823, 2]) + prim_fails(primitives.MULTIPLY, [1073741823, 1073741823]) + prim_fails(primitives.MULTIPLY, [1073741823, -4]) + prim_fails(primitives.MULTIPLY, [-1073741823, 2]) + def test_small_int_divide(): assert prim(primitives.DIVIDE, [6,3]).value == 2 @@ -164,7 +173,6 @@ assert prim(primitives.FLOAT_TRUNCATED, [4.5]).value == 4 assert prim(primitives.FLOAT_TRUNCATED, [4.6]).value == 4 - def test_at(): w_obj = mockclass(0, varsized=True).as_class_get_shadow().new(1) w_obj.store(0, "foo") @@ -241,6 +249,10 @@ w_v = prim(primitives.INST_VAR_AT, ["abc", 1]) assert w_v.value == ord("b") +def test_class(): + assert prim(primitives.CLASS, ["string"]) == classtable.w_String + assert prim(primitives.CLASS, [1]) == classtable.w_SmallInteger + def test_as_oop(): py.test.skip("not yet clear what AS_OOP returns: hash or header?") w_obj = mockclass(0).as_class_get_shadow().new() @@ -359,3 +371,8 @@ self should: [1 become: 2] raise: Error. """ + +# Note: +# primitives.PRIMITIVE_BLOCK_COPY is tested in test_interpreter +# primitives.PRIMITIVE_VALUE is tested in test_interpreter +# primitives.PRIMITIVE_VALUE_WITH_ARGS is tested in test_interpreter From tverwaes at codespeak.net Fri Oct 26 14:58:26 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 26 Oct 2007 14:58:26 +0200 (CEST) Subject: [pypy-svn] r48058 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026125826.0885C80C8@code0.codespeak.net> Author: tverwaes Date: Fri Oct 26 14:58:25 2007 New Revision: 48058 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py Log: fixing fallback for stream primitives Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 14:58:25 2007 @@ -375,13 +375,13 @@ self.callPrimitiveAndPush(primitives.SIZE, "size", 0, interp) def bytecodePrimNext(self, interp): - raise MissingBytecode("bytecodePrimNext") + self.callPrimitiveAndPush(primitives.NEXT, "next", 0, interp) def bytecodePrimNextPut(self, interp): - raise MissingBytecode("bytecodePrimNextPut") + self.callPrimitiveAndPush(primitives.NEXT_PUT, "nextPut:", 1, interp) def bytecodePrimAtEnd(self, interp): - raise MissingBytecode("bytecodePrimAtEnd") + self.callPrimitiveAndPush(primitives.AT_END, "atEnd", 0, interp) def bytecodePrimEquivalent(self, interp): raise MissingBytecode("bytecodePrimEquivalent") Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Fri Oct 26 14:58:25 2007 @@ -116,6 +116,9 @@ W_AbstractObjectWithClassReference.__init__(self, w_class) self._vars = [None] * size + def at(self, index0): + return self.fetch(index0) + def fetch(self, n0): return self._vars[n0] @@ -152,7 +155,11 @@ def __init__(self, w_class, size): W_AbstractObjectWithClassReference.__init__(self, w_class) self.bytes = ['\x00'] * size - + + def at(self, index0): + from pypy.lang.smalltalk import objtable + return objtable.wrap_chr(self.getbyte(index0)) + def getbyte(self, n): return ord(self.bytes[n]) @@ -184,6 +191,10 @@ W_AbstractObjectWithClassReference.__init__(self, w_class) self.words = [0] * size + def at(self, index0): + from pypy.lang.smalltalk import objtable + return objtable.wrap_int(self.getword(index0)) + def getword(self, n): return self.words[n] Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 14:58:25 2007 @@ -337,6 +337,13 @@ return w_val # ___________________________________________________________________________ +# Stream Primitives + +NEXT = 65 +NEXT_PUT = 66 +AT_END = 67 + +# ___________________________________________________________________________ # Storage Management Primitives OBJECT_AT = 68 From oscar at codespeak.net Fri Oct 26 15:05:23 2007 From: oscar at codespeak.net (oscar at codespeak.net) Date: Fri, 26 Oct 2007 15:05:23 +0200 (CEST) Subject: [pypy-svn] r48059 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026130523.941BD80CB@code0.codespeak.net> Author: oscar Date: Fri Oct 26 15:05:22 2007 New Revision: 48059 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py Log: (toon, oscar) bytecodePrimEquivalent Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 15:05:22 2007 @@ -384,7 +384,7 @@ self.callPrimitiveAndPush(primitives.AT_END, "atEnd", 0, interp) def bytecodePrimEquivalent(self, interp): - raise MissingBytecode("bytecodePrimEquivalent") + self.callPrimitiveAndPush(primitives.EQUIVALENT, "==", 1, interp) def bytecodePrimClass(self, interp): self.callPrimitiveAndPush( From niko at codespeak.net Fri Oct 26 15:07:57 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 15:07:57 +0200 (CEST) Subject: [pypy-svn] r48060 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026130757.5BE9080CB@code0.codespeak.net> Author: niko Date: Fri Oct 26 15:07:57 2007 New Revision: 48060 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/objtable.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_model.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (niko, lukas, arigo) 1. initialize all object data to w_nil, not None 2. fix inst_var_at, inst_var_at_put and write better tests Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Fri Oct 26 15:07:57 2007 @@ -3,6 +3,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.lang.smalltalk import constants from pypy.tool.pairtype import extendabletype +from pypy.rlib.objectmodel import instantiate class W_Object(object): __slots__ = () # no RPython-level instance variables allowed in W_Object @@ -114,7 +115,7 @@ def __init__(self, w_class, size): W_AbstractObjectWithClassReference.__init__(self, w_class) - self._vars = [None] * size + self._vars = [w_nil] * size def at(self, index0): return self.fetch(index0) @@ -427,11 +428,10 @@ return w_MethodContext def fetch(self, index): - from pypy.lang.smalltalk import objtable if index == constants.MTHDCTX_METHOD: return self.w_method() elif index == constants.MTHDCTX_RECEIVER_MAP: # what is this thing? - return objtable.w_nil + return w_nil elif index == constants.MTHDCTX_RECEIVER: return self.w_receiver elif index >= constants.MTHDCTX_TEMP_FRAME_START: @@ -447,3 +447,8 @@ else: return W_ContextPart.fetch(self, index) +# Use black magic to create w_nil without running the constructor, +# thus allowing it to be used even in the constructor of its own +# class. Note that we patch its class in objtable. +w_nil = instantiate(W_PointersObject) +w_nil._vars = [] Modified: pypy/dist/pypy/lang/smalltalk/objtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/objtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/objtable.py Fri Oct 26 15:07:57 2007 @@ -49,12 +49,17 @@ w_true = ct.classtable['w_True'].as_class_get_shadow().new() w_false = ct.classtable['w_False'].as_class_get_shadow().new() -w_nil = ct.classtable['w_UndefinedObject'].as_class_get_shadow().new() w_mone = wrap_int(-1) w_zero = wrap_int(0) w_one = wrap_int(1) w_two = wrap_int(2) +# Very special nil hack: in order to allow W_PointersObject's to +# initialize their fields to nil, we have to create it in the model +# package, and then patch up its fields here: +w_nil = model.w_nil +w_nil.w_class = ct.classtable['w_UndefinedObject'] + objtable = {} for name in constants.objects_in_special_object_table: Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 15:07:57 2007 @@ -393,23 +393,22 @@ @expose_primitive(INST_VAR_AT, unwrap_spec=[object, int]) def func(interp, w_rcvr, idx): - # I *think* this is the correct behavior, but I'm not quite sure. - # Might be restricted to fixed length fields? - # XXX this doesn't look correct. Our guess is that INST_VAR_AT - # is used to access *only* the fixed length fields. + "Fetches a fixed field from the object, and fails otherwise" shadow = w_rcvr.shadow_of_my_class() - if idx < 0: - raise PrimitiveFailedError() - if idx < shadow.instsize(): - return w_rcvr.fetch(idx) - idx -= shadow.instsize() - if idx < w_rcvr.size(): - return w_subscript(w_rcvr, idx) - raise PrimitiveFailedError() - - at expose_primitive(INST_VAR_AT_PUT, unwrap_spec=[object]) -def func(interp, w_rcvr): - raise PrimitiveNotYetWrittenError() + assert_bounds(idx, 0, shadow.instsize()) + # only pointers have non-0 size + assert isinstance(w_rcvr, model.W_PointersObject) + return w_rcvr.fetch(idx) + + at expose_primitive(INST_VAR_AT_PUT, unwrap_spec=[object, int, object]) +def func(interp, w_rcvr, idx, w_value): + "Stores a value into a fixed field from the object, and fails otherwise" + shadow = w_rcvr.shadow_of_my_class() + assert_bounds(idx, 0, shadow.instsize()) + # only pointers have non-0 size + assert isinstance(w_rcvr, model.W_PointersObject) + w_rcvr.store(idx, w_value) + return w_value @expose_primitive(AS_OOP, unwrap_spec=[object]) def func(interp, w_rcvr): Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 15:07:57 2007 @@ -174,7 +174,7 @@ if test_index == index: assert w_object.fetch(test_index) == interp.TRUE else: - assert w_object.fetch(test_index) == None + assert w_object.fetch(test_index) is objtable.w_nil def test_storeAndPopTemporaryVariableBytecode(bytecode=storeAndPopTemporaryVariableBytecode): for index in range(8): Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Fri Oct 26 15:07:57 2007 @@ -1,5 +1,5 @@ import py -from pypy.lang.smalltalk import model, shadow +from pypy.lang.smalltalk import model, shadow, objtable from pypy.lang.smalltalk.shadow import MethodNotFound import pypy.lang.smalltalk.classtable as ct @@ -17,7 +17,7 @@ w_myinstance = w_mycls.as_class_get_shadow().new() assert isinstance(w_myinstance, model.W_PointersObject) assert w_myinstance.getclass() is w_mycls - assert w_myinstance.fetch(0) is None + assert w_myinstance.fetch(0) is objtable.w_nil py.test.raises(IndexError, lambda: w_myinstance.fetch(3)) w_myinstance.store(1, w_myinstance) assert w_myinstance.fetch(1) is w_myinstance Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 15:07:57 2007 @@ -242,12 +242,16 @@ prim_fails(primitives.NEW_WITH_ARG, [w_Object, 20]) def test_inst_var_at(): - # I am not entirely sure if this is what this primitive is - # supposed to do, so the test may be bogus: w_v = prim(primitives.INST_VAR_AT, ["q", constants.CHARACTER_VALUE_INDEX]) assert w_v.value == ord("q") - w_v = prim(primitives.INST_VAR_AT, ["abc", 1]) - assert w_v.value == ord("b") + +def test_inst_var_at_put(): + w_q = classtable.w_Character.as_class_get_shadow().new() + vidx = constants.CHARACTER_VALUE_INDEX + ordq = ord("q") + assert prim(primitives.INST_VAR_AT, [w_q, vidx]) == objtable.w_nil + assert prim(primitives.INST_VAR_AT_PUT, [w_q, vidx, ordq]).value == ordq + assert prim(primitives.INST_VAR_AT, [w_q, vidx]).value == ordq def test_class(): assert prim(primitives.CLASS, ["string"]) == classtable.w_String From niko at codespeak.net Fri Oct 26 15:31:18 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 15:31:18 +0200 (CEST) Subject: [pypy-svn] r48061 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026133118.A93858124@code0.codespeak.net> Author: niko Date: Fri Oct 26 15:31:17 2007 New Revision: 48061 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (niko, toon, arigo) 1. fix AT, AT_PUT to be 0-based 2. rename at to be at0, etc Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Fri Oct 26 15:31:17 2007 @@ -117,9 +117,12 @@ W_AbstractObjectWithClassReference.__init__(self, w_class) self._vars = [w_nil] * size - def at(self, index0): + def at0(self, index0): return self.fetch(index0) - + + def atput0(self, index0, w_value): + self.store(index0, w_value) + def fetch(self, n0): return self._vars[n0] @@ -157,10 +160,13 @@ W_AbstractObjectWithClassReference.__init__(self, w_class) self.bytes = ['\x00'] * size - def at(self, index0): + def at0(self, index0): from pypy.lang.smalltalk import objtable - return objtable.wrap_chr(self.getbyte(index0)) + return objtable.wrap_int(self.getbyte(index0)) + def atput0(self, index0, w_value): + self.setbyte(index0, unwrap_int(w_value)) + def getbyte(self, n): return ord(self.bytes[n]) @@ -192,9 +198,12 @@ W_AbstractObjectWithClassReference.__init__(self, w_class) self.words = [0] * size - def at(self, index0): + def at0(self, index0): from pypy.lang.smalltalk import objtable return objtable.wrap_int(self.getword(index0)) + + def atput0(self, index0, w_value): + self.setword(index0, unwrap_int(w_value)) def getword(self, n): return self.words[n] Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 15:31:17 2007 @@ -294,26 +294,17 @@ STRING_AT = 63 STRING_AT_PUT = 64 -def common_at(w_obj, w_index1): - index1 = unwrap_int(w_index1) - assert_valid_index(index1-1, w_obj) - return w_obj, index1-1 - -def common_at_put(w_obj, w_idx, w_val): - idx = unwrap_int(w_idx) - assert_valid_index(idx-1, w_obj) - return w_obj, idx-1, w_val - @expose_primitive(AT, unwrap_spec=[object, int]) def func(interp, w_obj, n1): n0 = n1 - 1 assert_valid_index(n0, w_obj) - return w_obj.fetch(n0) + return w_obj.at0(n0) - at expose_primitive(AT_PUT, unwrap_spec=[object, object, object]) -def func(interp, w_obj, w_idx, w_val): - [w_obj, idx, w_val] = common_at_put(w_obj, w_idx, w_val) - w_obj.store(idx, w_val) + at expose_primitive(AT_PUT, unwrap_spec=[object, int, object]) +def func(interp, w_obj, n1, w_val): + n0 = n1 - 1 + assert_valid_index(n0, w_obj) + w_obj.atput0(n0, w_val) return w_val @expose_primitive(SIZE, unwrap_spec=[object]) @@ -322,18 +313,20 @@ raise PrimitiveFailedError() return wrap_int(w_obj.size()) - at expose_primitive(STRING_AT, unwrap_spec=[object, object]) -def func(interp, w_obj, w_idx): - w_obj, idx = common_at(w_obj, w_idx) - byte = w_obj.getbyte(idx) + at expose_primitive(STRING_AT, unwrap_spec=[object, int]) +def func(interp, w_obj, n1): + n0 = n1 - 1 + assert_valid_index(n0, w_obj) + byte = w_obj.getbyte(n0) return objtable.CharacterTable[byte] - at expose_primitive(STRING_AT_PUT, unwrap_spec=[object, object, object]) -def func(interp, w_obj, w_idx, w_val): - w_obj, idx, w_val = common_at_put(w_obj, w_idx, w_val) + at expose_primitive(STRING_AT_PUT, unwrap_spec=[object, int, object]) +def func(interp, w_obj, n1, w_val): + n0 = n1 - 1 + assert_valid_index(n0, w_obj) if w_val.getclass() is not classtable.w_Character: raise PrimitiveFailedError() - w_obj.setbyte(idx, objtable.ord_w_char(w_val)) + w_obj.setbyte(n0, objtable.ord_w_char(w_val)) return w_val # ___________________________________________________________________________ @@ -392,22 +385,24 @@ raise PrimitiveNotYetWrittenError @expose_primitive(INST_VAR_AT, unwrap_spec=[object, int]) -def func(interp, w_rcvr, idx): +def func(interp, w_rcvr, idx_1): "Fetches a fixed field from the object, and fails otherwise" shadow = w_rcvr.shadow_of_my_class() - assert_bounds(idx, 0, shadow.instsize()) - # only pointers have non-0 size + idx_0 = idx_1 - 1 + assert_bounds(idx_0, 0, shadow.instsize()) + # only pointers have non-0 size assert isinstance(w_rcvr, model.W_PointersObject) - return w_rcvr.fetch(idx) + return w_rcvr.fetch(idx_0) @expose_primitive(INST_VAR_AT_PUT, unwrap_spec=[object, int, object]) -def func(interp, w_rcvr, idx, w_value): +def func(interp, w_rcvr, idx_1, w_value): "Stores a value into a fixed field from the object, and fails otherwise" + idx_0 = idx_1 - 1 shadow = w_rcvr.shadow_of_my_class() - assert_bounds(idx, 0, shadow.instsize()) + assert_bounds(idx_0, 0, shadow.instsize()) # only pointers have non-0 size assert isinstance(w_rcvr, model.W_PointersObject) - w_rcvr.store(idx, w_value) + w_rcvr.store(idx_0, w_value) return w_value @expose_primitive(AS_OOP, unwrap_spec=[object]) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 15:31:17 2007 @@ -242,12 +242,15 @@ prim_fails(primitives.NEW_WITH_ARG, [w_Object, 20]) def test_inst_var_at(): - w_v = prim(primitives.INST_VAR_AT, ["q", constants.CHARACTER_VALUE_INDEX]) + # n.b.: 1-based indexing! + w_v = prim(primitives.INST_VAR_AT, + ["q", constants.CHARACTER_VALUE_INDEX+1]) assert w_v.value == ord("q") def test_inst_var_at_put(): + # n.b.: 1-based indexing! w_q = classtable.w_Character.as_class_get_shadow().new() - vidx = constants.CHARACTER_VALUE_INDEX + vidx = constants.CHARACTER_VALUE_INDEX+1 ordq = ord("q") assert prim(primitives.INST_VAR_AT, [w_q, vidx]) == objtable.w_nil assert prim(primitives.INST_VAR_AT_PUT, [w_q, vidx, ordq]).value == ordq @@ -377,6 +380,9 @@ """ # Note: +# primitives.NEXT is unimplemented as it is a performance optimization +# primitives.NEXT_PUT is unimplemented as it is a performance optimization +# primitives.AT_END is unimplemented as it is a performance optimization # primitives.PRIMITIVE_BLOCK_COPY is tested in test_interpreter # primitives.PRIMITIVE_VALUE is tested in test_interpreter # primitives.PRIMITIVE_VALUE_WITH_ARGS is tested in test_interpreter From niko at codespeak.net Fri Oct 26 15:35:47 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 15:35:47 +0200 (CEST) Subject: [pypy-svn] r48062 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026133547.20E95812A@code0.codespeak.net> Author: niko Date: Fri Oct 26 15:35:45 2007 New Revision: 48062 Modified: pypy/dist/pypy/lang/smalltalk/objtable.py pypy/dist/pypy/lang/smalltalk/primitives.py Log: (niko, toon) add a new kind of unwrap_spec that indices a 1-based index which should be converted to a 0-based index. Removes the n0 = n1-1 lines all over the code. Also convert names like idx1 to n1 Modified: pypy/dist/pypy/lang/smalltalk/objtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/objtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/objtable.py Fri Oct 26 15:35:45 2007 @@ -35,6 +35,17 @@ else: return w_false +def wrap_list(lst_w_obj): + """ + Converts a Python list of wrapper objects into + a wrapped smalltalk array + """ + lstlen = len(lit) + res = ct.w_Array.as_class_get_shadow().new(lstlen) + for i in range(lstlen): + res.storevarpointer(i, fakeliteral(lit[i])) + return res + # ___________________________________________________________________________ # Global Data Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 15:35:45 2007 @@ -53,6 +53,10 @@ # So all optional primitives will default to the bytecode implementation prim_table = [make_failing(i) for i in range(576)] +# indicates that what is pushed is an index1, but it is unwrapped and +# converted to an index0 +index1_0 = object() + def expose_primitive(code, unwrap_spec=None): # some serious magic, don't look from pypy.rlib.unroll import unrolling_iterable @@ -76,7 +80,7 @@ prim_table[code] = func return func for spec in unwrap_spec: - assert spec in (int, float, object) + assert spec in (int, float, object, index1_0) len_unwrap_spec = len(unwrap_spec) assert (len_unwrap_spec == len(inspect.getargspec(func)[0]) + 1, "wrong number of arguments") @@ -93,6 +97,8 @@ arg = frame.stack[index] if spec is int: args += (unwrap_int(arg), ) + elif spec is index1_0: + args += (unwrap_int(arg)-1, ) elif spec is float: args += (unwrap_float(arg), ) elif spec is object: @@ -294,15 +300,13 @@ STRING_AT = 63 STRING_AT_PUT = 64 - at expose_primitive(AT, unwrap_spec=[object, int]) -def func(interp, w_obj, n1): - n0 = n1 - 1 + at expose_primitive(AT, unwrap_spec=[object, index1_0]) +def func(interp, w_obj, n0): assert_valid_index(n0, w_obj) return w_obj.at0(n0) - at expose_primitive(AT_PUT, unwrap_spec=[object, int, object]) -def func(interp, w_obj, n1, w_val): - n0 = n1 - 1 + at expose_primitive(AT_PUT, unwrap_spec=[object, index1_0, object]) +def func(interp, w_obj, n0, w_val): assert_valid_index(n0, w_obj) w_obj.atput0(n0, w_val) return w_val @@ -313,16 +317,14 @@ raise PrimitiveFailedError() return wrap_int(w_obj.size()) - at expose_primitive(STRING_AT, unwrap_spec=[object, int]) -def func(interp, w_obj, n1): - n0 = n1 - 1 + at expose_primitive(STRING_AT, unwrap_spec=[object, index1_0]) +def func(interp, w_obj, n0): assert_valid_index(n0, w_obj) byte = w_obj.getbyte(n0) return objtable.CharacterTable[byte] - at expose_primitive(STRING_AT_PUT, unwrap_spec=[object, int, object]) -def func(interp, w_obj, n1, w_val): - n0 = n1 - 1 + at expose_primitive(STRING_AT_PUT, unwrap_spec=[object, index1_0, object]) +def func(interp, w_obj, n0, w_val): assert_valid_index(n0, w_obj) if w_val.getclass() is not classtable.w_Character: raise PrimitiveFailedError() @@ -352,15 +354,13 @@ NEXT_INSTANCE = 78 NEW_METHOD = 79 - at expose_primitive(OBJECT_AT, unwrap_spec=[object, int]) -def func(interp, w_rcvr, n1): - n0 = n1 - 1 + at expose_primitive(OBJECT_AT, unwrap_spec=[object, index1_0]) +def func(interp, w_rcvr, n0): assert_bounds(n0, 0, w_rcvr.shadow_of_my_class().instance_size) return w_rcvr.fetch(n0) - at expose_primitive(OBJECT_AT_PUT, unwrap_spec=[object, int, object]) -def func(interp, w_rcvr, n1, w_val): - n0 = n1 - 1 + at expose_primitive(OBJECT_AT_PUT, unwrap_spec=[object, index1_0, object]) +def func(interp, w_rcvr, n0, w_val): assert_bounds(n0, 0, w_rcvr.shadow_of_my_class().instance_size) w_rcvr.store(n0, w_val) return w_val @@ -384,25 +384,23 @@ def func(interp, w_obj1, w_obj2): raise PrimitiveNotYetWrittenError - at expose_primitive(INST_VAR_AT, unwrap_spec=[object, int]) -def func(interp, w_rcvr, idx_1): + at expose_primitive(INST_VAR_AT, unwrap_spec=[object, index1_0]) +def func(interp, w_rcvr, n0): "Fetches a fixed field from the object, and fails otherwise" shadow = w_rcvr.shadow_of_my_class() - idx_0 = idx_1 - 1 - assert_bounds(idx_0, 0, shadow.instsize()) + assert_bounds(n0, 0, shadow.instsize()) # only pointers have non-0 size assert isinstance(w_rcvr, model.W_PointersObject) - return w_rcvr.fetch(idx_0) + return w_rcvr.fetch(n0) - at expose_primitive(INST_VAR_AT_PUT, unwrap_spec=[object, int, object]) -def func(interp, w_rcvr, idx_1, w_value): + at expose_primitive(INST_VAR_AT_PUT, unwrap_spec=[object, index1_0, object]) +def func(interp, w_rcvr, n0, w_value): "Stores a value into a fixed field from the object, and fails otherwise" - idx_0 = idx_1 - 1 shadow = w_rcvr.shadow_of_my_class() - assert_bounds(idx_0, 0, shadow.instsize()) + assert_bounds(n0, 0, shadow.instsize()) # only pointers have non-0 size assert isinstance(w_rcvr, model.W_PointersObject) - w_rcvr.store(idx_0, w_value) + w_rcvr.store(n0, w_value) return w_value @expose_primitive(AS_OOP, unwrap_spec=[object]) From cfbolz at codespeak.net Fri Oct 26 15:36:24 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 26 Oct 2007 15:36:24 +0200 (CEST) Subject: [pypy-svn] r48063 - in pypy/dist/pypy/lang/smalltalk: . test tool Message-ID: <20071026133624.98D388139@code0.codespeak.net> Author: cfbolz Date: Fri Oct 26 15:36:24 2007 New Revision: 48063 Added: pypy/dist/pypy/lang/smalltalk/tool/bitmanipulation.py (contents, props changed) pypy/dist/pypy/lang/smalltalk/tool/test_bitmanipulation.py (contents, props changed) Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py Log: try to be a bit cleverer with the bit splitting. doesn't completely work yet, due to an inexactness in the annotator. Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Fri Oct 26 15:36:24 2007 @@ -3,6 +3,7 @@ from pypy.lang.smalltalk import model from pypy.lang.smalltalk import objtable from pypy.rlib import objectmodel +from pypy.lang.smalltalk.tool.bitmanipulation import splitter def chrs2int(b): assert len(b) == 4 @@ -18,18 +19,6 @@ first = first - 0x100 return first << 24 | ord(b[2]) << 16 | ord(b[1]) << 8 | ord(b[0]) -def splitbits(integer, lengths): - #XXX we can later let the tool chain mask and unroll this - result = [] - sum = 0 - for length in lengths: - sum += length - n = integer & ((1<= 0 - result.append(n) - integer = integer >> length - assert sum <= 32 - return result # ____________________________________________________________ # @@ -182,23 +171,23 @@ def read_1wordobjectheader(self): kind, size, format, classid, idhash = ( - splitbits(self.stream.next(), [2,6,4,5,12])) + splitter[2,6,4,5,12](self.stream.next())) assert kind == 3 return ImageChunk(size, format, classid, idhash), self.stream.count - 4 def read_2wordobjectheader(self): assert self.stream.peek() & 3 == 1 #kind classid = self.stream.next() - 01 # remove headertype to get pointer - kind, size, format, _, idhash = splitbits(self.stream.next(), [2,6,4,5,12]) + kind, size, format, _, idhash = splitter[2,6,4,5,12](self.stream.next()) assert kind == 1 return ImageChunk(size, format, classid, idhash), self.stream.count - 4 def read_3wordobjectheader(self): - kind, size = splitbits(self.stream.next(), [2,30]) + kind, size = splitter[2,30](self.stream.next()) assert kind == 0 - assert splitbits(self.stream.peek(), [2])[0] == 0 #kind + assert splitter[2](self.stream.peek())[0] == 0 #kind classid = self.stream.next() - 00 # remove headertype to get pointer - kind, _, format, _, idhash = splitbits(self.stream.next(), [2,6,4,5,12]) + kind, _, format, _, idhash = splitter[2,6,4,5,12](self.stream.next()) assert kind == 0 return ImageChunk(size, format, classid, idhash), self.stream.count - 4 @@ -379,7 +368,7 @@ #(index 28) 1 bit: high-bit of primitive number (#primitive) #(index 29) 1 bit: flag bit, ignored by the VM (#flag) _, primitive, literalsize, islarge, tempsize, numargs, highbit = ( - splitbits(header, [1,9,8,1,6,4,1])) + splitter[1,9,8,1,6,4,1](header)) primitive = primitive + (highbit << 10) ##XXX todo, check this literals = [self.decode_pointer(pointer).w_object for pointer in self.chunk.data[:literalsize+1]] Modified: pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py Fri Oct 26 15:36:24 2007 @@ -70,18 +70,7 @@ stream.next() assert stream.count == 8 -def test_simple_splitbits(): - assert ([1] * 4) == squeakimage.splitbits(0x01010101, [8,8,8,8]) - assert ([255] * 4) == squeakimage.splitbits(0xFfFfFfFf, [8,8,8,8]) - -def test_fancy_splitbits(): - assert [4,3,2,1] == squeakimage.splitbits(0x01020304, [8,8,8,8]) - assert [1,3,7,15] == squeakimage.splitbits(0xFfFfFfFf, [1,2,3,4]) - -def test_format_splitbits(): - x = 0xAA - assert [x & 3] == squeakimage.splitbits(x, [2]) - + def test_simple_joinbits(): assert 0x01010101 == joinbits(([1] * 4), [8,8,8,8]) assert 0xFfFfFfFf == joinbits([255] * 4, [8,8,8,8]) Added: pypy/dist/pypy/lang/smalltalk/tool/bitmanipulation.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/tool/bitmanipulation.py Fri Oct 26 15:36:24 2007 @@ -0,0 +1,28 @@ +from pypy.rlib import unroll + +# terrible hack to make the annotator think this is a real dict +__name__ = '__builtin__' + +class BitSplitter(dict): + def __getitem__(self, lengths): + if isinstance(lengths, int): + lengths = (lengths, ) + if lengths in self: + return dict.__getitem__(self, lengths) + unrolling_lenghts = unroll.unrolling_iterable(lengths) + def splitbits(integer): + result = () + sum = 0 + for length in unrolling_lenghts: + sum += length + n = integer & ((1<= 0 + result += (n, ) + integer = integer >> length + assert sum <= 32 + return result + splitbits.func_name += "_" + "_".join([str(i) for i in lengths]) + self[lengths] = splitbits + return splitbits + +splitter = BitSplitter() Added: pypy/dist/pypy/lang/smalltalk/tool/test_bitmanipulation.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/tool/test_bitmanipulation.py Fri Oct 26 15:36:24 2007 @@ -0,0 +1,15 @@ +from pypy.lang.smalltalk.tool.bitmanipulation import splitter + + +def test_simple_splitbits(): + assert ((1, ) * 4) == splitter[8,8,8,8](0x01010101) + assert ((255, ) * 4) == splitter[8,8,8,8](0xFfFfFfFf) + +def test_fancy_splitbits(): + assert (4,3,2,1) == splitter[8,8,8,8](0x01020304) + assert (1,3,7,15) == splitter[1,2,3,4](0xFfFfFfFf) + +def test_format_splitbits(): + x = 0xAA + assert (x & 3, ) == splitter[2](x) + From lukas at codespeak.net Fri Oct 26 15:44:22 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Fri, 26 Oct 2007 15:44:22 +0200 (CEST) Subject: [pypy-svn] r48064 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071026134422.EBE1E809C@code0.codespeak.net> Author: lukas Date: Fri Oct 26 15:44:22 2007 New Revision: 48064 Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: added tests for SIZE primitive Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 15:44:22 2007 @@ -190,6 +190,16 @@ def test_invalid_at_put(): w_obj = mockclass(0).as_class_get_shadow().new() prim_fails(primitives.AT_PUT, [w_obj, 1, 22]) + +def test_size(): + w_obj = mockclass(0, varsized=True).as_class_get_shadow().new(0) + assert prim(primitives.SIZE, [w_obj]).value == 0 + w_obj = mockclass(3, varsized=True).as_class_get_shadow().new(5) + assert prim(primitives.SIZE, [w_obj]).value == 5 + +def test_size(): + w_obj = mockclass(0).as_class_get_shadow().new(0) + prim_fails(primitives.SIZE, [w_obj]) def test_string_at(): assert prim(primitives.STRING_AT, ["foobar", 4]) == wrap("b") From arigo at codespeak.net Fri Oct 26 15:48:07 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 26 Oct 2007 15:48:07 +0200 (CEST) Subject: [pypy-svn] r48065 - in pypy/dist/pypy/annotation: . test Message-ID: <20071026134807.1BB91813A@code0.codespeak.net> Author: arigo Date: Fri Oct 26 15:48:06 2007 New Revision: 48065 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/test/test_annrpython.py Log: Found out that (I think that) I can safely reintroduce one useful special constant-folding case in compare_helper: comparisons that statically look like 'x < 0' and 'x >= 0' where x is, so far, known to be non-negative. Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Fri Oct 26 15:48:06 2007 @@ -21,7 +21,7 @@ from pypy.annotation.model import SomeGenericCallable from pypy.annotation.model import SomeExternalInstance from pypy.annotation.bookkeeper import getbookkeeper -from pypy.objspace.flow.model import Variable +from pypy.objspace.flow.model import Variable, Constant from pypy.annotation.listdef import ListDef from pypy.rlib import rarithmetic from pypy.rpython import extregistry @@ -334,6 +334,15 @@ SomeInteger(nonneg=True, knowntype=tointtype(int1))) if knowntypedata: r.knowntypedata = knowntypedata + # a special case for 'x < 0' or 'x >= 0', + # where 0 is a flow graph Constant + # (in this case we are sure that it cannot become a r_uint later) + if isinstance(op.args[1], Constant) and op.args[1].value == 0: + if int1.nonneg: + if opname == 'lt': + r.const = False + if opname == 'ge': + r.const = True return r def lt(intint): return intint._compare_helper('lt', operator.lt) Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Fri Oct 26 15:48:06 2007 @@ -2849,6 +2849,18 @@ a = self.RPythonAnnotator() a.build_types(f, [int, int]) + def test_compare_with_zero(self): + def g(): + should_not_see_this + def f(n): + assert n >= 0 + if n < 0: + g() + if not (n >= 0): + g() + a = self.RPythonAnnotator() + a.build_types(f, [int]) + def test_r_singlefloat(self): z = r_singlefloat(0.4) def g(n): From niko at codespeak.net Fri Oct 26 16:20:12 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 16:20:12 +0200 (CEST) Subject: [pypy-svn] r48066 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026142012.32434813A@code0.codespeak.net> Author: niko Date: Fri Oct 26 16:20:10 2007 New Revision: 48066 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (all) change at0 and atput0 to use character in bytes-based objects. we can't figure out if this is correct, but it makes the Squeak's 0 tinyBenchmarks run! Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Fri Oct 26 16:20:10 2007 @@ -162,10 +162,11 @@ def at0(self, index0): from pypy.lang.smalltalk import objtable - return objtable.wrap_int(self.getbyte(index0)) + return objtable.CharacterTable[self.getbyte(index0)] def atput0(self, index0, w_value): - self.setbyte(index0, unwrap_int(w_value)) + from pypy.lang.smalltalk import objtable + self.setbyte(index0, objtable.ord_w_char(w_value)) def getbyte(self, n): return ord(self.bytes[n]) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 16:20:10 2007 @@ -187,6 +187,16 @@ assert prim(primitives.AT_PUT, [w_obj, 1, 22]).value == 22 assert prim(primitives.AT, [w_obj, 1]).value == 22 +def test_at_bytes(): + assert prim(primitives.AT, ["abc", 1]) is wrap("a") + +def test_at_put_bytes(): + w_str = wrap("abc") + assert prim(primitives.AT_PUT, [w_str, 1, "d"]) is wrap("d") + assert prim(primitives.AT, [w_str, 1]) is wrap("d") + assert prim(primitives.AT, [w_str, 2]) is wrap("b") + assert prim(primitives.AT, [w_str, 3]) is wrap("c") + def test_invalid_at_put(): w_obj = mockclass(0).as_class_get_shadow().new() prim_fails(primitives.AT_PUT, [w_obj, 1, 22]) From arigo at codespeak.net Fri Oct 26 16:31:14 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 26 Oct 2007 16:31:14 +0200 (CEST) Subject: [pypy-svn] r48067 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071026143114.3AA338146@code0.codespeak.net> Author: arigo Date: Fri Oct 26 16:31:13 2007 New Revision: 48067 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: Far-fetched translation fix. In truth, targetfibsmalltalk should not really just call this function. Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 16:31:13 2007 @@ -62,7 +62,7 @@ res.storevarpointer(i, fakeliteral(lit[i])) return res return lit - return ["methodheader"] + [fakeliteral(lit) for lit in literals] + return [fakesymbol("methodheader")]+[fakeliteral(lit) for lit in literals] def new_interpreter(bytes, receiver=objtable.w_nil): assert isinstance(bytes, str) From rxe at codespeak.net Fri Oct 26 16:33:53 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 26 Oct 2007 16:33:53 +0200 (CEST) Subject: [pypy-svn] r48068 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071026143353.BC8C3814F@code0.codespeak.net> Author: rxe Date: Fri Oct 26 16:33:53 2007 New Revision: 48068 Modified: pypy/dist/pypy/translator/llvm/test/test_lltype.py Log: fix / sync up tests a bit Modified: pypy/dist/pypy/translator/llvm/test/test_lltype.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lltype.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lltype.py Fri Oct 26 16:33:53 2007 @@ -7,6 +7,25 @@ from pypy.translator.llvm.test.runtest import * +def test_simple(): + S = GcStruct("s", ('v', Signed)) + def llf(): + s = malloc(S) + return s.v + fn = compile_function(llf, []) + assert fn() == 0 + +def test_simple2(): + S = Struct("s", ('v', Signed)) + S2 = GcStruct("s2", ('a',S), ('b',S)) + def llf(): + s = malloc(S2) + s.a.v = 6 + s.b.v = 12 + return s.a.v + s.b.v + fn = compile_function(llf, []) + assert fn() == 18 + S = Struct("base", ('a', Signed), ('b', Signed)) def test_struct_constant1(): @@ -86,21 +105,6 @@ f = compile_function(struct_constant, []) assert f() == struct_constant() -def test_struct_constant6(): - U = Struct('inlined', ('z', Signed)) - T = GcStruct('subtest', ('y', Signed)) - S = GcStruct('test', ('x', Ptr(T)), ('u', U), ('p', Ptr(U))) - - s = malloc(S) - s.x = malloc(T) - s.x.y = 42 - s.u.z = -100 - s.p = s.u - def struct_constant(): - return s.x.y + s.p.z - f = compile_function(struct_constant, []) - assert f() == struct_constant() - def test_aliasing(): B = Struct('B', ('x', Signed)) A = Array(B) @@ -239,6 +243,22 @@ f = compile_function(floats_fn, []) assert f() == floats_fn() +def test_simple_fixedsizearray(): + A2 = FixedSizeArray(Signed, 2) + S = GcStruct("s", ("a2", A2)) + def llf(): + s = malloc(S) + a2 = s.a2 + a2[0] = -1 + a2.item1 = -2 + assert a2[0] == -1 + assert a2[1] == -2 + assert a2.item1 == -2 + return s.a2.item0 - s.a2[1] + fn = compile_function(llf, []) + res = fn() + assert fn() == 1 + def test_fixedsizearray(): S = Struct("s", ('v', Signed)) A7 = FixedSizeArray(Signed, 7) @@ -270,9 +290,9 @@ def test_recursivearray(): A = ForwardReference() A.become(FixedSizeArray(Struct("S", ('a', Ptr(A))), 5)) - TREE = GcStruct("TREE", ("root", A), ("other", A)) + TREE = Struct("TREE", ("root", A), ("other", A)) + tree = malloc(TREE, immortal=True) def llf(): - tree = malloc(TREE) tree.root[0].a = tree.root tree.root[1].a = tree.other assert tree.root[0].a[0].a[0].a[0].a[0].a[1].a == tree.other @@ -292,7 +312,7 @@ s = '' for i in range(5): s += chr(64+a[i]) - assert s == "HELLO0" + assert s == "HELLO" return 0 fn = compile_function(llf, []) fn() @@ -314,8 +334,8 @@ def test_more_prebuilt_arrays(): A = FixedSizeArray(Struct('s1', ('x', Signed)), 5) - S = GcStruct('s', ('a1', Ptr(A)), ('a2', A)) - s = malloc(S, zero=True) + S = Struct('s', ('a1', Ptr(A)), ('a2', A)) + s = malloc(S, zero=True, immortal=True) s.a1 = malloc(A, immortal=True) s.a1[2].x = 50 s.a2[2].x = 60 @@ -379,6 +399,28 @@ res = fn(4) assert res == 0 + 10 + 30 + 1000 +def test_structarray_add(): + from pypy.rpython.lltypesystem import llmemory + S = Struct("S", ("x", Signed)) + PS = Ptr(S) + size = llmemory.sizeof(S) + A = GcArray(S) + itemoffset = llmemory.itemoffsetof(A, 0) + def llf(n): + a = malloc(A, 5) + a[0].x = 1 + a[1].x = 2 + a[2].x = 3 + a[3].x = 42 + a[4].x = 4 + adr_s = llmemory.cast_ptr_to_adr(a) + adr_s += itemoffset + size * n + s = llmemory.cast_adr_to_ptr(adr_s, PS) + return s.x + fn = compile_function(llf, [int]) + res = fn(3) + assert res == 42 + def test_direct_fieldptr(): S = GcStruct('S', ('x', Signed), ('y', Signed)) def llf(n): @@ -391,6 +433,20 @@ res = fn(34) assert res == 34 +def test_union(): + py.test.skip("unions!!") + U = Struct('U', ('s', Signed), ('c', Char), + hints={'union': True}) + u = malloc(U, immortal=True) + def llf(c): + u.s = 0x10203040 + u.c = chr(c) + return u.s + + fn = compile_function(llf, [int]) + res = fn(0x33) + assert res in [0x10203033, 0x33203040] + def test_prebuilt_simple_subarrays(): a2 = malloc(FixedSizeArray(Signed, 5), immortal=True) a2[1] = 42 @@ -425,20 +481,3 @@ fn = compile_function(llf, []) res = fn() assert res == 8765 - -def test_pointer2fixedsizearray(): - A = FixedSizeArray(Signed, 1) - EmbedS = GcStruct('S', ('data', A)) - S = GcStruct('S', ('c_data', Ptr(A))) - - e = malloc(EmbedS, zero=True) - s = malloc(S, zero=True) - c_data = s.c_data = e.data - c_data[0] = 42 - - def llf(): - return s.c_data[0] - - fn = compile_function(llf, []) - res = fn() - assert res == 42 From niko at codespeak.net Fri Oct 26 16:43:03 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 16:43:03 +0200 (CEST) Subject: [pypy-svn] r48069 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026144303.63430814F@code0.codespeak.net> Author: niko Date: Fri Oct 26 16:43:03 2007 New Revision: 48069 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (all) back out our last change, so that byte objects expect integers again Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Fri Oct 26 16:43:03 2007 @@ -162,11 +162,10 @@ def at0(self, index0): from pypy.lang.smalltalk import objtable - return objtable.CharacterTable[self.getbyte(index0)] + return objtable.wrap_int(self.getbyte(index0)) def atput0(self, index0, w_value): - from pypy.lang.smalltalk import objtable - self.setbyte(index0, objtable.ord_w_char(w_value)) + self.setbyte(index0, unwrap_int(w_value)) def getbyte(self, n): return ord(self.bytes[n]) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 16:43:03 2007 @@ -308,8 +308,11 @@ @expose_primitive(AT_PUT, unwrap_spec=[object, index1_0, object]) def func(interp, w_obj, n0, w_val): assert_valid_index(n0, w_obj) - w_obj.atput0(n0, w_val) - return w_val + try: + w_obj.atput0(n0, w_val) + return w_val + except model.UnwrappingError: + raise PrimitiveFailedError() @expose_primitive(SIZE, unwrap_spec=[object]) def func(interp, w_obj): Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 16:43:03 2007 @@ -187,15 +187,13 @@ assert prim(primitives.AT_PUT, [w_obj, 1, 22]).value == 22 assert prim(primitives.AT, [w_obj, 1]).value == 22 -def test_at_bytes(): - assert prim(primitives.AT, ["abc", 1]) is wrap("a") - -def test_at_put_bytes(): +def test_at_and_at_put_bytes(): w_str = wrap("abc") - assert prim(primitives.AT_PUT, [w_str, 1, "d"]) is wrap("d") - assert prim(primitives.AT, [w_str, 1]) is wrap("d") - assert prim(primitives.AT, [w_str, 2]) is wrap("b") - assert prim(primitives.AT, [w_str, 3]) is wrap("c") + prim_fails(primitives.AT_PUT, [w_str, 1, "d"]) + assert prim(primitives.AT_PUT, [w_str, 1, ord('d')]).value == ord('d') + assert prim(primitives.AT, [w_str, 1]).value == ord('d') + assert prim(primitives.AT, [w_str, 2]).value == ord('b') + assert prim(primitives.AT, [w_str, 3]).value == ord('c') def test_invalid_at_put(): w_obj = mockclass(0).as_class_get_shadow().new() From lukas at codespeak.net Fri Oct 26 16:43:24 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Fri, 26 Oct 2007 16:43:24 +0200 (CEST) Subject: [pypy-svn] r48070 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026144324.E90E7814F@code0.codespeak.net> Author: lukas Date: Fri Oct 26 16:43:24 2007 New Revision: 48070 Added: pypy/dist/pypy/lang/smalltalk/find_untested_bytecodes.sh (contents, props changed) Modified: pypy/dist/pypy/lang/smalltalk/find_untested_prims.sh Log: added a script to find untested bytecodes Added: pypy/dist/pypy/lang/smalltalk/find_untested_bytecodes.sh ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/find_untested_bytecodes.sh Fri Oct 26 16:43:24 2007 @@ -0,0 +1,17 @@ +#!/bin/sh + +awk '/W_ContextPart\./ {print $0;}' interpreter.py \ + | sed -e 's/.*W_ContextPart\.\(.*\)),/\1/' \ + | sort -u \ + > bcodes.1 +cat test/test_interpreter.py \ + | sed 's/[^A-Za-z]/ /g' \ + | tr ' ' '\n' \ + | grep '[a-z]' \ + | sort -u \ + > bcodes.2 + +echo Untested byte-codes: +diff bcodes.1 bcodes.2 \ + | grep "<" +rm bcodes.1 bcodes.2 \ No newline at end of file Modified: pypy/dist/pypy/lang/smalltalk/find_untested_prims.sh ============================================================================== --- pypy/dist/pypy/lang/smalltalk/find_untested_prims.sh (original) +++ pypy/dist/pypy/lang/smalltalk/find_untested_prims.sh Fri Oct 26 16:43:24 2007 @@ -1,5 +1,5 @@ #!/bin/sh grep '^[A-Z][A-Z_]' primitives.py | awk '{print $1;}' | sort | uniq > pcodes.1 grep 'primitives.[A-Z][A-Z_]' test/test_primitives.py | sed 's/.*primitives.\([A-Z_]*\).*/\1/g' | sort | uniq > pcodes.2 -echo Unimplemented primitive codes: +echo Untested primitive codes: diff pcodes.1 pcodes.2 From niko at codespeak.net Fri Oct 26 16:58:05 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 16:58:05 +0200 (CEST) Subject: [pypy-svn] r48071 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026145805.DE8F1814F@code0.codespeak.net> Author: niko Date: Fri Oct 26 16:58:04 2007 New Revision: 48071 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (niko, toon) fix the bytecodeAt functions when working with strings Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 16:58:04 2007 @@ -366,10 +366,16 @@ self.callPrimitiveAndPush(primitives.BIT_OR, "|", 1, interp) def bytecodePrimAt(self, interp): - self.callPrimitiveAndPush(primitives.AT, "at:", 1, interp) + # n.b.: depending on the type of the receiver, this may invoke + # primitives.AT, primitives.STRING_AT, or something else for all + # I know. + #self.callPrimitiveAndPush(primitives.AT, "at:", 1, interp) + self._sendSelfSelector("at:", 1, interp) def bytecodePrimAtPut(self, interp): - self.callPrimitiveAndPush(primitives.AT_PUT, "at:put:", 2, interp) + # n.b. as above + #self.callPrimitiveAndPush(primitives.AT_PUT, "at:put:", 2, interp) + self._sendSelfSelector("at:put:", 2, interp) def bytecodePrimSize(self, interp): self.callPrimitiveAndPush(primitives.SIZE, "size", 0, interp) Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 16:58:04 2007 @@ -1,7 +1,7 @@ import py from pypy.lang.smalltalk import model, interpreter, primitives, shadow from pypy.lang.smalltalk import objtable -from pypy.lang.smalltalk.objtable import wrap_int +from pypy.lang.smalltalk.objtable import wrap_int, wrap_char import pypy.lang.smalltalk.classtable as ct mockclass = ct.bootstrap_class @@ -570,3 +570,23 @@ 1, "valueWithArguments:"]], test) +def test_bc_primBytecodeAt_string(): + # ^ 'a' at: 1 + def test(): + assert interpret_bc( + [ 32, 118, 192, 124], + fakeliterals("a")) == wrap_char("a") + run_with_faked_methods( + [[ct.w_String, primitives.STRING_AT, 1, "at:"]], + test) + +def test_bc_primBytecodeAtPut_string(): + # ^ 'a' at: 1 put:'b' + def test(): + assert interpret_bc( + [ 32, 118, 33, 193, 124 ], + fakeliterals("a", wrap_char("b"))) == wrap_char("b") + run_with_faked_methods( + [[ct.w_String, primitives.STRING_AT_PUT, 2, "at:put:"]], + test) + From lukas at codespeak.net Fri Oct 26 17:03:01 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Fri, 26 Oct 2007 17:03:01 +0200 (CEST) Subject: [pypy-svn] r48072 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071026150301.14CA9814F@code0.codespeak.net> Author: lukas Date: Fri Oct 26 17:03:01 2007 New Revision: 48072 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: added tests for bytecodes #bitAnd:, #bitOr:, #bitShift: Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 17:03:01 2007 @@ -235,6 +235,27 @@ interp.step() interp.step() assert interp.w_active_context.stack == [interp.ZERO, interp.ZERO] + +def test_bytecodePrimBitAnd(): + interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitAnd) + interp.step() + interp.step() + interp.step() + assert interp.w_active_context.stack[0].value == 0 + +def test_bytecodePrimBitOr(): + interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitOr) + interp.step() + interp.step() + interp.step() + assert interp.w_active_context.stack[0].value == 3 + +def test_bytecodePrimBitShift(): + interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitShift) + interp.step() + interp.step() + interp.step() + assert interp.w_active_context.stack[0].value == 4 # w_class - the class from which the method is going to be called # (and on which it is going to be installed) @@ -588,5 +609,4 @@ fakeliterals("a", wrap_char("b"))) == wrap_char("b") run_with_faked_methods( [[ct.w_String, primitives.STRING_AT_PUT, 2, "at:put:"]], - test) - + test) \ No newline at end of file From rxe at codespeak.net Fri Oct 26 17:04:49 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 26 Oct 2007 17:04:49 +0200 (CEST) Subject: [pypy-svn] r48073 - pypy/dist/pypy/translator/llvm Message-ID: <20071026150449.E3372815B@code0.codespeak.net> Author: rxe Date: Fri Oct 26 17:04:49 2007 New Revision: 48073 Modified: pypy/dist/pypy/translator/llvm/database.py Log: pyc files were hanging around (and it was a pointless import anyways) Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Fri Oct 26 17:04:49 2007 @@ -13,7 +13,6 @@ OpaqueTypeNode, ExtOpaqueTypeNode from pypy.rpython.lltypesystem import lltype, llmemory from pypy.objspace.flow.model import Constant, Variable -from pypy.rpython.memory.lladdress import NULL from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic from pypy.rlib.objectmodel import CDefinedIntSymbolic from pypy.rlib import objectmodel From xoraxax at codespeak.net Fri Oct 26 17:08:01 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 26 Oct 2007 17:08:01 +0200 (CEST) Subject: [pypy-svn] r48074 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071026150801.A398180B8@code0.codespeak.net> Author: xoraxax Date: Fri Oct 26 17:08:01 2007 New Revision: 48074 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: Added myself to the gothenburg sprint. Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Fri Oct 26 17:08:01 2007 @@ -16,6 +16,7 @@ Christian Tismer 19- after 26 Laura and Jacob's Maciej Fijalkowski 19-26 Laura and Jacob's Armin Rigo lives thereish +Alexander Schremmer 16-26 Laura and Jacob's ==================== ============== ===================== @@ -32,7 +33,6 @@ Niko Matsakis ? ? Leonardo Santagada ? ? Richard Emslie ? ? -Alexander Schremmer ? ? Aurelien Campeas ? ? Alexandre Fayolle ? ? Lene Wagner ? ? From lukas at codespeak.net Fri Oct 26 17:10:07 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Fri, 26 Oct 2007 17:10:07 +0200 (CEST) Subject: [pypy-svn] r48075 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071026151007.B07C180CE@code0.codespeak.net> Author: lukas Date: Fri Oct 26 17:10:07 2007 New Revision: 48075 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: added tests for bytecodes #+, #*, #class Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 17:10:07 2007 @@ -256,6 +256,26 @@ interp.step() interp.step() assert interp.w_active_context.stack[0].value == 4 + +def test_bytecodePrimClass(): + interp = new_interpreter(pushConstantOneBytecode + bytecodePrimClass) + interp.step() + interp.step() + assert interp.w_active_context.stack[0] == ct.w_SmallInteger + +def test_bytecodePrimSubtract(): + interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimSubtract) + interp.step() + interp.step() + interp.step() + assert interp.w_active_context.stack[0].value == -1 + +def test_bytecodePrimMultiply(): + interp = new_interpreter(pushConstantMinusOneBytecode + pushConstantTwoBytecode + bytecodePrimMultiply) + interp.step() + interp.step() + interp.step() + assert interp.w_active_context.stack[0].value == -2 # w_class - the class from which the method is going to be called # (and on which it is going to be installed) From lukas at codespeak.net Fri Oct 26 17:13:17 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Fri, 26 Oct 2007 17:13:17 +0200 (CEST) Subject: [pypy-svn] r48076 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071026151317.3469280BB@code0.codespeak.net> Author: lukas Date: Fri Oct 26 17:13:15 2007 New Revision: 48076 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: added tests for #// and #\\ bytecodes Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 17:13:15 2007 @@ -276,6 +276,22 @@ interp.step() interp.step() assert interp.w_active_context.stack[0].value == -2 + +def test_bytecodePrimDiv(): + interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimDiv) + interp.step() + interp.step() + interp.step() + assert interp.w_active_context.stack[0].value == -2 + +def test_bytecodePrimMod(): + interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimMod) + interp.step() + interp.step() + interp.step() + assert interp.w_active_context.stack[0].value == 0 + + # w_class - the class from which the method is going to be called # (and on which it is going to be installed) From xoraxax at codespeak.net Fri Oct 26 17:20:11 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 26 Oct 2007 17:20:11 +0200 (CEST) Subject: [pypy-svn] r48077 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071026152011.627FE80D6@code0.codespeak.net> Author: xoraxax Date: Fri Oct 26 17:20:11 2007 New Revision: 48077 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: Changed arrival date. Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Fri Oct 26 17:20:11 2007 @@ -16,7 +16,7 @@ Christian Tismer 19- after 26 Laura and Jacob's Maciej Fijalkowski 19-26 Laura and Jacob's Armin Rigo lives thereish -Alexander Schremmer 16-26 Laura and Jacob's +Alexander Schremmer 19-26 Laura and Jacob's ==================== ============== ===================== From lukas at codespeak.net Fri Oct 26 17:24:40 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Fri, 26 Oct 2007 17:24:40 +0200 (CEST) Subject: [pypy-svn] r48078 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071026152440.58D9780C8@code0.codespeak.net> Author: lukas Date: Fri Oct 26 17:24:39 2007 New Revision: 48078 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: added test for #== bytecode and improved several other tests to test harder Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 17:24:39 2007 @@ -188,47 +188,54 @@ assert interp.w_active_context.temps[test_index] == interp.TRUE else: assert interp.w_active_context.temps[test_index] == None - def test_pushConstantTrueBytecode(): interp = new_interpreter(pushConstantTrueBytecode) interp.step() - assert interp.w_active_context.top() == interp.TRUE + assert interp.w_active_context.pop() == interp.TRUE + assert interp.w_active_context.stack == [] def test_pushConstantFalseBytecode(): interp = new_interpreter(pushConstantFalseBytecode) interp.step() - assert interp.w_active_context.top() == interp.FALSE + assert interp.w_active_context.pop() == interp.FALSE + assert interp.w_active_context.stack == [] def test_pushConstantNilBytecode(): interp = new_interpreter(pushConstantNilBytecode) interp.step() - assert interp.w_active_context.top() == interp.NIL + assert interp.w_active_context.pop() == interp.NIL + assert interp.w_active_context.stack == [] def test_pushConstantMinusOneBytecode(): interp = new_interpreter(pushConstantMinusOneBytecode) interp.step() - assert interp.w_active_context.top() == interp.MONE + assert interp.w_active_context.pop() == interp.MONE + assert interp.w_active_context.stack == [] def test_pushConstantZeroBytecode(): interp = new_interpreter(pushConstantZeroBytecode) interp.step() - assert interp.w_active_context.top() == interp.ZERO + assert interp.w_active_context.pop() == interp.ZERO + assert interp.w_active_context.stack == [] def test_pushConstantOneBytecode(): interp = new_interpreter(pushConstantOneBytecode) interp.step() - assert interp.w_active_context.top() == interp.ONE + assert interp.w_active_context.pop() == interp.ONE + assert interp.w_active_context.stack == [] def test_pushConstantTwoBytecode(): interp = new_interpreter(pushConstantTwoBytecode) interp.step() - assert interp.w_active_context.top() + assert interp.w_active_context.pop() == interp.TWO + assert interp.w_active_context.stack == [] def test_pushActiveContextBytecode(): interp = new_interpreter(pushActiveContextBytecode) interp.step() - assert interp.w_active_context.top() == interp.w_active_context + assert interp.w_active_context.pop() == interp.w_active_context + assert interp.w_active_context.stack == [] def test_duplicateTopBytecode(): interp = new_interpreter(pushConstantZeroBytecode + duplicateTopBytecode) @@ -241,57 +248,86 @@ interp.step() interp.step() interp.step() - assert interp.w_active_context.stack[0].value == 0 + assert interp.w_active_context.pop().value == 0 + assert interp.w_active_context.stack == [] def test_bytecodePrimBitOr(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitOr) interp.step() interp.step() interp.step() - assert interp.w_active_context.stack[0].value == 3 + assert interp.w_active_context.pop().value == 3 + assert interp.w_active_context.stack == [] def test_bytecodePrimBitShift(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitShift) interp.step() interp.step() interp.step() - assert interp.w_active_context.stack[0].value == 4 - + assert interp.w_active_context.pop().value == 4 + assert interp.w_active_context.stack == [] + def test_bytecodePrimClass(): interp = new_interpreter(pushConstantOneBytecode + bytecodePrimClass) interp.step() interp.step() - assert interp.w_active_context.stack[0] == ct.w_SmallInteger + assert interp.w_active_context.pop() == ct.w_SmallInteger + assert interp.w_active_context.stack == [] def test_bytecodePrimSubtract(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimSubtract) interp.step() interp.step() interp.step() - assert interp.w_active_context.stack[0].value == -1 + assert interp.w_active_context.pop().value == -1 + assert interp.w_active_context.stack == [] def test_bytecodePrimMultiply(): interp = new_interpreter(pushConstantMinusOneBytecode + pushConstantTwoBytecode + bytecodePrimMultiply) interp.step() interp.step() interp.step() - assert interp.w_active_context.stack[0].value == -2 + assert interp.w_active_context.pop().value == -2 + assert interp.w_active_context.stack == [] + +def test_bytecodePrimDivide(): + interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimDivide) + interp.step() + interp.step() + interp.step() + assert interp.w_active_context.pop().value == -2 + assert interp.w_active_context.stack == [] def test_bytecodePrimDiv(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimDiv) interp.step() interp.step() interp.step() - assert interp.w_active_context.stack[0].value == -2 + assert interp.w_active_context.pop().value == -2 + assert interp.w_active_context.stack == [] def test_bytecodePrimMod(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimMod) interp.step() interp.step() interp.step() - assert interp.w_active_context.stack[0].value == 0 - + assert interp.w_active_context.pop().value == 0 + assert interp.w_active_context.stack == [] +def test_bytecodePrimEquivalent(): + interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimEquivalent) + interp.step() + interp.step() + interp.step() + assert interp.w_active_context.pop() == interpreter.Interpreter.FALSE + assert interp.w_active_context.stack == [] + + interp = new_interpreter(pushConstantOneBytecode + pushConstantOneBytecode + bytecodePrimEquivalent) + interp.step() + interp.step() + interp.step() + assert interp.w_active_context.pop() == interpreter.Interpreter.TRUE + assert interp.w_active_context.stack == [] # w_class - the class from which the method is going to be called # (and on which it is going to be installed) From niko at codespeak.net Fri Oct 26 17:27:40 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 17:27:40 +0200 (CEST) Subject: [pypy-svn] r48079 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026152740.A41CC80D6@code0.codespeak.net> Author: niko Date: Fri Oct 26 17:27:40 2007 New Revision: 48079 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: (niko, toon) change AT and AT_PUT to fetch from the variable-sized region Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Fri Oct 26 17:27:40 2007 @@ -118,10 +118,10 @@ self._vars = [w_nil] * size def at0(self, index0): - return self.fetch(index0) + return self.fetchvarpointer(index0) def atput0(self, index0, w_value): - self.store(index0, w_value) + self.storevarpointer(index0, w_value) def fetch(self, n0): return self._vars[n0] Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 17:27:40 2007 @@ -580,9 +580,9 @@ storeAssociation(doubleExtendedDoAnythingBytecode + chr(7<<5) + chr(0)) -def interpret_bc(bcodes, literals): +def interpret_bc(bcodes, literals, receiver=objtable.w_nil): bcode = "".join([chr(x) for x in bcodes]) - interp = new_interpreter(bcode) + interp = new_interpreter(bcode, receiver=receiver) interp.w_active_context.w_method().literals = literals return interp.interpret() @@ -681,4 +681,36 @@ fakeliterals("a", wrap_char("b"))) == wrap_char("b") run_with_faked_methods( [[ct.w_String, primitives.STRING_AT_PUT, 2, "at:put:"]], - test) \ No newline at end of file + test) + +def test_bc_primBytecodeAt_with_instvars(): + # ^ self at: 1 + w_fakeclass = mockclass(1, name='fakeclass', varsized=True) + w_fakeinst = w_fakeclass.as_class_get_shadow().new(1) + w_fakeinst.store(0, wrap_char("a")) # static slot 0: instance variable + w_fakeinst.store(1, wrap_char("b")) # varying slot 1 + def test(): + assert objtable.ord_w_char(interpret_bc( + [112, 118, 192, 124], + fakeliterals(), + receiver=w_fakeinst)) == ord("b") + run_with_faked_methods( + [[w_fakeclass, primitives.AT, 1, "at:"]], + test) + +def test_bc_primBytecodeAtPut_with_instvars(): + # ^ self at: 1 put: #b + w_fakeclass = mockclass(1, name='fakeclass', varsized=True) + w_fakeinst = w_fakeclass.as_class_get_shadow().new(1) + w_fakeinst.store(0, wrap_char("a")) # static slot 0: instance variable + w_fakeinst.store(1, wrap_char("a")) # varying slot 1 + def test(): + assert objtable.ord_w_char(interpret_bc( + [0x70, 0x76, 0x20, 0xc1, 0x7c], + fakeliterals(wrap_char("b")), + receiver=w_fakeinst)) == ord("b") + assert objtable.ord_w_char(w_fakeinst.fetch(0)) == ord("a") + assert objtable.ord_w_char(w_fakeinst.fetch(1)) == ord("b") + run_with_faked_methods( + [[w_fakeclass, primitives.AT_PUT, 2, "at:put:"]], + test) From niko at codespeak.net Fri Oct 26 17:29:45 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 17:29:45 +0200 (CEST) Subject: [pypy-svn] r48080 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071026152945.9285A80D6@code0.codespeak.net> Author: niko Date: Fri Oct 26 17:29:45 2007 New Revision: 48080 Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (niko, toon) add tests to ensure that inst_var_at[_put] don't go past their bounds Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 17:29:45 2007 @@ -265,6 +265,10 @@ ["q", constants.CHARACTER_VALUE_INDEX+1]) assert w_v.value == ord("q") +def test_inst_var_at_invalid(): + # n.b.: 1-based indexing! (and an invalid index) + prim_fails(primitives.INST_VAR_AT, ["q", constants.CHARACTER_VALUE_INDEX+2]) + def test_inst_var_at_put(): # n.b.: 1-based indexing! w_q = classtable.w_Character.as_class_get_shadow().new() @@ -274,6 +278,11 @@ assert prim(primitives.INST_VAR_AT_PUT, [w_q, vidx, ordq]).value == ordq assert prim(primitives.INST_VAR_AT, [w_q, vidx]).value == ordq +def test_inst_var_at_put_invalid(): + # n.b.: 1-based indexing! (and an invalid index) + prim_fails(primitives.INST_VAR_AT_PUT, + ["q", constants.CHARACTER_VALUE_INDEX+2, "t"]) + def test_class(): assert prim(primitives.CLASS, ["string"]) == classtable.w_String assert prim(primitives.CLASS, [1]) == classtable.w_SmallInteger From lukas at codespeak.net Fri Oct 26 17:39:46 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Fri, 26 Oct 2007 17:39:46 +0200 (CEST) Subject: [pypy-svn] r48081 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071026153946.69D9A8154@code0.codespeak.net> Author: lukas Date: Fri Oct 26 17:39:46 2007 New Revision: 48081 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: added tests for #new, #new: bytecodes Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 17:39:46 2007 @@ -328,6 +328,27 @@ interp.step() assert interp.w_active_context.pop() == interpreter.Interpreter.TRUE assert interp.w_active_context.stack == [] + +def test_bytecodePrimNew(): + w_fakeclass = mockclass(1, name='fakeclass', varsized=False) + interp = new_interpreter(bytecodePrimNew) + interp.w_active_context.push(w_fakeclass) + interp.step() + w_fakeinst = interp.w_active_context.pop() + assert interp.w_active_context.stack == [] + assert w_fakeinst.getclass() == w_fakeclass + assert w_fakeinst.size() == 1 + +def test_bytecodePrimNewWithArg(): + w_fakeclass = mockclass(1, name='fakeclass', varsized=True) + interp = new_interpreter(bytecodePrimNewWithArg) + interp.w_active_context.push(w_fakeclass) + interp.w_active_context.push(interpreter.Interpreter.TWO) + interp.step() + w_fakeinst = interp.w_active_context.pop() + assert interp.w_active_context.stack == [] + assert w_fakeinst.getclass() == w_fakeclass + assert w_fakeinst.size() == 3 # w_class - the class from which the method is going to be called # (and on which it is going to be installed) @@ -586,6 +607,7 @@ interp.w_active_context.w_method().literals = literals return interp.interpret() +# tests: bytecodePrimValue & bytecodePrimValueWithArg def test_bc_3_plus_4(): # value0 # " (self >> #value0) byteCode " From lukas at codespeak.net Fri Oct 26 17:59:08 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Fri, 26 Oct 2007 17:59:08 +0200 (CEST) Subject: [pypy-svn] r48082 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026155908.5A9728150@code0.codespeak.net> Author: lukas Date: Fri Oct 26 17:59:06 2007 New Revision: 48082 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: added test for #size bytecode and fixed the bug found in primitive and interpreter and associated tests Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 17:59:06 2007 @@ -318,7 +318,7 @@ def func(interp, w_obj): if not w_obj.shadow_of_my_class().isvariable(): raise PrimitiveFailedError() - return wrap_int(w_obj.size()) + return wrap_int(w_obj.size() - w_obj.shadow_of_my_class().instsize()) @expose_primitive(STRING_AT, unwrap_spec=[object, index1_0]) def func(interp, w_obj, n0): Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 17:59:06 2007 @@ -349,6 +349,15 @@ assert interp.w_active_context.stack == [] assert w_fakeinst.getclass() == w_fakeclass assert w_fakeinst.size() == 3 + +def test_bytecodePrimSize(): + w_fakeclass = mockclass(2, name='fakeclass', varsized=True) + w_fakeinst = w_fakeclass.as_class_get_shadow().new(5) + interp = new_interpreter(bytecodePrimSize) + interp.w_active_context.push(w_fakeinst) + interp.step() + assert interp.w_active_context.pop().value == 5 + assert interp.w_active_context.stack == [] # w_class - the class from which the method is going to be called # (and on which it is going to be installed) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 17:59:06 2007 @@ -204,10 +204,6 @@ assert prim(primitives.SIZE, [w_obj]).value == 0 w_obj = mockclass(3, varsized=True).as_class_get_shadow().new(5) assert prim(primitives.SIZE, [w_obj]).value == 5 - -def test_size(): - w_obj = mockclass(0).as_class_get_shadow().new(0) - prim_fails(primitives.SIZE, [w_obj]) def test_string_at(): assert prim(primitives.STRING_AT, ["foobar", 4]) == wrap("b") From niko at codespeak.net Fri Oct 26 18:01:04 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 18:01:04 +0200 (CEST) Subject: [pypy-svn] r48083 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026160104.3DE788167@code0.codespeak.net> Author: niko Date: Fri Oct 26 18:01:03 2007 New Revision: 48083 Modified: pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (niko, toon) implement objectAt: and objectAt:put: properly. Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Fri Oct 26 18:01:03 2007 @@ -40,7 +40,14 @@ MTHDCTX_RECEIVER_MAP = 4 MTHDCTX_RECEIVER = 5 MTHDCTX_TEMP_FRAME_START = 6 -# ----- special objects indices ------- + +# ___________________________________________________________________________ +# Miscellaneous constants + +LITERAL_START = 1 # index of the first literal after the method header + +# ___________________________________________________________________________ +# Special objects indices SO_NIL = 0 SO_FALSE = 1 Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Fri Oct 26 18:01:03 2007 @@ -264,7 +264,7 @@ return w_CompiledMethod def getliteral(self, index): - return self.literals[index + 1] # header of compiledmethod at index 0 + return self.literals[index + constants.LITERAL_START] def getliteralsymbol(self, index): w_literal = self.getliteral(index) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 18:01:03 2007 @@ -359,13 +359,17 @@ @expose_primitive(OBJECT_AT, unwrap_spec=[object, index1_0]) def func(interp, w_rcvr, n0): - assert_bounds(n0, 0, w_rcvr.shadow_of_my_class().instance_size) - return w_rcvr.fetch(n0) + if not isinstance(w_rcvr, model.W_CompiledMethod): + raise PrimitiveFailedError() + assert_bounds(n0, 0, len(w_rcvr.literals)) + return w_rcvr.literals[n0] @expose_primitive(OBJECT_AT_PUT, unwrap_spec=[object, index1_0, object]) def func(interp, w_rcvr, n0, w_val): - assert_bounds(n0, 0, w_rcvr.shadow_of_my_class().instance_size) - w_rcvr.store(n0, w_val) + if not isinstance(w_rcvr, model.W_CompiledMethod): + raise PrimitiveFailedError() + assert_bounds(n0, 0, len(w_rcvr.literals)) + w_rcvr.literals[n0] = w_val return w_val @expose_primitive(NEW, unwrap_spec=[object]) Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 18:01:03 2007 @@ -745,3 +745,27 @@ run_with_faked_methods( [[w_fakeclass, primitives.AT_PUT, 2, "at:put:"]], test) + +def test_bc_objectAtAndAtPut(): + # ^ self objectAt: 1. yields the method header + # ^ self objectAt: 2. yields the first literal (22) + # ^ self objectAt: 2 put: 3. changes the first literal to 3 + # ^ self objectAt: 2. yields the new first literal (3) + prim_meth = model.W_CompiledMethod(0, "") + prim_meth.literals = fakeliterals(22) + mhs = fakesymbol("methodheader") + oal = fakeliterals("objectAt:") + oalp = fakeliterals("objectAt:put:", 3) + def test(): + assert interpret_bc( + [112, 118, 224, 124], oal, receiver=prim_meth) == mhs + assert interpret_bc( + [112, 119, 224, 124], oal, receiver=prim_meth).value == 22 + assert interpret_bc( + [112, 119, 33, 240, 124], oalp, receiver=prim_meth).value == 3 + assert interpret_bc( + [112, 119, 224, 124], oal, receiver=prim_meth).value == 3 + run_with_faked_methods( + [[ct.w_CompiledMethod, primitives.OBJECT_AT, 1, "objectAt:"], + [ct.w_CompiledMethod, primitives.OBJECT_AT_PUT, 2, "objectAt:put:"]], + test) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 18:01:03 2007 @@ -215,18 +215,9 @@ for i in range(len(exp)): assert prim(primitives.STRING_AT, [test_str, i]) == wrap(exp[i]) -def test_object_at(): - w_v = prim(primitives.OBJECT_AT, ["q", constants.CHARACTER_VALUE_INDEX+1]) - assert w_v.value == ord("q") - def test_invalid_object_at(): prim_fails(primitives.OBJECT_AT, ["q", constants.CHARACTER_VALUE_INDEX+2]) -def test_object_at_put(): - w_obj = mockclass(1).as_class_get_shadow().new() - assert prim(primitives.OBJECT_AT_PUT, [w_obj, 1, "q"]) is wrap("q") - assert prim(primitives.OBJECT_AT, [w_obj, 1]) is wrap("q") - def test_invalid_object_at_put(): w_obj = mockclass(1).as_class_get_shadow().new() prim_fails(primitives.OBJECT_AT_PUT, [w_obj, 2, 42]) @@ -409,3 +400,5 @@ # primitives.PRIMITIVE_BLOCK_COPY is tested in test_interpreter # primitives.PRIMITIVE_VALUE is tested in test_interpreter # primitives.PRIMITIVE_VALUE_WITH_ARGS is tested in test_interpreter +# primitives.OBJECT_AT is tested in test_interpreter +# primitives.OBJECT_AT_PUT is tested in test_interpreter From lukas at codespeak.net Fri Oct 26 18:15:51 2007 From: lukas at codespeak.net (lukas at codespeak.net) Date: Fri, 26 Oct 2007 18:15:51 +0200 (CEST) Subject: [pypy-svn] r48084 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026161551.C68C98148@code0.codespeak.net> Author: lukas Date: Fri Oct 26 18:15:51 2007 New Revision: 48084 Modified: pypy/dist/pypy/lang/smalltalk/find_untested_prims.sh pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: added a test for primitive MILLISECOND_CLOCK as requested Modified: pypy/dist/pypy/lang/smalltalk/find_untested_prims.sh ============================================================================== --- pypy/dist/pypy/lang/smalltalk/find_untested_prims.sh (original) +++ pypy/dist/pypy/lang/smalltalk/find_untested_prims.sh Fri Oct 26 18:15:51 2007 @@ -2,4 +2,4 @@ grep '^[A-Z][A-Z_]' primitives.py | awk '{print $1;}' | sort | uniq > pcodes.1 grep 'primitives.[A-Z][A-Z_]' test/test_primitives.py | sed 's/.*primitives.\([A-Z_]*\).*/\1/g' | sort | uniq > pcodes.2 echo Untested primitive codes: -diff pcodes.1 pcodes.2 +diff pcodes.1 pcodes.2 | grep "<" Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 18:15:51 2007 @@ -361,6 +361,13 @@ assert equals_ttp(-1,2,-4) assert equals_ttp(1.5,0,1.5) assert equals_ttp(1.5,-1,0.75) + +def test_primtiive_milliseconds_clock(): + import time + start = prim(primitives.MILLISECOND_CLOCK, [0]).value + time.sleep(0.010) + stop = prim(primitives.MILLISECOND_CLOCK, [0]).value + assert start + 10 <= stop def test_inc_gc(): # Should not fail :-) From niko at codespeak.net Fri Oct 26 18:28:05 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 18:28:05 +0200 (CEST) Subject: [pypy-svn] r48085 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026162805.4D4FB8158@code0.codespeak.net> Author: niko Date: Fri Oct 26 18:28:05 2007 New Revision: 48085 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: (niko, toon) add SECONDS and the 256 INST_VAR_AT_x primitives Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 18:28:05 2007 @@ -150,6 +150,7 @@ # note: argcount does not include rcvr w_result = func(interp, argcount) except primitives.PrimitiveFailedError: + print "PRIMITIVE FAILED: %d %s" % (method.primitive, selector,) pass # ignore this error and fall back to the Smalltalk version else: # the primitive succeeded @@ -315,6 +316,7 @@ # note that argcount does not include self self.push(primitives.prim_table[primitive](interp, argcount)) except primitives.PrimitiveFailedError: + print "PRIMITIVE FAILED: %s" % (selector,) self._sendSelfSelector(selector, argcount, interp) def bytecodePrimAdd(self, interp): Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 18:28:05 2007 @@ -519,6 +519,7 @@ #____________________________________________________________________________ # Time Primitives MILLISECOND_CLOCK = 135 +SECONDS_CLOCK = 137 @expose_primitive(MILLISECOND_CLOCK, unwrap_spec=[object]) def func(interp, w_arg): @@ -526,6 +527,11 @@ import math return wrap_int(int(math.fmod(time.time()*1000,1073741823/2))) + at expose_primitive(SECONDS_CLOCK, unwrap_spec=[object]) +def func(interp, w_arg): + import time + return wrap_int(0) # wrap_int(int(time.time())) + # ___________________________________________________________________________ # Boolean Primitives @@ -715,3 +721,22 @@ @expose_primitive(PRIMITIVE_FLUSH_CACHE, unwrap_spec=[object]) def func(interp, w_rcvr): raise PrimitiveNotYetWrittenError() + +# ___________________________________________________________________________ +# PrimitiveLoadInstVar +# +# These are some wacky bytecodes in squeak. They are defined to do +# the following: +# primitiveLoadInstVar +# | thisReceiver | +# thisReceiver := self popStack. +# self push: (self fetchPointer: primitiveIndex-264 ofObject: thisReceiver) + +for i in range(264, 520): + def make_prim(i): + @expose_primitive(i, unwrap_spec=[object]) + def func(interp, w_object): + return w_object.fetch(i - 264) + globals()["INST_VAR_AT_%d" % (i-264)] = i + make_prim(i) + Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Oct 26 18:28:05 2007 @@ -377,6 +377,11 @@ # Should not fail :-) prim(primitives.FULL_GC, [42]) # Dummy arg +def test_seconds_clock(): + import time + now = int(time.time()) + assert (prim(primitives.SECONDS_CLOCK, [42]).value - now) <= 2 + def test_become(): py.test.skip("implement me!") """ @@ -399,6 +404,11 @@ self should: [1 become: 2] raise: Error. """ + +def test_load_inst_var(): + " try to test the LoadInstVar primitives a little " + w_v = prim(primitives.INST_VAR_AT_0, ["q"]) + assert w_v.value == ord("q") # Note: # primitives.NEXT is unimplemented as it is a performance optimization @@ -409,3 +419,4 @@ # primitives.PRIMITIVE_VALUE_WITH_ARGS is tested in test_interpreter # primitives.OBJECT_AT is tested in test_interpreter # primitives.OBJECT_AT_PUT is tested in test_interpreter + From niko at codespeak.net Fri Oct 26 18:33:40 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 18:33:40 +0200 (CEST) Subject: [pypy-svn] r48086 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026163340.B20D68161@code0.codespeak.net> Author: niko Date: Fri Oct 26 18:33:40 2007 New Revision: 48086 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py Log: remove debugging printout Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 18:33:40 2007 @@ -150,7 +150,7 @@ # note: argcount does not include rcvr w_result = func(interp, argcount) except primitives.PrimitiveFailedError: - print "PRIMITIVE FAILED: %d %s" % (method.primitive, selector,) + #print "PRIMITIVE FAILED: %d %s" % (method.primitive, selector,) pass # ignore this error and fall back to the Smalltalk version else: # the primitive succeeded @@ -316,7 +316,7 @@ # note that argcount does not include self self.push(primitives.prim_table[primitive](interp, argcount)) except primitives.PrimitiveFailedError: - print "PRIMITIVE FAILED: %s" % (selector,) + #print "PRIMITIVE FAILED: %s" % (selector,) self._sendSelfSelector(selector, argcount, interp) def bytecodePrimAdd(self, interp): From arigo at codespeak.net Fri Oct 26 18:52:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 26 Oct 2007 18:52:20 +0200 (CEST) Subject: [pypy-svn] r48087 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026165220.8FBB98163@code0.codespeak.net> Author: arigo Date: Fri Oct 26 18:52:20 2007 New Revision: 48087 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py Log: pop_n() is called very often and was causing unnecessary allocations. Fixing it gives a 2x speed-up in the translated version! :-) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Fri Oct 26 18:52:20 2007 @@ -377,8 +377,14 @@ assert n >= 0 start = len(self.stack) - n assert start >= 0 # XXX what if this fails? + del self.stack[start:] + + def pop_and_return_n(self, n): + assert n >= 0 + start = len(self.stack) - n + assert start >= 0 # XXX what if this fails? res = self.stack[start:] - self.stack = self.stack[:start] + del self.stack[start:] return res class W_BlockContext(W_ContextPart): Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 18:52:20 2007 @@ -668,7 +668,7 @@ # Initialize the block stack with the arguments that were # pushed. Also pop the receiver. - block_args = frame.pop_n(exp_arg_cnt) + block_args = frame.pop_and_return_n(exp_arg_cnt) w_block_ctx.push_all(block_args) frame.pop() From arigo at codespeak.net Fri Oct 26 18:57:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 26 Oct 2007 18:57:35 +0200 (CEST) Subject: [pypy-svn] r48088 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071026165735.D6B44816B@code0.codespeak.net> Author: arigo Date: Fri Oct 26 18:57:33 2007 New Revision: 48088 Modified: pypy/dist/pypy/lang/smalltalk/shadow.py Log: Profiling is great. Another interesting speed-up. Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Fri Oct 26 18:57:33 2007 @@ -176,12 +176,15 @@ def lookup(self, selector): look_in_shadow = self - while selector not in look_in_shadow.methoddict: + while True: + try: + return look_in_shadow.methoddict[selector] + except KeyError: + pass look_in_shadow = look_in_shadow.s_superclass if look_in_shadow is None: # attach information on the exception, for debugging. raise MethodNotFound(self, selector) - return look_in_shadow.methoddict[selector] def installmethod(self, selector, method): "NOT_RPYTHON" # this is only for testing. From fijal at codespeak.net Fri Oct 26 19:52:09 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 26 Oct 2007 19:52:09 +0200 (CEST) Subject: [pypy-svn] r48089 - pypy/dist/pypy/translator/c Message-ID: <20071026175209.ED4FD815D@code0.codespeak.net> Author: fijal Date: Fri Oct 26 19:52:09 2007 New Revision: 48089 Modified: pypy/dist/pypy/translator/c/database.py pypy/dist/pypy/translator/c/external.py pypy/dist/pypy/translator/c/genc.py Log: Fix a bit translator/c to call prepare_inline_helpers Modified: pypy/dist/pypy/translator/c/database.py ============================================================================== --- pypy/dist/pypy/translator/c/database.py (original) +++ pypy/dist/pypy/translator/c/database.py Fri Oct 26 19:52:09 2007 @@ -392,3 +392,11 @@ for graph in node.graphs_to_patch(): graphs.append(graph) self.gctransformer.prepare_inline_helpers(graphs) + + def all_graphs(self): + graphs = [] + for node in self.containerlist: + if node.nodekind == 'func': + for graph in node.graphs_to_patch(): + graphs.append(graph) + return graphs Modified: pypy/dist/pypy/translator/c/external.py ============================================================================== --- pypy/dist/pypy/translator/c/external.py (original) +++ pypy/dist/pypy/translator/c/external.py Fri Oct 26 19:52:09 2007 @@ -15,6 +15,9 @@ self.argtypenames = [db.gettype(T) for T in self.FUNCTYPE.ARGS] self.resulttypename = db.gettype(self.FUNCTYPE.RESULT) + def graphs_to_patch(self): + return [] + def name(self, cname): #virtual return cname Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Fri Oct 26 19:52:09 2007 @@ -173,11 +173,7 @@ # actually generating the source. if db is None: db = self.build_database() - graphs = [] - for node in db.containerlist: - if isinstance(node, FuncNode): - for graph in node.graphs_to_patch(): - graphs.append(graph) + graphs = db.all_graphs() db.gctransformer.prepare_inline_helpers(graphs) for node in db.containerlist: if isinstance(node, FuncNode): @@ -596,6 +592,8 @@ print >> f, '#include "src/g_include.h"' print >> f blank = True + graphs = database.all_graphs() + database.gctransformer.prepare_inline_helpers(graphs) for node in database.globalcontainers(): if blank: print >> f From niko at codespeak.net Fri Oct 26 20:07:26 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 26 Oct 2007 20:07:26 +0200 (CEST) Subject: [pypy-svn] r48090 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071026180726.BFE7C816A@code0.codespeak.net> Author: niko Date: Fri Oct 26 20:07:25 2007 New Revision: 48090 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: (niko, toon, lukas, arigo) fixed various problems on the way towards making the squeak compiler "work" Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Fri Oct 26 20:07:25 2007 @@ -24,6 +24,8 @@ ZERO = objtable.w_zero ONE = objtable.w_one TWO = objtable.w_two + + _w_last_active_context = None def __init__(self): self.w_active_context = None @@ -35,15 +37,29 @@ except ReturnFromTopLevel, e: return e.object + def should_trace(self): + return (not objectmodel.we_are_translated()) and option.bc_trace + def step(self): next = self.w_active_context.getNextBytecode() if not objectmodel.we_are_translated(): bytecodeimpl = BYTECODE_TABLE[next] - if option.bc_trace: - print "About to execute bytecode at %d (%d:%s):" % ( + if self.should_trace(): + if self._w_last_active_context != self.w_active_context: + cnt = 0 + p = self.w_active_context + while p is not None: + cnt += 1 + p = p.w_sender + self._last_indent = " " * cnt + self._w_last_active_context = self.w_active_context + print "%sStack=%s" % ( + self._last_indent, + repr(self.w_active_context.stack),) + print "%sBytecode at %d (%d:%s):" % ( + self._last_indent, self.w_active_context.pc, next, bytecodeimpl.__name__,) - print " Stack=%s" % (repr(self.w_active_context.stack),) bytecodeimpl(self.w_active_context, self) else: for code, bytecodeimpl in unrolling_bytecode_table: @@ -141,6 +157,9 @@ def _sendSelector(self, selector, argcount, interp, receiver, receiverclassshadow): + if interp.should_trace(): + print "%sSending selector %s to %s" % ( + interp._last_indent, selector, receiver) assert argcount >= 0 method = receiverclassshadow.lookup(selector) # XXX catch MethodNotFound here and send doesNotUnderstand: @@ -150,10 +169,12 @@ # note: argcount does not include rcvr w_result = func(interp, argcount) except primitives.PrimitiveFailedError: - #print "PRIMITIVE FAILED: %d %s" % (method.primitive, selector,) + if interp.should_trace(): + print "PRIMITIVE FAILED: %d %s" % (method.primitive, selector,) pass # ignore this error and fall back to the Smalltalk version else: # the primitive succeeded + assert w_result self.push(w_result) return start = len(self.stack) - argcount @@ -312,12 +333,13 @@ def callPrimitiveAndPush(self, primitive, selector, argcount, interp): + #XXX do something clever later try: # note that argcount does not include self self.push(primitives.prim_table[primitive](interp, argcount)) except primitives.PrimitiveFailedError: - #print "PRIMITIVE FAILED: %s" % (selector,) self._sendSelfSelector(selector, argcount, interp) + #self._sendSelfSelector(selector, argcount, interp) def bytecodePrimAdd(self, interp): self.callPrimitiveAndPush(primitives.ADD, "+", 1, interp) @@ -350,7 +372,7 @@ self.callPrimitiveAndPush(primitives.DIVIDE, "/", 1, interp) def bytecodePrimMod(self, interp): - self.callPrimitiveAndPush(primitives.MOD, "\\", 1, interp) + self.callPrimitiveAndPush(primitives.MOD, "\\\\", 1, interp) def bytecodePrimMakePoint(self, interp): raise MissingBytecode("bytecodePrimMakePoint") @@ -362,10 +384,10 @@ self.callPrimitiveAndPush(primitives.DIV, "//", 1, interp) def bytecodePrimBitAnd(self, interp): - self.callPrimitiveAndPush(primitives.BIT_AND, "&", 1, interp) + self.callPrimitiveAndPush(primitives.BIT_AND, "bitAnd:", 1, interp) def bytecodePrimBitOr(self, interp): - self.callPrimitiveAndPush(primitives.BIT_OR, "|", 1, interp) + self.callPrimitiveAndPush(primitives.BIT_OR, "bitOr:", 1, interp) def bytecodePrimAt(self, interp): # n.b.: depending on the type of the receiver, this may invoke @@ -408,7 +430,7 @@ def bytecodePrimValueWithArg(self, interp): self.callPrimitiveAndPush( - primitives.PRIMITIVE_VALUE, "value", 1, interp) + primitives.PRIMITIVE_VALUE, "value:", 1, interp) def bytecodePrimDo(self, interp): self._sendSelfSelector("do:", 1, interp) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Fri Oct 26 20:07:25 2007 @@ -155,6 +155,7 @@ shadow.check_for_updates() return shadow + class W_BytesObject(W_AbstractObjectWithClassReference): def __init__(self, w_class, size): W_AbstractObjectWithClassReference.__init__(self, w_class) @@ -361,10 +362,12 @@ return self.stack.pop() def push(self, w_v): + assert w_v self.stack.append(w_v) def push_all(self, lst): " Equivalent to 'for x in lst: self.push(x)' where x is a lst " + assert None not in lst self.stack += lst def top(self): @@ -436,7 +439,7 @@ W_ContextPart.__init__(self, self, w_sender) self._w_method = w_method self.w_receiver = w_receiver - self.temps = arguments + [None] * w_method.tempsize + self.temps = arguments + [w_nil] * w_method.tempsize def getclass(self): from pypy.lang.smalltalk.classtable import w_MethodContext Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Oct 26 20:07:25 2007 @@ -530,7 +530,8 @@ @expose_primitive(SECONDS_CLOCK, unwrap_spec=[object]) def func(interp, w_arg): import time - return wrap_int(0) # wrap_int(int(time.time())) + return wrap_int(0x23910d6c) # HACK: too big for a small int! + #return wrap_int(int(time.time())) # ___________________________________________________________________________ # Boolean Primitives @@ -673,6 +674,8 @@ frame.pop() finalize_block_ctx(interp, w_block_ctx, frame) + return w_block_ctx + @expose_primitive(PRIMITIVE_VALUE_WITH_ARGS, unwrap_spec=[object, object]) def func(interp, w_block_ctx, w_args): @@ -691,6 +694,7 @@ w_block_ctx.push(w_args.fetchvarpointer(i)) finalize_block_ctx(interp, w_block_ctx, interp.w_active_context) + return w_block_ctx @expose_primitive(PRIMITIVE_PERFORM) def func(interp, argument_count): Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Fri Oct 26 20:07:25 2007 @@ -80,7 +80,7 @@ assert w_frame.w_receiver == "receiver" assert w_frame.gettemp(0) == "foo" assert w_frame.gettemp(1) == "bar" - assert w_frame.gettemp(2) == None + assert w_frame.gettemp(2) is objtable.w_nil w_frame.settemp(2, "spam") assert w_frame.gettemp(2) == "spam" assert w_frame.getNextBytecode() == ord("h") Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Fri Oct 26 20:07:25 2007 @@ -226,7 +226,7 @@ assert w_false is objtable.w_false def test_compile_method(): - py.test.skip("Todo, make compiling of methods in mini.image work.") + py.test.skip("not quite yet") sourcecode = """fib ^self < 2 ifTrue: [ 1 ] From tverwaes at codespeak.net Sat Oct 27 05:47:46 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Sat, 27 Oct 2007 05:47:46 +0200 (CEST) Subject: [pypy-svn] r48097 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071027034746.DBA9D8136@code0.codespeak.net> Author: tverwaes Date: Sat Oct 27 05:47:45 2007 New Revision: 48097 Modified: pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/shadow.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: added a shitload of small bugfixes and a start to being indexable in every type of object smalltalk ever gives to us. We really need this apparently, since while trying to compile a method, even the MethodContext class get's a "new(basicSize)" message based on size(a MethodContext), continued by copying everything over using at: and at:put: (and intvarat:, but I didn not get that far yet). Progress based on trying to compile the fib in int. Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Sat Oct 27 05:47:45 2007 @@ -29,6 +29,8 @@ CTXPART_PC_INDEX = 1 CTXPART_STACKP_INDEX = 2 +METHOD_HEADER_INDEX = 0 + # Extends CTXPART_* BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX = 3 BLKCTX_INITIAL_IP_INDEX = 4 @@ -45,6 +47,7 @@ # Miscellaneous constants LITERAL_START = 1 # index of the first literal after the method header +BYTES_PER_WORD = 4 # ___________________________________________________________________________ # Special objects indices Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Sat Oct 27 05:47:45 2007 @@ -44,15 +44,16 @@ next = self.w_active_context.getNextBytecode() if not objectmodel.we_are_translated(): bytecodeimpl = BYTECODE_TABLE[next] + if self._w_last_active_context != self.w_active_context: + cnt = 0 + p = self.w_active_context + while p is not None: + cnt += 1 + p = p.w_sender + self._last_indent = " " * cnt + self._w_last_active_context = self.w_active_context if self.should_trace(): - if self._w_last_active_context != self.w_active_context: - cnt = 0 - p = self.w_active_context - while p is not None: - cnt += 1 - p = p.w_sender - self._last_indent = " " * cnt - self._w_last_active_context = self.w_active_context + print "%sStack=%s" % ( self._last_indent, repr(self.w_active_context.stack),) @@ -158,12 +159,16 @@ def _sendSelector(self, selector, argcount, interp, receiver, receiverclassshadow): if interp.should_trace(): - print "%sSending selector %s to %s" % ( - interp._last_indent, selector, receiver) + print "%sSending selector %r to %r with: %r" % ( + interp._last_indent, selector, receiver, + [self.stack[i-argcount] for i in range(argcount)]) + pass assert argcount >= 0 method = receiverclassshadow.lookup(selector) # XXX catch MethodNotFound here and send doesNotUnderstand: if method.primitive: + if interp.should_trace(): + print "%sActually calling primitive %d" % (interp._last_indent,method.primitive,) func = primitives.prim_table[method.primitive] try: # note: argcount does not include rcvr @@ -174,8 +179,10 @@ pass # ignore this error and fall back to the Smalltalk version else: # the primitive succeeded - assert w_result - self.push(w_result) + # Make sure that primitives that do not want to return + # anything, don't have to return anything (aBlock value) + if w_result: + self.push(w_result) return start = len(self.stack) - argcount assert start >= 0 # XXX check in the Blue Book what to do in this case @@ -212,7 +219,7 @@ raise MissingBytecode("unknownBytecode") def extendedVariableTypeAndIndex(self): - descriptor = self.getByte() + descriptor = self.getbyte() return ((descriptor >> 6) & 3), (descriptor & 63) def extendedPushBytecode(self, interp): @@ -246,7 +253,7 @@ self.pop() def getExtendedSelectorArgcount(self): - descriptor = self.getByte() + descriptor = self.getbyte() return ((self.w_method().getliteralsymbol(descriptor & 31)), (descriptor >> 5)) @@ -255,8 +262,8 @@ self._sendSelfSelector(selector, argcount, interp) def doubleExtendedDoAnythingBytecode(self, interp): - second = self.getByte() - third = self.getByte() + second = self.getbyte() + third = self.getbyte() opType = second >> 5 if opType == 0: # selfsend @@ -291,7 +298,7 @@ self._sendSuperSelector(selector, argcount, interp) def secondExtendedSendBytecode(self, interp): - descriptor = self.getByte() + descriptor = self.getbyte() selector = self.w_method().getliteralsymbol(descriptor & 63) argcount = descriptor >> 6 self._sendSelfSelector(selector, argcount, interp) @@ -320,10 +327,10 @@ self.jumpConditional(interp.FALSE,self.shortJumpPosition()) def longUnconditionalJump(self, interp): - self.jump((((self.currentBytecode & 7) - 4) << 8) + self.getByte()) + self.jump((((self.currentBytecode & 7) - 4) << 8) + self.getbyte()) def longJumpPosition(self): - return ((self.currentBytecode & 3) << 8) + self.getByte() + return ((self.currentBytecode & 3) << 8) + self.getbyte() def longJumpIfTrue(self, interp): self.jumpConditional(interp.TRUE,self.longJumpPosition()) @@ -333,10 +340,15 @@ def callPrimitiveAndPush(self, primitive, selector, argcount, interp): - #XXX do something clever later + # XXX XXX REMEMBER TO SWITCH COMMENT AND ACTUAL CODE BEFORE + # TESTING ON AN IMAGE, AND TO LEAVE AS IS FOR TESTCODE (and + # vice versa) try: # note that argcount does not include self - self.push(primitives.prim_table[primitive](interp, argcount)) + w_result = primitives.prim_table[primitive](interp, argcount) + if w_result: # enable primitives not to return values to + # the stack + self.push(w_result) except primitives.PrimitiveFailedError: self._sendSelfSelector(selector, argcount, interp) #self._sendSelfSelector(selector, argcount, interp) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Sat Oct 27 05:47:45 2007 @@ -11,6 +11,9 @@ def size(self): return 0 + def varsize(self): + return self.size() + def getclass(self): raise NotImplementedError @@ -23,6 +26,12 @@ def shadow_of_my_class(self): return self.getclass().as_class_get_shadow() + def shallow_equals(self,other): + return self == other + + def equals(self, other): + return self.shallow_equals(other) + class W_SmallInteger(W_Object): __slots__ = ('value',) # the only allowed slot here @@ -42,6 +51,11 @@ def __repr__(self): return "W_SmallInteger(%d)" % self.value + def shallow_equals(self, other): + if not isinstance(other, W_SmallInteger): + return False + return self.value == other.value + class UnwrappingError(Exception): pass @@ -67,6 +81,11 @@ def __repr__(self): return "W_Float(%f)" % self.value + def shallow_equals(self, other): + if not isinstance(other, W_Float): + return False + return self.value == other.value + class W_AbstractObjectWithIdentityHash(W_Object): #XXX maybe this is too extreme, but it's very random hash_generator = rrandom.Random() @@ -132,12 +151,16 @@ self._vars[n0] = w_value def fetchvarpointer(self, idx): - instsize = self.getclass().as_class_get_shadow().instsize() - return self._vars[idx+instsize] + return self._vars[idx+self.instsize()] def storevarpointer(self, idx, value): - instsize = self.getclass().as_class_get_shadow().instsize() - self._vars[idx+instsize] = value + self._vars[idx+self.instsize()] = value + + def varsize(self): + return self.size() - self.shadow_of_my_class().instsize() + + def instsize(self): + return self.getclass().as_class_get_shadow().instsize() def size(self): return len(self._vars) @@ -155,6 +178,17 @@ shadow.check_for_updates() return shadow + def equals(self, other): + if not isinstance(other, W_PointersObject): + return False + if not other.getclass() == self.getclass(): + return False + if not other.size() == self.size(): + return False + for i in range(self.size()): + if not other.fetch(i).shallow_equals(self.fetch(i)): + return False + return True class W_BytesObject(W_AbstractObjectWithClassReference): def __init__(self, w_class, size): @@ -194,6 +228,11 @@ return False return True + def shallow_equals(self, other): + if not isinstance(other,W_BytesObject): + return False + return self.bytes == other.bytes + class W_WordsObject(W_AbstractObjectWithClassReference): def __init__(self, w_class, size): W_AbstractObjectWithClassReference.__init__(self, w_class) @@ -219,6 +258,11 @@ return (W_AbstractObjectWithClassReference.invariant(self) and isinstance(self.words, list)) + def shallow_equals(self, other): + if not isinstance(other,W_WordsObject): + return False + return self.words == other.words + class W_CompiledMethod(W_AbstractObjectWithIdentityHash): """My instances are methods suitable for interpretation by the virtual machine. This is the only class in the system whose instances intermix both indexable pointer fields and indexable integer fields. @@ -242,7 +286,7 @@ The trailer has two variant formats. In the first variant, the last byte is at least 252 and the last four bytes represent a source pointer into one of the sources files (see #sourcePointer). In the second variant, the last byte is less than 252, and the last several bytes are a compressed version of the names of the method's temporary variables. The number of bytes used for this purpose is the value of the last byte in the method. """ - def __init__(self, literalsize, bytes, argsize=0, + def __init__(self, literalsize, bytes=[], argsize=0, tempsize=0, primitive=0, w_compiledin=None): self.literals = [None] * literalsize self.w_compiledin = w_compiledin @@ -295,6 +339,38 @@ hasattr(self, 'primitive') and self.primitive is not None) + def size(self): + return self.varsize() + + def staticsize(self): + return len(self.literals) * constants.BYTES_PER_WORD + + def varsize(self): + # XXX + return self.staticsize() + len(self.bytes) + + def at0(self, index0): + # XXX + from pypy.lang.smalltalk import objtable + index0 = index0 - self.staticsize() + if index0 < 0: + # XXX Do something useful with this.... we are not a block + # of memory as smalltalk expects but wrapped in py-os + return objtable.wrap_int(0) + return objtable.wrap_int(ord(self.bytes[index0])) + + def atput0(self, index0, w_value): + index0 = index0 - self.staticsize() + if index0 < 0: + # XXX Do something useful with this.... we are not a block + # of memory as smalltalk expects but wrapped in py-os + self.staticsize(),w_value) + else: + self.setbyte(index0, chr(unwrap_int(w_value))) + + def setbyte(self, index0, chr): + self.bytes[index0] = chr + class W_ContextPart(W_AbstractObjectWithIdentityHash): __metaclass__ = extendabletype @@ -316,7 +392,10 @@ def fetch(self, index): from pypy.lang.smalltalk import objtable if index == constants.CTXPART_SENDER_INDEX: - return self.w_sender + if self.w_sender: + return self.w_sender + else: + return objtable.w_nil elif index == constants.CTXPART_PC_INDEX: return objtable.wrap_int(self.pc) elif index == constants.CTXPART_STACKP_INDEX: @@ -334,14 +413,14 @@ def w_method(self): return self.w_home._w_method - def getByte(self): + def getbyte(self): bytecode = self.w_method().bytes[self.pc] currentBytecode = ord(bytecode) self.pc = self.pc + 1 return currentBytecode def getNextBytecode(self): - self.currentBytecode = self.getByte() + self.currentBytecode = self.getbyte() return self.currentBytecode # ______________________________________________________________________ @@ -435,7 +514,8 @@ W_ContextPart.store(self, index, value) class W_MethodContext(W_ContextPart): - def __init__(self, w_method, w_receiver, arguments, w_sender = None): + def __init__(self, w_method, w_receiver, + arguments, w_sender=None): W_ContextPart.__init__(self, self, w_sender) self._w_method = w_method self.w_receiver = w_receiver Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Sat Oct 27 05:47:45 2007 @@ -4,6 +4,7 @@ from pypy.lang.smalltalk import model, shadow from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import objtable +from pypy.lang.smalltalk import constants from pypy.rlib import rarithmetic class PrimitiveFailedError(Exception): @@ -80,7 +81,7 @@ prim_table[code] = func return func for spec in unwrap_spec: - assert spec in (int, float, object, index1_0) + assert spec in (int, float, object, index1_0, str) len_unwrap_spec = len(unwrap_spec) assert (len_unwrap_spec == len(inspect.getargspec(func)[0]) + 1, "wrong number of arguments") @@ -103,6 +104,8 @@ args += (unwrap_float(arg), ) elif spec is object: args += (arg, ) + elif spec is str: + args += (arg.as_string(), ) else: assert 0, "this should never happen" res = func(interp, *args) @@ -318,7 +321,7 @@ def func(interp, w_obj): if not w_obj.shadow_of_my_class().isvariable(): raise PrimitiveFailedError() - return wrap_int(w_obj.size() - w_obj.shadow_of_my_class().instsize()) + return objtable.wrap_int(w_obj.varsize()) @expose_primitive(STRING_AT, unwrap_spec=[object, index1_0]) def func(interp, w_obj, n0): @@ -435,9 +438,16 @@ # it returns the "next" instance after w_obj. raise PrimitiveNotYetWrittenError() - at expose_primitive(NEW_METHOD, unwrap_spec=[object]) -def func(interp, w_mthd): - raise PrimitiveNotYetWrittenError() + at expose_primitive(NEW_METHOD, unwrap_spec=[object, int, object]) +def func(interp, w_class, bytecount, w_header): + header = unwrap_int(w_header) + literalcount = ((header >> 10) & 255) + 1 + w_method = w_class.as_class_get_shadow().new(literalcount) + w_method.literals[constants.METHOD_HEADER_INDEX] = w_header + for i in range(0,literalcount): + w_method.literals[i+1] = objtable.w_nil + w_method.bytes = [None] * bytecount + return w_method # ___________________________________________________________________________ # Control Primitives @@ -451,10 +461,7 @@ @expose_primitive(EQUIVALENT, unwrap_spec=[object, object]) def func(interp, w_arg, w_rcvr): - # XXX this is bogus in the presence of (our implementation of) become, - # as we might plan to implement become by copying all fields from one - # object to the other - return objtable.wrap_bool(w_arg is w_rcvr) + return objtable.wrap_bool(w_arg.equals(w_rcvr)) @expose_primitive(CLASS, unwrap_spec=[object]) def func(interp, w_obj): @@ -671,10 +678,12 @@ # pushed. Also pop the receiver. block_args = frame.pop_and_return_n(exp_arg_cnt) w_block_ctx.push_all(block_args) - frame.pop() + frame.pop() finalize_block_ctx(interp, w_block_ctx, frame) - return w_block_ctx + # Value is a special case of primitive which does not return + # anything + return None @expose_primitive(PRIMITIVE_VALUE_WITH_ARGS, unwrap_spec=[object, object]) @@ -693,18 +702,33 @@ for i in range(exp_arg_cnt): w_block_ctx.push(w_args.fetchvarpointer(i)) + # XXX Check original logic. Image does not test this anyway + # because falls back to value + internal implementation + finalize_block_ctx(interp, w_block_ctx, interp.w_active_context) - return w_block_ctx + + # Value: is a special case of primitive which does not return + # anything + return None @expose_primitive(PRIMITIVE_PERFORM) -def func(interp, argument_count): - # XXX we can implement this when lookup on shadow class is done - raise PrimitiveNotYetWrittenError() +def func(interp, argcount): + raise PrimitiveFailedError() @expose_primitive(PRIMITIVE_PERFORM_WITH_ARGS, - unwrap_spec=[object, object, object]) -def func(interp, w_rcvr, w_sel, w_args): - raise PrimitiveNotYetWrittenError() + unwrap_spec=[object, str, object]) +def func(interp, w_rcvr, sel, w_args): + w_method = w_rcvr.shadow_of_my_class().lookup(sel) + assert w_method + + w_frame = w_method.create_frame(w_rcvr, + [w_args.fetch(i) for i in range(w_args.size())]) + + w_frame.w_sender = interp.w_active_context + interp.w_active_context = w_frame + + # Don't put anything on the stack + return None @expose_primitive(PRIMITIVE_SIGNAL, unwrap_spec=[object]) def func(interp, w_rcvr): Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Sat Oct 27 05:47:45 2007 @@ -120,10 +120,12 @@ if w_cls == classtable.w_BlockContext: return model.W_BlockContext(None, None, 0, 0) elif w_cls == classtable.w_MethodContext: - return model.W_MethodContext(None, None, []) + return model.W_MethodContext(None, None, [], extrasize) if self.instance_kind == POINTERS: return model.W_PointersObject(w_cls, self.instance_size+extrasize) + elif self.instance_kind == COMPILED_METHOD: + return model.W_CompiledMethod(extrasize+constants.LITERAL_START, w_compiledin = w_cls) elif self.instance_kind == WORDS: return model.W_WordsObject(w_cls, extrasize) elif self.instance_kind == BYTES: Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Sat Oct 27 05:47:45 2007 @@ -363,6 +363,7 @@ assert equals_ttp(1.5,-1,0.75) def test_primtiive_milliseconds_clock(): + py.test.skip("Crashes from time to time. who cares about it?") import time start = prim(primitives.MILLISECOND_CLOCK, [0]).value time.sleep(0.010) From cfbolz at codespeak.net Sat Oct 27 11:49:34 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 27 Oct 2007 11:49:34 +0200 (CEST) Subject: [pypy-svn] r48100 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071027094934.105788164@code0.codespeak.net> Author: cfbolz Date: Sat Oct 27 11:49:33 2007 New Revision: 48100 Modified: pypy/dist/pypy/lang/smalltalk/model.py Log: argh, naughty Toon (see revision 48097): fix a syntax error in the checked in code(!) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Sat Oct 27 11:49:33 2007 @@ -364,7 +364,7 @@ if index0 < 0: # XXX Do something useful with this.... we are not a block # of memory as smalltalk expects but wrapped in py-os - self.staticsize(),w_value) + raise NotImplementedError() else: self.setbyte(index0, chr(unwrap_int(w_value))) From cfbolz at codespeak.net Sat Oct 27 12:01:16 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 27 Oct 2007 12:01:16 +0200 (CEST) Subject: [pypy-svn] r48101 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071027100116.C1DAF8159@code0.codespeak.net> Author: cfbolz Date: Sat Oct 27 12:01:14 2007 New Revision: 48101 Modified: pypy/dist/pypy/lang/smalltalk/constants.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: replace all these nasty 32-bit centered magic numbers with proper constants Modified: pypy/dist/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/constants.py (original) +++ pypy/dist/pypy/lang/smalltalk/constants.py Sat Oct 27 12:01:14 2007 @@ -1,3 +1,4 @@ +from pypy.rlib.rarithmetic import LONG_BIT # ___________________________________________________________________________ # Slot Names @@ -116,3 +117,6 @@ "true": SO_TRUE, "false": SO_FALSE, } + +TAGGED_MAXINT = 2 ** (LONG_BIT - 2) - 1 +TAGGED_MININT = -2 ** (LONG_BIT - 2) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Sat Oct 27 12:01:14 2007 @@ -126,9 +126,9 @@ raise PrimitiveFailedError() def wrap_int(value): - if value > 1073741823: + if value > constants.TAGGED_MAXINT: raise PrimitiveFailedError() - if value < -1073741824: + if value < constants.TAGGED_MININT: raise PrimitiveFailedError() return objtable.wrap_int(value) @@ -532,7 +532,7 @@ def func(interp, w_arg): import time import math - return wrap_int(int(math.fmod(time.time()*1000,1073741823/2))) + return wrap_int(int(math.fmod(time.time()*1000, constants.TAGGED_MAXINT/2))) @expose_primitive(SECONDS_CLOCK, unwrap_spec=[object]) def func(interp, w_arg): Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Sat Oct 27 12:01:14 2007 @@ -51,22 +51,24 @@ assert prim(primitives.ADD, [3,4]).value == 7 def test_small_int_add_fail(): - prim_fails(primitives.ADD, [1073741823,2]) + prim_fails(primitives.ADD, [constants.TAGGED_MAXINT,2]) def test_small_int_minus(): assert prim(primitives.SUBTRACT, [5,9]).value == -4 def test_small_int_minus_fail(): - prim_fails(primitives.SUBTRACT, [-1073741823,2]) + prim_fails(primitives.SUBTRACT, [constants.TAGGED_MININT,1]) + prim_fails(primitives.SUBTRACT, + [constants.TAGGED_MININT, constants.TAGGED_MAXINT]) def test_small_int_multiply(): assert prim(primitives.MULTIPLY, [6,3]).value == 18 def test_small_int_multiply_overflow(): - prim_fails(primitives.MULTIPLY, [1073741823, 2]) - prim_fails(primitives.MULTIPLY, [1073741823, 1073741823]) - prim_fails(primitives.MULTIPLY, [1073741823, -4]) - prim_fails(primitives.MULTIPLY, [-1073741823, 2]) + prim_fails(primitives.MULTIPLY, [constants.TAGGED_MAXINT, 2]) + prim_fails(primitives.MULTIPLY, [constants.TAGGED_MAXINT, constants.TAGGED_MAXINT]) + prim_fails(primitives.MULTIPLY, [constants.TAGGED_MAXINT, -4]) + prim_fails(primitives.MULTIPLY, [constants.TAGGED_MININT, 2]) def test_small_int_divide(): assert prim(primitives.DIVIDE, [6,3]).value == 2 From cfbolz at codespeak.net Sat Oct 27 12:10:03 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 27 Oct 2007 12:10:03 +0200 (CEST) Subject: [pypy-svn] r48102 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071027101003.7B9808159@code0.codespeak.net> Author: cfbolz Date: Sat Oct 27 12:10:03 2007 New Revision: 48102 Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: better test for the milliseconds primitive Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Sat Oct 27 12:10:03 2007 @@ -364,13 +364,12 @@ assert equals_ttp(1.5,0,1.5) assert equals_ttp(1.5,-1,0.75) -def test_primtiive_milliseconds_clock(): - py.test.skip("Crashes from time to time. who cares about it?") +def test_primitive_milliseconds_clock(): import time start = prim(primitives.MILLISECOND_CLOCK, [0]).value - time.sleep(0.010) + time.sleep(0.3) stop = prim(primitives.MILLISECOND_CLOCK, [0]).value - assert start + 10 <= stop + assert start + 250 <= stop def test_inc_gc(): # Should not fail :-) From rxe at codespeak.net Sat Oct 27 13:55:08 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 27 Oct 2007 13:55:08 +0200 (CEST) Subject: [pypy-svn] r48103 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071027115508.126008162@code0.codespeak.net> Author: rxe Date: Sat Oct 27 13:55:06 2007 New Revision: 48103 Modified: pypy/dist/pypy/translator/llvm/test/test_lladdresses.py pypy/dist/pypy/translator/llvm/test/test_rffi.py pypy/dist/pypy/translator/llvm/test/test_symbolic.py pypy/dist/pypy/translator/llvm/test/test_typed.py Log: minor fixes/updates to tests. all pass (on os/x) but that is because all the problem tests have been skipped. Modified: pypy/dist/pypy/translator/llvm/test/test_lladdresses.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lladdresses.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lladdresses.py Sat Oct 27 13:55:06 2007 @@ -110,22 +110,8 @@ fn = compile_function(f, [int]) assert fn(1) == 2 -# def test_flavored_varmalloc_raw(): -# py.test.skip("test_flavored_varmalloc_raw not working - or maybe it will never to work?") -# A = lltype.Array(lltype.Signed) -# VARS = lltype.GcStruct('test', ('a', lltype.Signed), ('b', A)) -# def f(x, y): -# #s = lltype.flavored_malloc('gc', VARS, x) -# s = lltype.malloc(VARS, n=x, flavor='gc') -# s.a = 42 -# s.b[0] = y * 2 -# return s.b[0] - s.a - -# fn = compile_function(f, [int, int]) -# assert fn(2, 24) == 6 - def test_flavored_malloc_stack(): - py.test.skip("Broken and noone wants to fix it") + py.test.skip("operation stack_malloc not found") class A(object): _alloc_flavor_ = "stack" def __init__(self, val): Modified: pypy/dist/pypy/translator/llvm/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rffi.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rffi.py Sat Oct 27 13:55:06 2007 @@ -11,6 +11,8 @@ from pypy.translator.llvm.test.runtest import * +py.test.skip("rffi not there yet for llvm") + def test_basic(): c_source = py.code.Source(""" int z(int x) Modified: pypy/dist/pypy/translator/llvm/test/test_symbolic.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_symbolic.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_symbolic.py Sat Oct 27 13:55:06 2007 @@ -2,7 +2,6 @@ from pypy.translator.interactive import Translation from pypy import conftest from pypy.rpython.lltypesystem import llmemory, lltype -from pypy.rpython.memory import lladdress from pypy.rlib.objectmodel import ComputedIntSymbolic from pypy.translator.llvm.test.runtest import * @@ -82,11 +81,11 @@ sizeofs = llmemory.sizeof(STRUCT) offsety = llmemory.offsetof(STRUCT, 'y') def f(): - adr = lladdress.raw_malloc(sizeofs) + adr = llmemory.raw_malloc(sizeofs) s = llmemory.cast_adr_to_ptr(adr, STRUCTPTR) s.y = 5 # does not crash result = (adr + offsety).signed[0] * 10 + int(offsety < sizeofs) - lladdress.raw_free(adr) + llmemory.raw_free(adr) return result fn = compile_function(f, []) @@ -133,7 +132,7 @@ sizeofsbase = llmemory.sizeof(SBASE) offset_toa = offsetofs(SBASE, 'b', 's2', 'a') def complex_struct(): - adr = lladdress.raw_malloc(sizeofsbase) + adr = llmemory.raw_malloc(sizeofsbase) s = llmemory.cast_adr_to_ptr(adr, SBASEPTR) s.b.s2.a = 42 return (adr + offset_toa).signed[0] @@ -167,11 +166,10 @@ assert fn(21) == 42 def test_itemoffset_void(): - py.test.skip("not supported") + py.test.skip("inprogress") A = lltype.GcArray(lltype.Void) s = llmemory.sizeof(A, 1) s += llmemory.sizeof(lltype.Signed) - print s def f(): return s fn = compile_function(f, []) Modified: pypy/dist/pypy/translator/llvm/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_typed.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_typed.py Sat Oct 27 13:55:06 2007 @@ -54,7 +54,7 @@ assert result def test_is(): - py.test.skip("Broken and noone wants to fix it") + py.test.skip("inprogress") def testfn(): l1 = [] return l1 is l1 @@ -69,7 +69,7 @@ assert result == False def test_nones(): - py.test.skip("Broken and noone wants to fix it") + py.test.skip("inprogress") a = [None] * 4 def nones(): a.append(None) From tverwaes at codespeak.net Sat Oct 27 14:33:00 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Sat, 27 Oct 2007 14:33:00 +0200 (CEST) Subject: [pypy-svn] r48104 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071027123300.34E35816D@code0.codespeak.net> Author: tverwaes Date: Sat Oct 27 14:32:59 2007 New Revision: 48104 Modified: pypy/dist/pypy/lang/smalltalk/model.py Log: tagging not implemented Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Sat Oct 27 14:32:59 2007 @@ -356,7 +356,7 @@ if index0 < 0: # XXX Do something useful with this.... we are not a block # of memory as smalltalk expects but wrapped in py-os - return objtable.wrap_int(0) + raise NotImplementedError() return objtable.wrap_int(ord(self.bytes[index0])) def atput0(self, index0, w_value): From fijal at codespeak.net Sat Oct 27 15:09:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 27 Oct 2007 15:09:57 +0200 (CEST) Subject: [pypy-svn] r48105 - pypy/dist/pypy/module/zipimport/test Message-ID: <20071027130957.EFB5C816A@code0.codespeak.net> Author: fijal Date: Sat Oct 27 15:09:56 2007 New Revision: 48105 Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py Log: How it could ever work. Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/test/test_zipimport.py Sat Oct 27 15:09:56 2007 @@ -149,6 +149,7 @@ assert 'uu' not in sys.modules def test_force_py(self): + import sys m0 = ord(self.test_pyc[0]) m0 ^= 0x04 test_pyc = chr(m0) + self.test_pyc[1:] @@ -172,7 +173,7 @@ del sys.modules['uuu'] def test_package(self): - import os + import os, sys self.writefile(self, "xx"+os.sep+"__init__.py", "") self.writefile(self, "xx"+os.sep+"yy.py", "def f(x): return x") mod = __import__("xx", globals(), locals(), ['yy']) From rxe at codespeak.net Sat Oct 27 15:19:39 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 27 Oct 2007 15:19:39 +0200 (CEST) Subject: [pypy-svn] r48106 - pypy/dist/pypy/translator/llvm Message-ID: <20071027131939.0EA99816C@code0.codespeak.net> Author: rxe Date: Sat Oct 27 15:19:39 2007 New Revision: 48106 Modified: pypy/dist/pypy/translator/llvm/opwriter.py Log: interiorfield/getelementptr is all we need Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Sat Oct 27 15:19:39 2007 @@ -1,7 +1,6 @@ from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype from pypy.translator.llvm.log import log -from pypy.translator.llvm.structnode import getindexhelper log = log.opwriter @@ -94,12 +93,11 @@ def _arrayindices(self, arg): ARRAYTYPE = arg.concretetype.TO + indices = [] if isinstance(ARRAYTYPE, lltype.Array): - # skip the length field - indices = [(self.uword, 1)] + indices.append((self.uword, 1)) else: assert isinstance(ARRAYTYPE, lltype.FixedSizeArray) - indices = [] return indices def write_operation(self, op): @@ -319,11 +317,12 @@ self.codewriter.call(opr.retref, 'sbyte*', '%pypy_malloc', [word], [opr.argrefs[0]]) - def _interiorhelper(self, TYPE, args): + def to_getelementptr(self, TYPE, args): indices = [] - # XXX this mess is because an FixedSizeArray can sometimes be an Array and sometimes a Struct for arg in args: name = None + # XXX this is because FixedSizeArray can sometimes be accessed + # like an Array and then sometimes a Struct if arg.concretetype is lltype.Void: name = arg.value assert name in list(TYPE._names) @@ -351,44 +350,52 @@ return TYPE, indices - def getfield(self, opr): - op = opr.op - if opr.rettype != "void": - index = getindexhelper(op.args[1].value, - op.args[0].concretetype.TO) - assert index != -1 - tmpvar = self._tmp() - self.codewriter.getelementptr(tmpvar, opr.argtypes[0], - opr.argrefs[0], [("uint", index)]) - self.codewriter.load(opr.retref, opr.rettype, tmpvar) - else: - self._skipped(opr) - def getinteriorfield(self, opr): if opr.rettype != "void": op = opr.op - _, indices = self._interiorhelper(op.args[0].concretetype.TO, op.args[1:]) + _, indices = self.to_getelementptr(op.args[0].concretetype.TO, op.args[1:]) tmpvar = self._tmp() self.codewriter.getelementptr(tmpvar, opr.argtypes[0], opr.argrefs[0], indices) self.codewriter.load(opr.retref, opr.rettype, tmpvar) else: self._skipped(opr) + # struct, name + getfield = getinteriorfield + # array, index | fixedsizearray index/name + getarrayitem = getinteriorfield + + def _getinteriorpointer(self, opr): + assert opr.rettype != "void" + op = opr.op + _, indices = self.to_getelementptr(op.args[0].concretetype.TO, op.args[1:]) + self.codewriter.getelementptr(opr.retref, opr.argtypes[0], opr.argrefs[0], indices) + + # struct, name + getsubstruct = _getinteriorpointer + # array, index | fixedsizearray, index/name + getarraysubstruct = _getinteriorpointer + def setinteriorfield(self, opr): op = opr.op if opr.argtypes[-1] != "void": print op.args, op.args[1:-1] - _, indices = self._interiorhelper(op.args[0].concretetype.TO, op.args[1:-1]) + _, indices = self.to_getelementptr(op.args[0].concretetype.TO, op.args[1:-1]) tmpvar = self._tmp() self.codewriter.getelementptr(tmpvar, opr.argtypes[0], opr.argrefs[0], indices) self.codewriter.store(opr.argtypes[-1], opr.argrefs[-1], tmpvar) else: self._skipped(opr) + bare_setinteriorfield = setinteriorfield + # struct, name, value + bare_setfield = setfield = setinteriorfield + # array, index, value | fixedsizearray, index/name, value + bare_setarrayitem = setarrayitem = setinteriorfield def getinteriorarraysize(self, opr): op = opr.op - TYPE, indices = self._interiorhelper(op.args[0].concretetype.TO, op.args[1:]) + TYPE, indices = self.to_getelementptr(op.args[0].concretetype.TO, op.args[1:]) if isinstance(TYPE, lltype.Array): # gets the length indices.append(("uint", 0)) @@ -397,10 +404,16 @@ else: assert isinstance(TYPE, lltype.FixedSizeArray) lengthref = TYPE.length - XXX # test + XXX # no tests for this self.codewriter.load(opr.retref, opr.rettype, lengthref) - def direct_fieldptr(self, opr): + # array | fixedsizearray + getarraysize = getinteriorarraysize + + def direct_fieldptr(self, opr): + from pypy.translator.llvm.structnode import getindexhelper + + # XXX use to_getelementptr ? op = opr.op assert opr.rettype != "void" index = getindexhelper(op.args[1].value, @@ -409,47 +422,14 @@ tmpvar = self._tmp() self.codewriter.getelementptr(tmpvar, opr.argtypes[0], opr.argrefs[0], [(self.uword, index)]) - # get element ptr gets a pointer to the right type, except the generated code really expected + + # getelementptr gets a pointer to the right type, except the generated code really expected # an array of size 1... so we just cast it element_type = self.db.repr_type(op.result.concretetype.TO.OF) + '*' self.codewriter.cast(opr.retref, element_type, tmpvar, opr.rettype) - def getsubstruct(self, opr): - index = getindexhelper(opr.op.args[1].value, - opr.op.args[0].concretetype.TO) - assert opr.rettype != "void" - indices = [(self.uword, index)] - self.codewriter.getelementptr(opr.retref, opr.argtypes[0], - opr.argrefs[0], indices) - - def setfield(self, opr): - op = opr.op - if opr.argtypes[2] != "void": - tmpvar = self._tmp() - index = getindexhelper(op.args[1].value, - op.args[0].concretetype.TO) - self.codewriter.getelementptr(tmpvar, opr.argtypes[0], - opr.argrefs[0], [(self.uword, index)]) - self.codewriter.store(opr.argtypes[2], opr.argrefs[2], tmpvar) - else: - self._skipped(opr) - - bare_setfield = setfield - - def getarrayitem(self, opr): - if opr.rettype == "void": - self._skipped(opr) - return - - array, index = opr.argrefs - arraytype, indextype = opr.argtypes - tmpvar = self._tmp() - - indices = self._arrayindices(opr.op.args[0]) + [(self.word, index)] - self.codewriter.getelementptr(tmpvar, arraytype, array, indices) - self.codewriter.load(opr.retref, opr.rettype, tmpvar) - def direct_arrayitems(self, opr): + # XXX use to_getelementptr ? assert opr.rettype != "void" array = opr.argrefs[0] @@ -458,7 +438,7 @@ tmpvar = self._tmp() self.codewriter.getelementptr(tmpvar, arraytype, array, indices) - # get element ptr gets a pointer to the right type, except the generated code really expected + # getelementptr gets a pointer to the right type, except the generated code really expected # an array of size 1... so we just cast it element_type = self.db.repr_type(opr.op.result.concretetype.TO.OF) + '*' self.codewriter.cast(opr.retref, element_type, tmpvar, opr.rettype) @@ -470,39 +450,10 @@ tmpvar = self._tmp() self.codewriter.getelementptr(tmpvar, arraytype, array, [(self.word, incr)]) - # get element ptr gets a pointer to the right type, except the generated code really expected + # getelementptr gets a pointer to the right type, except the generated code really expected # an array of size 1... so we just cast it element_type = self.db.repr_type(opr.op.result.concretetype.TO.OF) + '*' self.codewriter.cast(opr.retref, element_type, tmpvar, opr.rettype) - - def getarraysubstruct(self, opr): - array, index = opr.argrefs - arraytype, indextype = opr.argtypes - - indices = self._arrayindices(opr.op.args[0]) + [(self.word, index)] - self.codewriter.getelementptr(opr.retref, arraytype, array, indices) - - def setarrayitem(self, opr): - array, index, valuevar = opr.argrefs - arraytype, indextype, valuetype = opr.argtypes - tmpvar = self._tmp() - - if valuetype == "void": - self._skipped(opr) - return - - indices = self._arrayindices(opr.op.args[0]) + [(self.word, index)] - self.codewriter.getelementptr(tmpvar, arraytype, array, indices) - self.codewriter.store(valuetype, valuevar, tmpvar) - bare_setarrayitem = setarrayitem - - def getarraysize(self, opr): - ARRAYTYPE = opr.op.args[0].concretetype.TO - assert isinstance(ARRAYTYPE, lltype.Array) - tmpvar = self._tmp() - self.codewriter.getelementptr(tmpvar, opr.argtypes[0], - opr.argrefs[0], [(self.uword, 0)]) - self.codewriter.load(opr.retref, opr.rettype, tmpvar) def adr_delta(self, opr): addr1, addr2 = self._tmp(2) From rxe at codespeak.net Sat Oct 27 15:44:43 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 27 Oct 2007 15:44:43 +0200 (CEST) Subject: [pypy-svn] r48107 - pypy/dist/pypy/translator/llvm Message-ID: <20071027134443.947958164@code0.codespeak.net> Author: rxe Date: Sat Oct 27 15:44:42 2007 New Revision: 48107 Modified: pypy/dist/pypy/translator/llvm/arraynode.py pypy/dist/pypy/translator/llvm/structnode.py Log: hmmm, seems varsize mallocs are now delt with elsewhere (and why there is now an issue with VoidArrayNode). kill some code Modified: pypy/dist/pypy/translator/llvm/arraynode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/arraynode.py (original) +++ pypy/dist/pypy/translator/llvm/arraynode.py Sat Oct 27 15:44:42 2007 @@ -4,18 +4,13 @@ log = log.structnode class ArrayTypeNode(LLVMNode): - __slots__ = "db array arraytype ref constructor_ref constructor_decl".split() + __slots__ = "db array arraytype ref".split() def __init__(self, db, array): assert isinstance(array, lltype.Array) self.db = db self.array = array self.arraytype = arraytype = array.OF - # ref is used to reference the arraytype in llvm source - # constructor_ref is used to reference the constructor - # for the array type in llvm source code - # constructor_decl is used to declare the constructor - # for the array type (see writeimpl) name = "" if isinstance(arraytype, lltype.Ptr): name += "ptr_" @@ -26,11 +21,6 @@ name += str(arraytype) self.ref = self.make_ref('%arraytype_', name) - self.constructor_ref = self.make_ref('%new_array_', name) - self.constructor_decl = "%s * %s(%s %%len)" % \ - (self.ref, - self.constructor_ref, - self.db.get_machine_word()) def __str__(self): return "" % self.ref @@ -46,9 +36,6 @@ self.db.get_machine_word(), self.db.repr_type(self.arraytype)) - def var_malloc_info(self): - return self.array, () - class VoidArrayTypeNode(LLVMNode): __slots__ = "db array ref".split() @@ -147,7 +134,7 @@ s = "%s {%s}" % (self.get_typerepr(), value) return s - + class StrArrayNode(ArrayNode): __slots__ = "".split() Modified: pypy/dist/pypy/translator/llvm/structnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/structnode.py (original) +++ pypy/dist/pypy/translator/llvm/structnode.py Sat Oct 27 15:44:42 2007 @@ -63,38 +63,12 @@ class StructVarsizeTypeNode(StructTypeNode): - __slots__ = "constructor_ref constructor_decl".split() - def __init__(self, db, struct): super(StructVarsizeTypeNode, self).__init__(db, struct) - prefix = '%new_varsizestruct_' - self.constructor_ref = self.make_ref(prefix, self.name) - self.constructor_decl = "%s * %s(%s %%len)" % \ - (self.ref, - self.constructor_ref, - self.db.get_machine_word()) def __str__(self): return "" %(self.ref,) - # ______________________________________________________________________ - # main entry points from genllvm - - def var_malloc_info(self): - # build up a list of indices to get to the last - # var-sized struct (or rather the according array) - indices_to_array = [] - current = self.struct - while isinstance(current, lltype.Struct): - last_pos = len(current._names_without_voids()) - 1 - # struct requires uint consts - indices_to_array.append(("uint", last_pos)) - name = current._names_without_voids()[-1] - current = current._flds[name] - assert isinstance(current, lltype.Array) - - return current, indices_to_array - class StructNode(ConstantLLVMNode): """ A struct constant. Can simply contain a primitive, From rxe at codespeak.net Sat Oct 27 15:59:01 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 27 Oct 2007 15:59:01 +0200 (CEST) Subject: [pypy-svn] r48108 - pypy/dist/pypy/translator/llvm Message-ID: <20071027135901.346AF8164@code0.codespeak.net> Author: rxe Date: Sat Oct 27 15:58:59 2007 New Revision: 48108 Modified: pypy/dist/pypy/translator/llvm/database.py Log: void arrays dont really fit into the generic way of getting offsets, so special case them. also add some code for preparing offsets, which is likely to be only needed in small test code Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Sat Oct 27 15:58:59 2007 @@ -194,7 +194,10 @@ return ## elif ct is llmemory.WeakGcAddress: ## return # XXX sometime soon + else: + if isinstance(const_or_var.value, llmemory.AddressOffset): + self.prepare_offset(const_or_var.value) return else: assert isinstance(ct, lltype.Ptr), "Preparation of non primitive and non pointer" @@ -223,6 +226,12 @@ self.prepare_type(const_or_var.concretetype) self.prepare_arg_value(const_or_var) + def prepare_offset(self, offset): + if isinstance(offset, llmemory.CompositeOffset): + for value in offset.offsets: + self.prepare_offset(value) + else: + self.prepare_type(offset.TYPE) def setup_all(self): while self._pendingsetup: @@ -322,9 +331,6 @@ self._tmpcount += 1 return "%tmp_" + str(count) - def repr_constructor(self, type_): - return self.obj2node[type_].constructor_ref - def repr_name(self, obj): " simply returns a reference to constant value " return self.obj2node[obj].ref @@ -496,15 +502,12 @@ repr = 0 elif isinstance(value, llmemory.AddressOffset): - from_, indices, to = self.get_offset(value) - indices_as_str = ", ".join("%s %s" % (w, i) for w, i in indices) - r = self.database.repr_type - repr = "cast(%s* getelementptr(%s* null, %s) to int)" % (r(to), - r(from_), - indices_as_str) + repr = self.repr_offset(value) + elif isinstance(value, ComputedIntSymbolic): # force the ComputedIntSymbolic to become a real integer value now repr = '%d' % value.compute_fn() + elif isinstance(value, CDefinedIntSymbolic): if value is objectmodel.malloc_zero_filled: repr = '1' @@ -517,8 +520,34 @@ return repr + def repr_offset(self, value): + from_, indices, to = self.get_offset(value) + + # void array special cases + if isinstance(from_, lltype.Array) and from_.OF is lltype.Void: + assert not isinstance(value, (llmemory.FieldOffset, llmemory.ItemOffset)) + if isinstance(value, llmemory.ArrayLengthOffset): + pass # ok cases! + elif isinstance(value, llmemory.ArrayItemsOffset): + to = from_ + indices = [(self.database.get_machine_word(), 1)] + else: + s = value.offsets[0] + isinstance(value, llmemory.CompositeOffset) + return self.repr_offset(s) + + if from_ is lltype.Void: + assert isinstance(value, llmemory.ItemOffset) + return "0" + + r = self.database.repr_type + indices_as_str = ", ".join("%s %s" % (w, i) for w, i in indices) + return "cast(%s* getelementptr(%s* null, %s) to int)" % (r(to), + r(from_), + indices_as_str) + def get_offset(self, value, initialindices=None): - " return (from_type, (indices, ...), to_type) " + " return (from_type, (indices, ...), to_type) " word = self.database.get_machine_word() uword = self.database.get_machine_uword() indices = initialindices or [(word, 0)] From rxe at codespeak.net Sat Oct 27 16:01:43 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 27 Oct 2007 16:01:43 +0200 (CEST) Subject: [pypy-svn] r48109 - pypy/dist/pypy/translator/llvm Message-ID: <20071027140143.C69218173@code0.codespeak.net> Author: rxe Date: Sat Oct 27 16:01:43 2007 New Revision: 48109 Modified: pypy/dist/pypy/translator/llvm/arraynode.py pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/opwriter.py Log: attempt at arrays with no length Modified: pypy/dist/pypy/translator/llvm/arraynode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/arraynode.py (original) +++ pypy/dist/pypy/translator/llvm/arraynode.py Sat Oct 27 16:01:43 2007 @@ -32,9 +32,13 @@ # entry points from genllvm # def writedatatypedecl(self, codewriter): - codewriter.arraydef(self.ref, - self.db.get_machine_word(), - self.db.repr_type(self.arraytype)) + if self.array._hints.get("nolength", False): + codewriter.arraynolendef(self.ref, + self.db.repr_type(self.arraytype)) + else: + codewriter.arraydef(self.ref, + self.db.get_machine_word(), + self.db.repr_type(self.arraytype)) class VoidArrayTypeNode(LLVMNode): __slots__ = "db array ref".split() @@ -46,6 +50,7 @@ self.ref = "%arraytype_Void" def writedatatypedecl(self, codewriter): + assert not self.array._hints.get("nolength", False) codewriter.typedef(self.ref, "{ %s }" % self.db.get_machine_word()) class ArrayNode(ConstantLLVMNode): @@ -135,6 +140,29 @@ s = "%s {%s}" % (self.get_typerepr(), value) return s +class ArrayNoLengthNode(ArrayNode): + def get_typerepr(self): + arraylen = self.get_arrayvalue()[0] + typeval = self.db.repr_type(self.arraytype) + return "{ [%s x %s] }" % (arraylen, typeval) + + def get_childref(self, index): + return "getelementptr(%s* %s, int 0, int %s)" %( + self.get_typerepr(), + self.ref, + index) + + def constantvalue(self): + physicallen, arrayrepr = self.get_arrayvalue() + typeval = self.db.repr_type(self.arraytype) + + value = "[%s x %s] %s" % (physicallen, + typeval, + arrayrepr) + + s = "%s {%s}" % (self.get_typerepr(), value) + return s + class StrArrayNode(ArrayNode): __slots__ = "".split() Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Sat Oct 27 16:01:43 2007 @@ -8,7 +8,7 @@ StructTypeNode, StructVarsizeTypeNode, getindexhelper, \ FixedSizeArrayTypeNode, FixedSizeArrayNode from pypy.translator.llvm.arraynode import ArrayNode, StrArrayNode, \ - VoidArrayNode, ArrayTypeNode, VoidArrayTypeNode + VoidArrayNode, ArrayNoLengthNode, ArrayTypeNode, VoidArrayTypeNode from pypy.translator.llvm.opaquenode import OpaqueNode, ExtOpaqueNode, \ OpaqueTypeNode, ExtOpaqueTypeNode from pypy.rpython.lltypesystem import lltype, llmemory @@ -82,7 +82,10 @@ elif type_.OF is lltype.Void: node = VoidArrayNode(self, value) else: - node = ArrayNode(self, value) + if type_._hints.get("nolength", False): + node = ArrayNoLengthNode(self, value) + else: + node = ArrayNode(self, value) elif isinstance(type_, lltype.OpaqueType): if hasattr(type_, '_exttypeinfo'): @@ -571,13 +574,14 @@ # jumps to the place where the array length is stored from_ = value.TYPE # or assert isinstance(value.TYPE, lltype.Array) - indices.append((uword, 0)) + if not value.TYPE._hints.get("nolength", False): + indices.append((uword, 0)) to = lltype.Signed elif isinstance(value, llmemory.ArrayItemsOffset): # jumps to the beginning of array area from_ = value.TYPE - if not isinstance(value.TYPE, lltype.FixedSizeArray): + if not isinstance(value.TYPE, lltype.FixedSizeArray) and not value.TYPE._hints.get("nolength", False): indices.append((uword, 1)) indices.append((word, 0)) # go to the 1st item to = value.TYPE.OF Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Sat Oct 27 16:01:43 2007 @@ -95,7 +95,9 @@ ARRAYTYPE = arg.concretetype.TO indices = [] if isinstance(ARRAYTYPE, lltype.Array): - indices.append((self.uword, 1)) + if not ARRAYTYPE._hints.get("nolength", False): + # skip the length field + indices.append((self.uword, 1)) else: assert isinstance(ARRAYTYPE, lltype.FixedSizeArray) return indices @@ -336,7 +338,8 @@ TYPE = TYPE.OF elif isinstance(TYPE, lltype.Array): - indices.append(("uint", 1)) + if not TYPE._hints.get("nolength", False): + indices.append(("uint", 1)) indices.append(("int", indexref)) TYPE = TYPE.OF From rxe at codespeak.net Sat Oct 27 16:08:58 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 27 Oct 2007 16:08:58 +0200 (CEST) Subject: [pypy-svn] r48110 - pypy/dist/pypy/translator/llvm Message-ID: <20071027140858.61C178178@code0.codespeak.net> Author: rxe Date: Sat Oct 27 16:08:58 2007 New Revision: 48110 Modified: pypy/dist/pypy/translator/llvm/codewriter.py Log: oops that went with rev 48108 Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Sat Oct 27 16:08:58 2007 @@ -78,6 +78,9 @@ def arraydef(self, name, lentype, typerepr): self.typedef(name, "{ %s, [0 x %s] }" % (lentype, typerepr)) + def arraynolendef(self, name, typerepr): + self.typedef(name, "[0 x %s]" % (typerepr)) + def fixedarraydef(self, name, arraylen, typerepr): self.typedef(name, "[%s x %s]" % (arraylen, typerepr)) From rxe at codespeak.net Sat Oct 27 16:19:23 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 27 Oct 2007 16:19:23 +0200 (CEST) Subject: [pypy-svn] r48111 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071027141923.E14318170@code0.codespeak.net> Author: rxe Date: Sat Oct 27 16:19:23 2007 New Revision: 48111 Modified: pypy/dist/pypy/translator/llvm/test/test_symbolic.py pypy/dist/pypy/translator/llvm/test/test_typed.py Log: enable tests Modified: pypy/dist/pypy/translator/llvm/test/test_symbolic.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_symbolic.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_symbolic.py Sat Oct 27 16:19:23 2007 @@ -22,7 +22,6 @@ assert res == 12 def test_sizeof_array_with_no_length(): - py.test.skip("inprogress") A = lltype.GcArray(lltype.Signed, hints={'nolength': True}) a = lltype.malloc(A, 5, zero=True) @@ -166,7 +165,6 @@ assert fn(21) == 42 def test_itemoffset_void(): - py.test.skip("inprogress") A = lltype.GcArray(lltype.Void) s = llmemory.sizeof(A, 1) s += llmemory.sizeof(lltype.Signed) Modified: pypy/dist/pypy/translator/llvm/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_typed.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_typed.py Sat Oct 27 16:19:23 2007 @@ -54,7 +54,6 @@ assert result def test_is(): - py.test.skip("inprogress") def testfn(): l1 = [] return l1 is l1 @@ -69,7 +68,6 @@ assert result == False def test_nones(): - py.test.skip("inprogress") a = [None] * 4 def nones(): a.append(None) From cfbolz at codespeak.net Sat Oct 27 17:06:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 27 Oct 2007 17:06:40 +0200 (CEST) Subject: [pypy-svn] r48112 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071027150640.02F508162@code0.codespeak.net> Author: cfbolz Date: Sat Oct 27 17:06:39 2007 New Revision: 48112 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: add a new "no_result" argument to expose_primitive, which defaults to False. Make the primitive wrapper responsible for pushing the result onto the stack. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Sat Oct 27 17:06:39 2007 @@ -178,11 +178,7 @@ print "PRIMITIVE FAILED: %d %s" % (method.primitive, selector,) pass # ignore this error and fall back to the Smalltalk version else: - # the primitive succeeded - # Make sure that primitives that do not want to return - # anything, don't have to return anything (aBlock value) - if w_result: - self.push(w_result) + # the primitive pushes the result (if any) onto the stack itself return start = len(self.stack) - argcount assert start >= 0 # XXX check in the Blue Book what to do in this case @@ -346,9 +342,7 @@ try: # note that argcount does not include self w_result = primitives.prim_table[primitive](interp, argcount) - if w_result: # enable primitives not to return values to - # the stack - self.push(w_result) + # the primitive pushes the result (if any) onto the stack itself except primitives.PrimitiveFailedError: self._sendSelfSelector(selector, argcount, interp) #self._sendSelfSelector(selector, argcount, interp) Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Sat Oct 27 17:06:39 2007 @@ -58,7 +58,7 @@ # converted to an index0 index1_0 = object() -def expose_primitive(code, unwrap_spec=None): +def expose_primitive(code, unwrap_spec=None, no_result=False): # some serious magic, don't look from pypy.rlib.unroll import unrolling_iterable # heuristics to give it a nice name @@ -78,40 +78,46 @@ assert code not in prim_table func.func_name = "prim_" + name if unwrap_spec is None: - prim_table[code] = func - return func - for spec in unwrap_spec: - assert spec in (int, float, object, index1_0, str) - len_unwrap_spec = len(unwrap_spec) - assert (len_unwrap_spec == len(inspect.getargspec(func)[0]) + 1, - "wrong number of arguments") - unrolling_unwrap_spec = unrolling_iterable(enumerate(unwrap_spec)) - def wrapped(interp, argument_count_m1): - argument_count = argument_count_m1 + 1 # to account for the rcvr - frame = interp.w_active_context - assert argument_count == len_unwrap_spec - if len(frame.stack) < len_unwrap_spec: - raise PrimitiveFailedError() - args = () - for i, spec in unrolling_unwrap_spec: - index = -len_unwrap_spec + i - arg = frame.stack[index] - if spec is int: - args += (unwrap_int(arg), ) - elif spec is index1_0: - args += (unwrap_int(arg)-1, ) - elif spec is float: - args += (unwrap_float(arg), ) - elif spec is object: - args += (arg, ) - elif spec is str: - args += (arg.as_string(), ) - else: - assert 0, "this should never happen" - res = func(interp, *args) - frame.pop_n(len_unwrap_spec) # only if no exception occurs! - return res - + def wrapped(interp, argument_count_m1): + w_result = func(interp, argument_count_m1) + if not no_result: + assert w_result is not None + interp.w_active_context.push(w_result) + return w_result + else: + for spec in unwrap_spec: + assert spec in (int, float, object, index1_0, str) + len_unwrap_spec = len(unwrap_spec) + assert (len_unwrap_spec == len(inspect.getargspec(func)[0]) + 1, + "wrong number of arguments") + unrolling_unwrap_spec = unrolling_iterable(enumerate(unwrap_spec)) + def wrapped(interp, argument_count_m1): + argument_count = argument_count_m1 + 1 # to account for the rcvr + frame = interp.w_active_context + assert argument_count == len_unwrap_spec + if len(frame.stack) < len_unwrap_spec: + raise PrimitiveFailedError() + args = () + for i, spec in unrolling_unwrap_spec: + index = -len_unwrap_spec + i + arg = frame.stack[index] + if spec is int: + args += (unwrap_int(arg), ) + elif spec is index1_0: + args += (unwrap_int(arg)-1, ) + elif spec is float: + args += (unwrap_float(arg), ) + elif spec is object: + args += (arg, ) + elif spec is str: + args += (arg.as_string(), ) + else: + assert 0, "this should never happen" + w_result = func(interp, *args) + frame.pop_n(len_unwrap_spec) # only if no exception occurs! + if not no_result: + assert w_result is not None + interp.w_active_context.push(w_result) wrapped.func_name = "wrap_prim_" + name prim_table[code] = wrapped return func @@ -479,7 +485,7 @@ def func(interp, w_rcvr): raise PrimitiveNotYetWrittenError() - at expose_primitive(CHANGE_CLASS, unwrap_spec=[object, object]) + at expose_primitive(CHANGE_CLASS, unwrap_spec=[object, object], no_result=True) def func(interp, w_arg, w_rcvr): w_arg_class = w_arg.getclass() w_rcvr_class = w_rcvr.getclass() @@ -493,7 +499,7 @@ raise PrimitiveFailedError() # 2. Rcvr is an instance of a compact class and argument isn't - # or vice versa (?) + # or vice versa XXX we don't have to fail here, but for squeak it's a problem # 3. Format of rcvr is different from format of argument raise PrimitiveNotYetWrittenError() # XXX needs to work in the shadows @@ -506,7 +512,6 @@ raise PrimitiveFailedError() w_rcvr.w_class = w_arg.w_class - return # ___________________________________________________________________________ # Squeak Miscellaneous Primitives (128-149) @@ -655,7 +660,7 @@ w_block_ctx.w_sender = frame interp.w_active_context = w_block_ctx - at expose_primitive(PRIMITIVE_VALUE) + at expose_primitive(PRIMITIVE_VALUE, no_result=True) def func(interp, argument_count): # argument_count does NOT include the receiver. # This means that for argument_count == 3 the stack looks like: @@ -681,12 +686,9 @@ frame.pop() finalize_block_ctx(interp, w_block_ctx, frame) - # Value is a special case of primitive which does not return - # anything - return None - - at expose_primitive(PRIMITIVE_VALUE_WITH_ARGS, unwrap_spec=[object, object]) + at expose_primitive(PRIMITIVE_VALUE_WITH_ARGS, unwrap_spec=[object, object], + no_result=True) def func(interp, w_block_ctx, w_args): if not isinstance(w_block_ctx, model.W_BlockContext): raise PrimitiveFailedError() @@ -707,16 +709,13 @@ finalize_block_ctx(interp, w_block_ctx, interp.w_active_context) - # Value: is a special case of primitive which does not return - # anything - return None - @expose_primitive(PRIMITIVE_PERFORM) def func(interp, argcount): raise PrimitiveFailedError() @expose_primitive(PRIMITIVE_PERFORM_WITH_ARGS, - unwrap_spec=[object, str, object]) + unwrap_spec=[object, str, object], + no_result=True) def func(interp, w_rcvr, sel, w_args): w_method = w_rcvr.shadow_of_my_class().lookup(sel) assert w_method @@ -727,9 +726,6 @@ w_frame.w_sender = interp.w_active_context interp.w_active_context = w_frame - # Don't put anything on the stack - return None - @expose_primitive(PRIMITIVE_SIGNAL, unwrap_spec=[object]) def func(interp, w_rcvr): raise PrimitiveNotYetWrittenError() Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Sat Oct 27 17:06:39 2007 @@ -32,7 +32,8 @@ def prim(code, stack): interp, argument_count = mock(stack) - res = prim_table[code](interp, argument_count-1) + prim_table[code](interp, argument_count-1) + res = interp.w_active_context.pop() assert not len(interp.w_active_context.stack) # check args are consumed return res From fijal at codespeak.net Sat Oct 27 18:16:53 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 27 Oct 2007 18:16:53 +0200 (CEST) Subject: [pypy-svn] r48113 - pypy/dist/pypy/translator/c/test Message-ID: <20071027161653.2E7B0817D@code0.codespeak.net> Author: fijal Date: Sat Oct 27 18:16:52 2007 New Revision: 48113 Modified: pypy/dist/pypy/translator/c/test/test_database.py Log: Fix those tests Modified: pypy/dist/pypy/translator/c/test/test_database.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_database.py (original) +++ pypy/dist/pypy/translator/c/test/test_database.py Sat Oct 27 18:16:52 2007 @@ -8,6 +8,8 @@ def dump_on_stdout(database): + if database.gctransformer: + database.prepare_inline_helpers() print '/*********************************/' structdeflist = database.getstructdeflist() for node in structdeflist: From fijal at codespeak.net Sat Oct 27 18:31:17 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 27 Oct 2007 18:31:17 +0200 (CEST) Subject: [pypy-svn] r48114 - pypy/dist/pypy/translator/backendopt Message-ID: <20071027163117.6549B8174@code0.codespeak.net> Author: fijal Date: Sat Oct 27 18:31:17 2007 New Revision: 48114 Modified: pypy/dist/pypy/translator/backendopt/constfold.py Log: Revert 47798 as this have broke tests. I don't understand how to fix it properly though. Modified: pypy/dist/pypy/translator/backendopt/constfold.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/constfold.py (original) +++ pypy/dist/pypy/translator/backendopt/constfold.py Sat Oct 27 18:31:17 2007 @@ -31,7 +31,7 @@ except AttributeError: sideeffects = True else: - if op.tryfold and len(args) == len(vargs): + if len(args) == len(vargs): RESTYPE = spaceop.result.concretetype try: result = op(RESTYPE, *args) From arigo at codespeak.net Sat Oct 27 19:20:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 27 Oct 2007 19:20:23 +0200 (CEST) Subject: [pypy-svn] r48115 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071027172023.83676817B@code0.codespeak.net> Author: arigo Date: Sat Oct 27 19:20:22 2007 New Revision: 48115 Modified: pypy/dist/pypy/translator/backendopt/constfold.py pypy/dist/pypy/translator/backendopt/test/test_constfold.py Log: Reintroduce 47798 in a more conservative version. The flags in lloperation are not really consistent. Added a test for what I'm trying to achieve with this. Modified: pypy/dist/pypy/translator/backendopt/constfold.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/constfold.py (original) +++ pypy/dist/pypy/translator/backendopt/constfold.py Sat Oct 27 19:20:22 2007 @@ -31,7 +31,8 @@ except AttributeError: sideeffects = True else: - if len(args) == len(vargs): + sideeffects = op.sideeffects + if not sideeffects and len(args) == len(vargs): RESTYPE = spaceop.result.concretetype try: result = op(RESTYPE, *args) @@ -49,7 +50,6 @@ constants[spaceop.result] = Constant(result, RESTYPE) folded_count += 1 continue - sideeffects = op.sideeffects # failed to fold an operation, exit early if requested if exit_early: return folded_count Modified: pypy/dist/pypy/translator/backendopt/test/test_constfold.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_constfold.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_constfold.py Sat Oct 27 19:20:22 2007 @@ -3,6 +3,7 @@ from pypy.translator.translator import TranslationContext, graphof from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib import objectmodel from pypy.translator.backendopt.constfold import constant_fold_graph from pypy import conftest @@ -234,3 +235,13 @@ assert summary(graph) == {'getarrayitem': 1} check_graph(graph, [], 1234, t) + + +def test_dont_constfold_debug_print(): + def fn(): + llop.debug_print(lltype.Void, "hello world") + + graph, t = get_graph(fn, []) + assert summary(graph) == {'debug_print': 1} + constant_fold_graph(graph) + assert summary(graph) == {'debug_print': 1} From fijal at codespeak.net Sat Oct 27 23:46:13 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 27 Oct 2007 23:46:13 +0200 (CEST) Subject: [pypy-svn] r48116 - pypy/dist/pypy/annotation Message-ID: <20071027214613.7905E8177@code0.codespeak.net> Author: fijal Date: Sat Oct 27 23:46:11 2007 New Revision: 48116 Modified: pypy/dist/pypy/annotation/classdef.py Log: Remove somewhat old debugging code Modified: pypy/dist/pypy/annotation/classdef.py ============================================================================== --- pypy/dist/pypy/annotation/classdef.py (original) +++ pypy/dist/pypy/annotation/classdef.py Sat Oct 27 23:46:11 2007 @@ -299,12 +299,6 @@ def generalize_attr(self, attr, s_value=None): # if the attribute exists in a superclass, generalize there, # as imposed by invariant (I) - #start debug - #if self.name.endswith('W_Root') and attr == 'setdata': - # print 'NAME:',self.name - # import pdb - # pdb.set_trace() - #stop debug for clsdef in self.getmro(): if attr in clsdef.attrs: clsdef._generalize_attr(attr, s_value) From rxe at codespeak.net Sat Oct 27 23:48:45 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 27 Oct 2007 23:48:45 +0200 (CEST) Subject: [pypy-svn] r48117 - pypy/dist/pypy/translator/c/src Message-ID: <20071027214845.E92D38177@code0.codespeak.net> Author: rxe Date: Sat Oct 27 23:48:45 2007 New Revision: 48117 Removed: pypy/dist/pypy/translator/c/src/ll_time.h Modified: pypy/dist/pypy/translator/c/src/g_include.h Log: remove old c code for ll_time module - replaced with rffi version Modified: pypy/dist/pypy/translator/c/src/g_include.h ============================================================================== --- pypy/dist/pypy/translator/c/src/g_include.h (original) +++ pypy/dist/pypy/translator/c/src/g_include.h Sat Oct 27 23:48:45 2007 @@ -53,7 +53,6 @@ # include "src/rtyper.h" #ifndef AVR # include "src/ll_os.h" -# include "src/ll_time.h" # include "src/ll_math.h" # include "src/ll_strtod.h" # ifdef RPyExc_thread_error From rxe at codespeak.net Sun Oct 28 00:25:37 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sun, 28 Oct 2007 00:25:37 +0200 (CEST) Subject: [pypy-svn] r48119 - pypy/dist/pypy/translator/llvm/module Message-ID: <20071027222537.1CD52817C@code0.codespeak.net> Author: rxe Date: Sun Oct 28 00:25:36 2007 New Revision: 48119 Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c Log: remove ll_time.h from being compiled in Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c ============================================================================== --- pypy/dist/pypy/translator/llvm/module/genexterns.c (original) +++ pypy/dist/pypy/translator/llvm/module/genexterns.c Sun Oct 28 00:25:36 2007 @@ -1,6 +1,5 @@ // append some genc files here manually from python -//#include "c/src/support.h" #ifdef _RPyListOfString_New /* :-( */ # define HAVE_RPY_LIST_OF_STRING #endif @@ -8,9 +7,7 @@ #include "c/src/thread.h" //#include "c/src/ll_os.h" #include "c/src/ll_math.h" -#include "c/src/ll_time.h" #include "c/src/ll_strtod.h" -//#include "c/src/ll_thread.h" #include "c/src/stack.h" // setup code for ThreadLock Opaque types From rxe at codespeak.net Sun Oct 28 00:26:30 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sun, 28 Oct 2007 00:26:30 +0200 (CEST) Subject: [pypy-svn] r48120 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071027222630.08940817C@code0.codespeak.net> Author: rxe Date: Sun Oct 28 00:26:29 2007 New Revision: 48120 Modified: pypy/dist/pypy/translator/llvm/test/test_lltype.py Log: get more tests from genc Modified: pypy/dist/pypy/translator/llvm/test/test_lltype.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lltype.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lltype.py Sun Oct 28 00:26:29 2007 @@ -434,7 +434,7 @@ assert res == 34 def test_union(): - py.test.skip("unions!!") + py.test.skip("not unions!!") U = Struct('U', ('s', Signed), ('c', Char), hints={'union': True}) u = malloc(U, immortal=True) @@ -481,3 +481,479 @@ fn = compile_function(llf, []) res = fn() assert res == 8765 + +def test_malloc_array_void(): + A = GcArray(Void) + def llf(): + a1 = malloc(A, 5, zero=True) + return len(a1) + fn = compile_function(llf, []) + res = fn() + assert res == 5 + +def test_sizeof_void_array(): + from pypy.rpython.lltypesystem import llmemory + A = Array(Void) + size1 = llmemory.sizeof(A, 1) + size2 = llmemory.sizeof(A, 14) + def f(x): + if x: + return size1 + else: + return size2 + fn = compile_function(f, [int]) + res1 = fn(1) + res2 = fn(0) + assert res1 == res2 + +def test_null_padding(): + from pypy.rpython.lltypesystem import llmemory + from pypy.rpython.lltypesystem import rstr + chars_offset = llmemory.FieldOffset(rstr.STR, 'chars') + \ + llmemory.ArrayItemsOffset(rstr.STR.chars) + # sadly, there's no way of forcing this to fail if the strings + # are allocated in a region of memory such that they just + # happen to get a NUL byte anyway :/ (a debug build will + # always fail though) + def trailing_byte(s): + adr_s = llmemory.cast_ptr_to_adr(s) + return (adr_s + chars_offset).char[len(s)] + def f(x): + r = 0 + for i in range(x): + r += ord(trailing_byte(' '*(100-x*x))) + return r + fn = compile_function(f, [int]) + res = fn(10) + assert res == 0 + +def test_simplearray_nolength(): + A = GcArray(Signed, hints={'nolength': True}) + def llf(): + s = malloc(A, 5) + s[0] = 1 + s[1] = 2 + return s[0] + s[1] + fn = compile_function(llf, []) + assert fn() == 3 + +def test_array_nolength(): + A = Array(Signed, hints={'nolength': True}) + a1 = malloc(A, 3, immortal=True) + a1[0] = 30 + a1[1] = 300 + a1[2] = 3000 + a1dummy = malloc(A, 2, immortal=True) + + def f(n): + if n & 1: + src = a1dummy + else: + src = a1 + a2 = malloc(A, n, flavor='raw') + for i in range(n): + a2[i] = src[i % 3] + i + res = a2[n // 2] + free(a2, flavor='raw') + return res + + fn = compile_function(f, [int]) + res = fn(100) + assert res == 3050 + +def test_prebuilt_integers(): + from pypy.rlib.unroll import unrolling_iterable + from pypy.rpython.lltypesystem import rffi + class Prebuilt: + pass + p = Prebuilt() + NUMBER_TYPES = rffi.NUMBER_TYPES + names = unrolling_iterable([TYPE.__name__ for TYPE in NUMBER_TYPES]) + for name, TYPE in zip(names, NUMBER_TYPES): + value = cast_primitive(TYPE, 1) + setattr(p, name, value) + + def f(x): + total = x + for name in names: + total += rffi.cast(Signed, getattr(p, name)) + return total + + fn = compile_function(f, [int]) + res = fn(100) + assert res == 100 + len(list(names)) + +def test_array_nolength(): + A = Array(Signed, hints={'nolength': True}) + a1 = malloc(A, 3, immortal=True) + a1[0] = 30 + a1[1] = 300 + a1[2] = 3000 + a1dummy = malloc(A, 2, immortal=True) + + def f(n): + if n & 1: + src = a1dummy + else: + src = a1 + a2 = malloc(A, n, flavor='raw') + for i in range(n): + a2[i] = src[i % 3] + i + res = a2[n // 2] + free(a2, flavor='raw') + return res + + fn = compile_function(f, [int]) + res = fn(100) + assert res == 3050 + +def test_gcarray_nolength(): + A = GcArray(Signed, hints={'nolength': True}) + a1 = malloc(A, 3, immortal=True) + a1[0] = 30 + a1[1] = 300 + a1[2] = 3000 + a1dummy = malloc(A, 2, immortal=True) + + def f(n): + if n & 1: + src = a1dummy + else: + src = a1 + a2 = malloc(A, n) + for i in range(n): + a2[i] = src[i % 3] + i + res = a2[n // 2] + return res + + fn = compile_function(f, [int]) + res = fn(100) + assert res == 3050 + +def test_structarray_nolength(): + S = Struct('S', ('x', Signed)) + A = Array(S, hints={'nolength': True}) + a1 = malloc(A, 3, immortal=True) + a1[0].x = 30 + a1[1].x = 300 + a1[2].x = 3000 + a1dummy = malloc(A, 2, immortal=True) + + def f(n): + if n & 1: + src = a1dummy + else: + src = a1 + a2 = malloc(A, n, flavor='raw') + for i in range(n): + a2[i].x = src[i % 3].x + i + res = a2[n // 2].x + free(a2, flavor='raw') + return res + + fn = compile_function(f, [int]) + res = fn(100) + assert res == 3050 + +def test_zero_raw_malloc(): + S = Struct('S', ('x', Signed), ('y', Signed)) + def f(n): + for i in range(n): + p = malloc(S, flavor='raw', zero=True) + if p.x != 0 or p.y != 0: + return -1 + p.x = i + p.y = i + free(p, flavor='raw') + return 42 + + fn = compile_function(f, [int]) + res = fn(100) + assert res == 42 + +def test_zero_raw_malloc_varsize(): + # we don't support at the moment raw+zero mallocs with a length + # field to initialize + S = Struct('S', ('x', Signed), ('y', Array(Signed, hints={'nolength': True}))) + def f(n): + for length in range(n-1, -1, -1): + p = malloc(S, length, flavor='raw', zero=True) + if p.x != 0: + return -1 + p.x = n + for j in range(length): + if p.y[j] != 0: + return -3 + p.y[j] = n^j + free(p, flavor='raw') + return 42 + + fn = compile_function(f, [int]) + res = fn(100) + assert res == 42 + +def test_direct_ptradd_barebone(): + from pypy.rpython.lltypesystem import rffi + ARRAY_OF_CHAR = Array(Char, hints={'nolength': True}) + + def llf(): + data = "hello, world!" + a = malloc(ARRAY_OF_CHAR, len(data), flavor='raw') + for i in xrange(len(data)): + a[i] = data[i] + a2 = rffi.ptradd(a, 2) + assert typeOf(a2) == typeOf(a) == Ptr(ARRAY_OF_CHAR) + for i in xrange(len(data) - 2): + assert a2[i] == a[i + 2] + free(a, flavor='raw') + return 0 + + fn = compile_function(llf, []) + fn() + +def test_prebuilt_nolength_array(): + A = Array(Signed, hints={'nolength': True}) + a = malloc(A, 5, immortal=True) + a[0] = 8 + a[1] = 5 + a[2] = 12 + a[3] = 12 + a[4] = 15 + def llf(): + s = '' + for i in range(5): + s += chr(64+a[i]) + assert s == "HELLO" + return 0 + + fn = compile_function(llf, []) + fn() + +class TestLowLevelType(object): + def getcompiled(self, f, args=[]): + return compile_function(f, args) + + def test_direct_arrayitems(self): + py.test.skip("nolength ???") + for a in [malloc(GcArray(Signed), 5), + malloc(FixedSizeArray(Signed, 5), immortal=True), + malloc(Array(Signed, hints={'nolength': True}), 5, immortal=True), + ]: + a[0] = 0 + a[1] = 10 + a[2] = 20 + a[3] = 30 + a[4] = 40 + b0 = direct_arrayitems(a) + b1 = direct_ptradd(b0, 1) + b2 = direct_ptradd(b1, 1) + def llf(n): + b0 = direct_arrayitems(a) + b3 = direct_ptradd(direct_ptradd(b0, 5), -2) + saved = a[n] + a[n] = 1000 + try: + return b0[0] + b3[-2] + b2[1] + b1[3] + finally: + a[n] = saved + fn = self.getcompiled(llf, [int]) + res = fn(0) + assert res == 1000 + 10 + 30 + 40 + res = fn(1) + assert res == 0 + 1000 + 30 + 40 + res = fn(2) + assert res == 0 + 10 + 30 + 40 + res = fn(3) + assert res == 0 + 10 + 1000 + 40 + res = fn(4) + assert res == 0 + 10 + 30 + 1000 + + def test_arithmetic_cornercases(self): + py.test.skip("pyobject in this test - but why ???") + import operator, sys + from pypy.rlib.unroll import unrolling_iterable + from pypy.rlib.rarithmetic import r_longlong, r_ulonglong + + class Undefined: + def __eq__(self, other): + return True + undefined = Undefined() + + def getmin(cls): + if cls is int: + return -sys.maxint-1 + elif cls.SIGNED: + return cls(-(cls.MASK>>1)-1) + else: + return cls(0) + getmin._annspecialcase_ = 'specialize:memo' + + def getmax(cls): + if cls is int: + return sys.maxint + elif cls.SIGNED: + return cls(cls.MASK>>1) + else: + return cls(cls.MASK) + getmax._annspecialcase_ = 'specialize:memo' + maxlonglong = long(getmax(r_longlong)) + + classes = unrolling_iterable([int, r_uint, r_longlong, r_ulonglong]) + operators = unrolling_iterable([operator.add, + operator.sub, + operator.mul, + operator.floordiv, + operator.mod, + operator.lshift, + operator.rshift]) + def f(n): + result = () + for cls in classes: + values = [getmin(cls), getmax(cls)] + for OP in operators: + for x in values: + res1 = OP(x, n) + result += (res1,) + return result + + def assert_eq(a, b): + # for better error messages when it fails + assert len(a) == len(b) + for i in range(len(a)): + assert a[i] == b[i] + + fn = self.getcompiled(f, [int]) + res = fn(1) + print res + assert_eq(res, ( + # int + -sys.maxint, undefined, # add + undefined, sys.maxint-1, # sub + -sys.maxint-1, sys.maxint, # mul + -sys.maxint-1, sys.maxint, # floordiv + 0, 0, # mod + 0, -2, # lshift + (-sys.maxint-1)//2, sys.maxint//2, # rshift + # r_uint + 1, 0, # add + sys.maxint*2+1, sys.maxint*2, # sub + 0, sys.maxint*2+1, # mul + 0, sys.maxint*2+1, # floordiv + 0, 0, # mod + 0, sys.maxint*2, # lshift + 0, sys.maxint, # rshift + # r_longlong + -maxlonglong, undefined, # add + undefined, maxlonglong-1, # sub + -maxlonglong-1, maxlonglong, # mul + -maxlonglong-1, maxlonglong, # floordiv + 0, 0, # mod + 0, -2, # lshift + (-maxlonglong-1)//2, maxlonglong//2, # rshift + # r_ulonglong + 1, 0, # add + maxlonglong*2+1, maxlonglong*2, # sub + 0, maxlonglong*2+1, # mul + 0, maxlonglong*2+1, # floordiv + 0, 0, # mod + 0, maxlonglong*2, # lshift + 0, maxlonglong, # rshift + )) + + res = fn(5) + print res + assert_eq(res, ( + # int + -sys.maxint+4, undefined, # add + undefined, sys.maxint-5, # sub + undefined, undefined, # mul + (-sys.maxint-1)//5, sys.maxint//5, # floordiv + (-sys.maxint-1)%5, sys.maxint%5, # mod + 0, -32, # lshift + (-sys.maxint-1)//32, sys.maxint//32, # rshift + # r_uint + 5, 4, # add + sys.maxint*2-3, sys.maxint*2-4, # sub + 0, sys.maxint*2-3, # mul + 0, (sys.maxint*2+1)//5, # floordiv + 0, (sys.maxint*2+1)%5, # mod + 0, sys.maxint*2-30, # lshift + 0, sys.maxint>>4, # rshift + # r_longlong + -maxlonglong+4, undefined, # add + undefined, maxlonglong-5, # sub + undefined, undefined, # mul + (-maxlonglong-1)//5, maxlonglong//5, # floordiv + (-maxlonglong-1)%5, maxlonglong%5, # mod + 0, -32, # lshift + (-maxlonglong-1)//32, maxlonglong//32,# rshift + # r_ulonglong + 5, 4, # add + maxlonglong*2-3, maxlonglong*2-4, # sub + 0, maxlonglong*2-3, # mul + 0, (maxlonglong*2+1)//5, # floordiv + 0, (maxlonglong*2+1)%5, # mod + 0, maxlonglong*2-30, # lshift + 0, maxlonglong>>4, # rshift + )) + + def test_r_singlefloat(self): + py.test.skip("singlefloat") + + z = r_singlefloat(0.4) + + def g(n): + if n > 0: + return r_singlefloat(n * 0.1) + else: + return z + + def llf(n): + return float(g(n)) + + fn = self.getcompiled(llf, [int]) + res = fn(21) + assert res != 2.1 # precision lost + assert abs(res - 2.1) < 1E-6 + res = fn(-5) + assert res != 0.4 # precision lost + assert abs(res - 0.4) < 1E-6 + + def test_prebuilt_nolength_char_array(self): + py.test.skip("fails on the trunk too") + for lastchar in ('\x00', 'X'): + A = Array(Char, hints={'nolength': True}) + a = malloc(A, 5, immortal=True) + a[0] = '8' + a[1] = '5' + a[2] = '?' + a[3] = '!' + a[4] = lastchar + def llf(): + s = '' + for i in range(5): + print i + print s + s += a[i] + print s + assert s == "85?!" + lastchar + return 0 + + fn = self.getcompiled(llf) + fn() + + # XXX what does this do? + def test_cast_primitive(self): + py.test.skip("ullong_lshift operation") + def f(x): + x = cast_primitive(UnsignedLongLong, x) + x <<= 60 + x /= 3 + x <<= 1 + x = cast_primitive(SignedLongLong, x) + x >>= 32 + return cast_primitive(Signed, x) + fn = self.getcompiled(f, [int]) + res = fn(14) + assert res == -1789569707 + From rxe at codespeak.net Sun Oct 28 00:28:16 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sun, 28 Oct 2007 00:28:16 +0200 (CEST) Subject: [pypy-svn] r48121 - pypy/dist/pypy/translator/llvm Message-ID: <20071027222816.4DC51817C@code0.codespeak.net> Author: rxe Date: Sun Oct 28 00:28:16 2007 New Revision: 48121 Modified: pypy/dist/pypy/translator/llvm/buildllvm.py Log: yet another refactor to build process for genllvm. it is likely we yet need still another Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Sun Oct 28 00:28:16 2007 @@ -6,9 +6,9 @@ from pypy.translator.llvm.log import log from pypy.translator.tool import stdoutcapture -import distutils.sysconfig +def write_ctypes_module(genllvm, dllname): + """ use ctypes to create a temporary module """ -def write_ctypes_module(genllvm, dllname, targetpath): template = """ import ctypes from os.path import join, dirname, realpath @@ -50,19 +50,24 @@ import ctypes from pypy.rpython.lltypesystem import lltype + basename = genllvm.filename.purebasename + '_wrapper.py' + modfilename = genllvm.filename.new(basename = basename) + TO_CTYPES = {lltype.Bool: "ctypes.c_int", lltype.Float: "ctypes.c_double", lltype.Char: "ctypes.c_char", lltype.Signed: "ctypes.c_int", lltype.Unsigned: "ctypes.c_uint" } + name = genllvm.entrynode.ref.strip("%") g = genllvm.entrynode.graph returntype = TO_CTYPES[g.returnblock.inputargs[0].concretetype] inputargtypes = [TO_CTYPES[a.concretetype] for a in g.startblock.inputargs] args = '[%s]' % ", ".join(inputargtypes) - targetpath.write(template % locals()) + modfilename.write(template % locals()) + return modfilename.purebasename def llvm_is_on_path(): if py.path.local.sysfind("llvm-as") is None or \ @@ -70,7 +75,7 @@ return False return True -def _exe_version(exe, cache={}): +def exe_version(exe, cache={}): try: v = cache[exe] except KeyError: @@ -80,15 +85,7 @@ cache[exe] = v return v -llvm_version = lambda: _exe_version('llvm-as') - -def postfix(): - if llvm_version() >= 2.0: - return '.i32' - else: - return '' - -def _exe_version2(exe): +def exe_version2(exe): v = os.popen(exe + ' --version 2>&1').read() i = v.index(')') v = v[i+2:].split()[0].split('.') @@ -96,28 +93,23 @@ v = float(major) + float(minor) / 10.0 return v -gcc_version = lambda: _exe_version2('gcc') -llvm_gcc_version = lambda: _exe_version2('llvm-gcc') - -# def compile_module(module, source_files, object_files, library_files): +llvm_version = lambda: exe_version('llvm-as') +gcc_version = lambda: exe_version2('gcc') +llvm_gcc_version = lambda: exe_version2('llvm-gcc') + +def have_boehm(): + import distutils.sysconfig + from os.path import exists + libdir = distutils.sysconfig.EXEC_PREFIX + "/lib" + return exists(libdir + '/libgc.so') or exists(libdir + '/libgc.a') -# open("%s_setup.py" % module, "w").write(str(py.code.Source( -# ''' -# from distutils.core import setup -# from distutils.extension import Extension -# setup(name="%(module)s", -# ext_modules = [Extension( -# name = "%(module)s", -# sources = %(source_files)s, -# libraries = %(library_files)s, -# extra_objects = %(object_files)s)]) -# ''' % locals()))) -# cmd ="python %s_setup.py build_ext --inplace --force" % module -# log.build(cmd) -# py.process.cmdexec(cmd) +def postfix(): + if llvm_version() >= 2.0: + return '.i32' + else: + return '' class Builder(object): - def __init__(self, genllvm): self.genllvm = genllvm self.cmds = [] @@ -127,28 +119,18 @@ cmd = "gccas /dev/null -o /dev/null -debug-pass=Arguments 2>&1" gccas_output = os.popen(cmd) opts = gccas_output.read()[17:-1] + " " - else: - opts = '-std-compile-opts' - # these were added by Chris Lattner for some old version of llvm - # opts += "-globalopt -constmerge -ipsccp -deadargelim -inline " \ - # "-instcombine -scalarrepl -globalsmodref-aa -licm -load-vn " \ - # "-gcse -instcombine -simplifycfg -globaldce " - - # added try to reduce the amount of excessive inlining by us, llvm and gcc - # opts += "-inline-threshold=175 " #default: 200 + # these were added by Chris Lattner for some old version of llvm + # opts += "-globalopt -constmerge -ipsccp -deadargelim -inline " \ + # "-instcombine -scalarrepl -globalsmodref-aa -licm -load-vn " \ + # "-gcse -instcombine -simplifycfg -globaldce " - return opts - - def compile_bytecode(self, b): - # run llvm assembler and optimizer - opts = self.optimizations() + # added try to reduce the amount of excessive inlining by us, llvm and gcc + # opts += "-inline-threshold=175 " #default: 200 - if llvm_version() < 2.0: - self.cmds.append("llvm-as < %s.ll | opt %s -f -o %s.bc" % (b, opts, b)) else: - # we generate 1.x .ll files, so upgrade these first - self.cmds.append("llvm-upgrade < %s.ll | llvm-as | opt %s -f -o %s.bc" % (b, opts, b)) + opts = '-std-compile-opts' + return opts def execute_cmds(self): c = stdoutcapture.Capture(mixed_out_err=True) @@ -158,7 +140,6 @@ for cmd in self.cmds: log.build(cmd) py.process.cmdexec(cmd) - finally: foutput, ferror = c.done() except: @@ -169,68 +150,82 @@ log.build(data) raise - def make_module(self): + def cmds_bytecode(self, base): + # run llvm assembler and optimizer + opts = self.optimizations() + + if llvm_version() < 2.0: + self.cmds.append("llvm-as < %s.ll | opt %s -f -o %s.bc" % (base, opts, base)) + else: + # we generate 1.x .ll files, so upgrade these first + self.cmds.append("llvm-upgrade < %s.ll | llvm-as | opt %s -f -o %s.bc" % (base, opts, base)) + + def cmds_objects(self, base): + # XXX why this hack??? + use_gcc = True #self.genllvm.config.translation.llvm_via_c + if use_gcc: + self.cmds.append("llc %s.bc -march=c -f -o %s.c" % (base, base)) + self.cmds.append("gcc %s.c -c -O3 -fomit-frame-pointer" % base) + else: + self.cmds.append("llc -relocation-model=pic %s.bc -f -o %s.s" % (base, base)) + self.cmds.append("as %s.s -o %s.o" % (base, base)) + +# if (self.genllvm.config.translation.profopt is not None and +# not self.genllvm.config.translation.noprofopt): +# cmd = "gcc -fprofile-generate %s.c -c -O3 -pipe -o %s.o" % (base, base) +# self.cmds.append(cmd) +# cmd = "gcc -fprofile-generate %s.o %s %s -lm -pipe -o %s_gen" % \ +# (base, gc_libs_path, gc_libs, exename) +# self.cmds.append(cmd) +# self.cmds.append("./%s_gen %s" % (exename, self.genllvm.config.translation.profopt)) +# cmd = "gcc -fprofile-use %s.c -c -O3 -pipe -o %s.o" % (b, b) +# self.cmds.append(cmd) +# cmd = "gcc -fprofile-use %s.o %s %s -lm -pipe -o %s" % \ +# (b, gc_libs_path, gc_libs, exename) +# else: + + def setup(self): + # set up directories llvmfile = self.genllvm.filename # change into dirpath and store current path to change back - dirpath = llvmfile.dirpath() - lastdir = py.path.local() - dirpath.chdir() - - b = llvmfile.purebasename + self.dirpath = llvmfile.dirpath() + self.lastdir = py.path.local() + self.dirpath.chdir() - # generate the llvm bytecode from ll file - self.compile_bytecode(b) + return llvmfile.purebasename + + def make_module(self): + base = self.setup() + self.cmds_bytecode(base) + self.cmds_objects(base) + # link (ok this is a mess!) library_files = self.genllvm.db.gcpolicy.gc_libraries() gc_libs = ' '.join(['-l' + lib for lib in library_files]) if sys.platform == 'darwin': libdir = '/sw/lib' gc_libs_path = '-L%s -ldl' % libdir + self.cmds.append("gcc -O3 %s.o %s %s -lm -bundle -o %s.so" % (base, gc_libs_path, gc_libs, base)) else: - gc_libs_path = '-static' - - dllname = "%s.so" % b - - use_gcc = False #self.genllvm.config.translation.llvm_via_c - if not use_gcc: - self.cmds.append("llc -relocation-model=pic %s.bc -f -o %s.s" % (b, b)) - self.cmds.append("as %s.s -o %s.o" % (b, b)) - - else: - self.cmds.append("llc %s.bc -march=c -f -o %s.c" % (b, b)) - self.cmds.append("gcc %s.c -c -O2" % b) - self.cmds.append("gcc -O3 %s.o %s %s -lm -bundle -o %s" % (b, gc_libs_path, gc_libs, dllname)) + gc_libs_path = '-static' + XXX try: self.execute_cmds() - - # use ctypes to create module for CPython - basename = self.genllvm.filename.purebasename + '_wrapper.py' - modfilename = self.genllvm.filename.new(basename = basename) - write_ctypes_module(self.genllvm, dllname, modfilename) - - modname = modfilename.purebasename + modname = write_ctypes_module(self.genllvm, "%s.so" % base) finally: - lastdir.chdir() + self.lastdir.chdir() - return modname, str(dirpath) + return modname, str(self.dirpath) def make_standalone(self, exename): - llvmfile = self.genllvm.filename - - # change into dirpath and store current path to change back - dirpath = llvmfile.dirpath() - lastdir = py.path.local() - dirpath.chdir() - - b = llvmfile.purebasename - - # generate the llvm bytecode from ll file - self.compile_bytecode(b) + base = self.setup() + self.cmds_bytecode(base) + self.cmds_objects(base) object_files = ["-L/sw/lib"] library_files = self.genllvm.db.gcpolicy.gc_libraries() @@ -242,41 +237,11 @@ else: gc_libs_path = '-static' - source_files = [] - - use_gcc = self.genllvm.config.translation.llvm_via_c - - if not use_gcc: - self.cmds.append("llc %s.bc -f -o %s.s" % (b, b)) - self.cmds.append("as %s.s -o %s.o" % (b, b)) - - cmd = "gcc -O3 %s.o %s %s -lm -pipe -o %s" % (b, gc_libs_path, gc_libs, exename) - self.cmds.append(cmd) - object_files.append("%s.o" % b) - else: - self.cmds.append("llc %s.bc -march=c -f -o %s.c" % (b, b)) - if (self.genllvm.config.translation.profopt is not None and - not self.genllvm.config.translation.noprofopt): - cmd = "gcc -fprofile-generate %s.c -c -O3 -pipe -o %s.o" % (b, b) - self.cmds.append(cmd) - cmd = "gcc -fprofile-generate %s.o %s %s -lm -pipe -o %s_gen" % \ - (b, gc_libs_path, gc_libs, exename) - self.cmds.append(cmd) - self.cmds.append("./%s_gen %s" % (exename, self.genllvm.config.translation.profopt)) - cmd = "gcc -fprofile-use %s.c -c -O3 -pipe -o %s.o" % (b, b) - self.cmds.append(cmd) - cmd = "gcc -fprofile-use %s.o %s %s -lm -pipe -o %s" % \ - (b, gc_libs_path, gc_libs, exename) - else: - cmd = "gcc %s.c -c -O3 -pipe -fomit-frame-pointer" % b - self.cmds.append(cmd) - cmd = "gcc %s.o %s %s -lm -pipe -o %s" % (b, gc_libs_path, gc_libs, exename) - self.cmds.append(cmd) - source_files.append("%s.c" % b) + self.cmds.append("gcc -O3 %s.o %s %s -lm -pipe -o %s" % (base, gc_libs_path, gc_libs, exename)) try: self.execute_cmds() finally: - lastdir.chdir() + self.lastdir.chdir() - return str(dirpath.join(exename)) + return str(self.dirpath.join(exename)) From rxe at codespeak.net Sun Oct 28 02:24:12 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sun, 28 Oct 2007 02:24:12 +0200 (CEST) Subject: [pypy-svn] r48122 - pypy/dist/pypy/translator/llvm Message-ID: <20071028002412.668798198@code0.codespeak.net> Author: rxe Date: Sun Oct 28 02:24:11 2007 New Revision: 48122 Modified: pypy/dist/pypy/translator/llvm/arraynode.py pypy/dist/pypy/translator/llvm/gc.py pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/structnode.py Log: cosmetic. some changes from Hildesheim sprint, not sure what they were about or were valid Modified: pypy/dist/pypy/translator/llvm/arraynode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/arraynode.py (original) +++ pypy/dist/pypy/translator/llvm/arraynode.py Sun Oct 28 02:24:11 2007 @@ -121,7 +121,7 @@ return ref def get_childref(self, index): - return "getelementptr(%s* %s, int 0, uint 1, int %s)" %( + return "getelementptr(%s* %s, int 0, uint 1, int %s)" % ( self.get_typerepr(), self.ref, index) Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Sun Oct 28 02:24:11 2007 @@ -7,12 +7,6 @@ from pypy.translator.llvm.buildllvm import postfix -def have_boehm(): - import distutils.sysconfig - from os.path import exists - libdir = distutils.sysconfig.EXEC_PREFIX + "/lib" - return exists(libdir + '/libgc.so') or exists(libdir + '/libgc.a') - class GcPolicy: n_malloced = 0 def __init__(self, db): @@ -49,10 +43,7 @@ raise Exception, 'GcPolicy should not be used directly' def new(db, gcpolicy=None): - # """ factory """ if gcpolicy == 'boehm': - # XXX would be nice to localise this sort of thing? - #assert have_boehm(), 'warning: Boehm GC libary not found in /usr/lib' gcpolicy = BoehmGcPolicy(db) elif gcpolicy == 'ref': gcpolicy = RefcountingGcPolicy(db) Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Sun Oct 28 02:24:11 2007 @@ -428,6 +428,7 @@ # getelementptr gets a pointer to the right type, except the generated code really expected # an array of size 1... so we just cast it + #assert isinstance(opr.op.result.concretetype.TO, lltype.FixedSizeArray) #XXX why? element_type = self.db.repr_type(op.result.concretetype.TO.OF) + '*' self.codewriter.cast(opr.retref, element_type, tmpvar, opr.rettype) @@ -443,6 +444,7 @@ # getelementptr gets a pointer to the right type, except the generated code really expected # an array of size 1... so we just cast it + #assert isinstance(opr.op.result.concretetype.TO, lltype.FixedSizeArray) #XXX why? element_type = self.db.repr_type(opr.op.result.concretetype.TO.OF) + '*' self.codewriter.cast(opr.retref, element_type, tmpvar, opr.rettype) @@ -458,6 +460,15 @@ element_type = self.db.repr_type(opr.op.result.concretetype.TO.OF) + '*' self.codewriter.cast(opr.retref, element_type, tmpvar, opr.rettype) +# + if isinstance(opr.op.result.concretetype, lltype.FixedSizeArray): +# + tmpvar = self._tmp() +# + self.codewriter.getelementptr(tmpvar, arraytype, array, [(self.word, incr)]) +# + element_type = self.db.repr_type(opr.op.result.concretetype.TO.OF) + '*' +# + self.codewriter.cast(opr.retref, element_type, tmpvar, opr.rettype) +# + else: +# + self.codewriter.getelementptr(opr.retref, arraytype, array, [(self.word, incr)], getptr=False) +# + + def adr_delta(self, opr): addr1, addr2 = self._tmp(2) self.codewriter.cast(addr1, opr.argtypes[0], opr.argrefs[0], self.word) @@ -583,3 +594,4 @@ # If it gets this far it is always false self.codewriter.cast(opr.retref, 'bool', 'false', opr.rettype) + Modified: pypy/dist/pypy/translator/llvm/structnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/structnode.py (original) +++ pypy/dist/pypy/translator/llvm/structnode.py Sun Oct 28 02:24:11 2007 @@ -192,6 +192,8 @@ def setup(self): if isinstance(self.value, lltype._subarray): + # XXX what is this? + # self.value._parentstructure() p, c = lltype.parentlink(self.value) if p is not None: self.db.prepare_constant(lltype.typeOf(p), p) From rxe at codespeak.net Sun Oct 28 02:46:49 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sun, 28 Oct 2007 02:46:49 +0200 (CEST) Subject: [pypy-svn] r48123 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071028004649.037F5813B@code0.codespeak.net> Author: rxe Date: Sun Oct 28 02:46:49 2007 New Revision: 48123 Modified: pypy/dist/pypy/translator/llvm/buildllvm.py pypy/dist/pypy/translator/llvm/test/test_lltype.py Log: support for getting longlong/ulonglong ints from tests Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Sun Oct 28 02:46:49 2007 @@ -57,7 +57,9 @@ lltype.Float: "ctypes.c_double", lltype.Char: "ctypes.c_char", lltype.Signed: "ctypes.c_int", - lltype.Unsigned: "ctypes.c_uint" + lltype.Unsigned: "ctypes.c_uint", + lltype.SignedLongLong: "ctypes.c_longlong", + lltype.UnsignedLongLong: "ctypes.c_ulonglong" } name = genllvm.entrynode.ref.strip("%") @@ -162,7 +164,7 @@ def cmds_objects(self, base): # XXX why this hack??? - use_gcc = True #self.genllvm.config.translation.llvm_via_c + use_gcc = False #self.genllvm.config.translation.llvm_via_c if use_gcc: self.cmds.append("llc %s.bc -march=c -f -o %s.c" % (base, base)) self.cmds.append("gcc %s.c -c -O3 -fomit-frame-pointer" % base) Modified: pypy/dist/pypy/translator/llvm/test/test_lltype.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lltype.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lltype.py Sun Oct 28 02:46:49 2007 @@ -729,6 +729,19 @@ fn = compile_function(llf, []) fn() +def test_longlongs(): + def llf(n): + return r_longlong(n) * r_longlong(2**32) + fn = compile_function(llf, [int]) + assert fn(0) == 0 + assert fn(42) == 42 * 2**32 + assert fn(-42) == -42 * 2**32 + def llf(n): + return r_ulonglong(n) * r_ulonglong(2**32) + fn = compile_function(llf, [int]) + assert fn(0) == 0 + assert fn(42) == 42 * 2**32 + class TestLowLevelType(object): def getcompiled(self, f, args=[]): return compile_function(f, args) From fijal at codespeak.net Sun Oct 28 11:07:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 28 Oct 2007 11:07:41 +0100 (CET) Subject: [pypy-svn] r48124 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071028100741.007FC8164@code0.codespeak.net> Author: fijal Date: Sun Oct 28 11:07:40 2007 New Revision: 48124 Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: cleanup tabs Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Sun Oct 28 11:07:40 2007 @@ -389,23 +389,23 @@ py.test.skip("implement me!") """ testBecome - | p1 p2 a | - p1 := 1 at 2. - p2 := #(3 4 5). - a := p1 -> p2. - self assert: 1 at 2 = a key. - self assert: #(3 4 5) = a value. - self assert: p1 -> p2 = a. - self assert: p1 == a key. - self assert: p2 == a value. - p1 become: p2. - self assert: 1 at 2 = a value. - self assert: #(3 4 5) = a key. - self assert: p1 -> p2 = a. - self assert: p1 == a key. - self assert: p2 == a value. - - self should: [1 become: 2] raise: Error. + | p1 p2 a | + p1 := 1 at 2. + p2 := #(3 4 5). + a := p1 -> p2. + self assert: 1 at 2 = a key. + self assert: #(3 4 5) = a value. + self assert: p1 -> p2 = a. + self assert: p1 == a key. + self assert: p2 == a value. + p1 become: p2. + self assert: 1 at 2 = a value. + self assert: #(3 4 5) = a key. + self assert: p1 -> p2 = a. + self assert: p1 == a key. + self assert: p2 == a value. + + self should: [1 become: 2] raise: Error. """ def test_load_inst_var(): From fijal at codespeak.net Sun Oct 28 11:11:22 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 28 Oct 2007 11:11:22 +0100 (CET) Subject: [pypy-svn] r48125 - pypy/dist/lib-python Message-ID: <20071028101122.6369F8160@code0.codespeak.net> Author: fijal Date: Sun Oct 28 11:11:22 2007 New Revision: 48125 Modified: pypy/dist/lib-python/conftest.py Log: * use modules even if run with -E * add test_zlib, test_bz2 and test_zipimport as non-core test suites Modified: pypy/dist/lib-python/conftest.py ============================================================================== --- pypy/dist/lib-python/conftest.py (original) +++ pypy/dist/lib-python/conftest.py Sun Oct 28 11:11:22 2007 @@ -198,8 +198,8 @@ # but we want a py.test refactoring towards # more autonomy of colitems regarding # their representations - regrtest = self.parent.regrtest - space = gettestobjspace() + regrtest = self.parent.regrtest + space = gettestobjspace(usemodules=[]) res, output = self.call_capture(space, regrtest.run_file, space) outputpath = regrtest.getoutputpath() @@ -229,7 +229,7 @@ if hasattr(self, 'name2item'): return self.name2item = {} - space = gettestobjspace() + space = gettestobjspace(usemodules=self.regrtest.usemodules) if self.regrtest.dumbtest or self.regrtest.getoutputpath(): self.name2item['output'] = SimpleRunItem('output', self) return @@ -408,7 +408,7 @@ RegrTest('test_bsddb3.py', enabled=False), RegrTest('test_bufio.py', enabled=True, dumbtest=1, core=True), RegrTest('test_builtin.py', enabled=True, core=True), - RegrTest('test_bz2.py', enabled=False), + RegrTest('test_bz2.py', usemodules='bz2', enabled=True), RegrTest('test_calendar.py', enabled=True), RegrTest('test_call.py', enabled=True, core=True), RegrTest('test_capi.py', enabled=False, dumbtest=1), @@ -759,7 +759,7 @@ RegrTest('test_xpickle.py', enabled=False), RegrTest('test_xrange.py', enabled=True, core=True), RegrTest('test_zipfile.py', enabled=False, dumbtest=1), - RegrTest('test_zipimport.py', enabled=True, usemodules='zlib'), + RegrTest('test_zipimport.py', enabled=True, usemodules='zlib zipimport'), RegrTest('test_zlib.py', enabled=True, usemodules='zlib'), ] From cfbolz at codespeak.net Sun Oct 28 18:05:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 28 Oct 2007 18:05:12 +0100 (CET) Subject: [pypy-svn] r48132 - pypy/dist/pypy/doc Message-ID: <20071028170512.4BD7F813B@code0.codespeak.net> Author: cfbolz Date: Sun Oct 28 18:05:11 2007 New Revision: 48132 Modified: pypy/dist/pypy/doc/news.txt Log: bern sprint is finished Modified: pypy/dist/pypy/doc/news.txt ============================================================================== --- pypy/dist/pypy/doc/news.txt (original) +++ pypy/dist/pypy/doc/news.txt Sun Oct 28 18:05:11 2007 @@ -15,6 +15,22 @@ .. _eventhistory: eventhistory.html +PyPy/Squeak Sprint in Bern finished +=================================== + +The Bern sprint, being the first Squeak-PyPy-collaboration-sprint is finished. +The week was very intense and productive, see `Bern Sprint Summary blog post`_ +for a list of things we accomplished. We covered most of what happened during +the sprint in quite some detail on the `PyPy Squeak blog`_. The sprint was +hosted by the Software Composition Group of the University of Bern from the +22nd to the 26th of October 2007. + +.. _`Bern sprint announcement`: http://codespeak.net/pypy/extradoc/sprintinfo/bern2007/announce.html +.. _`people that are known to come`: http://codespeak.net/pypy/extradoc/sprintinfo/bern2007/people.html +.. _`Bern Sprint Summary blog post`: http://pypysqueak.blogspot.com/2007/10/bern-sprint-finished-summary.html +.. _`PyPy Squeak blog`: http://pypysqueak.blogspot.net + + PyPy Sprint in Gothenburg: 19nd-25th November 2007 ================================================================== @@ -33,22 +49,6 @@ -PyPy/Squeak Sprint in Bern: 22nd-26th October 2007 -================================================================== - -We are proud to announce the first Squeak-PyPy-collaboration-sprint. The plan -is to bring people from the Squeak/Smalltalk and the PyPy/Python communities -together to learn about each others projects and to think about collaboration -possibilies. The sprint will be hosted by the Software Composition Group of -the University of Bern from the 22nd to the 26th of October 2007. For more -information see the `Bern sprint announcement`_ or a list of the `people that -are known to come`_. - -.. _`Bern sprint announcement`: http://codespeak.net/pypy/extradoc/sprintinfo/bern2007/announce.html -.. _`people that are known to come`: http://codespeak.net/pypy/extradoc/sprintinfo/bern2007/people.html - - - PyPy Sprint at EuroPython, Vilnius is finished ================================================================== From santagada at codespeak.net Sun Oct 28 18:38:46 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sun, 28 Oct 2007 18:38:46 +0100 (CET) Subject: [pypy-svn] r48133 - pypy/dist/pypy/doc Message-ID: <20071028173846.6834C8137@code0.codespeak.net> Author: santagada Date: Sun Oct 28 18:38:45 2007 New Revision: 48133 Modified: pypy/dist/pypy/doc/news.txt Log: typo Modified: pypy/dist/pypy/doc/news.txt ============================================================================== --- pypy/dist/pypy/doc/news.txt (original) +++ pypy/dist/pypy/doc/news.txt Sun Oct 28 18:38:45 2007 @@ -28,7 +28,7 @@ .. _`Bern sprint announcement`: http://codespeak.net/pypy/extradoc/sprintinfo/bern2007/announce.html .. _`people that are known to come`: http://codespeak.net/pypy/extradoc/sprintinfo/bern2007/people.html .. _`Bern Sprint Summary blog post`: http://pypysqueak.blogspot.com/2007/10/bern-sprint-finished-summary.html -.. _`PyPy Squeak blog`: http://pypysqueak.blogspot.net +.. _`PyPy Squeak blog`: http://pypysqueak.blogspot.com From cfbolz at codespeak.net Mon Oct 29 12:06:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 29 Oct 2007 12:06:23 +0100 (CET) Subject: [pypy-svn] r48144 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071029110623.3DF96810F@code0.codespeak.net> Author: cfbolz Date: Mon Oct 29 12:06:21 2007 New Revision: 48144 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/shadow.py Log: add some "assert 0" before nonsensical code. They don't break any tests (!!!), which is a bad sign, but help to get translation further. Also, rename getbyte on W_ContextPart to getbytecode to avoid confusion. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Mon Oct 29 12:06:21 2007 @@ -215,7 +215,7 @@ raise MissingBytecode("unknownBytecode") def extendedVariableTypeAndIndex(self): - descriptor = self.getbyte() + descriptor = self.getbytecode() return ((descriptor >> 6) & 3), (descriptor & 63) def extendedPushBytecode(self, interp): @@ -249,7 +249,7 @@ self.pop() def getExtendedSelectorArgcount(self): - descriptor = self.getbyte() + descriptor = self.getbytecode() return ((self.w_method().getliteralsymbol(descriptor & 31)), (descriptor >> 5)) @@ -258,8 +258,8 @@ self._sendSelfSelector(selector, argcount, interp) def doubleExtendedDoAnythingBytecode(self, interp): - second = self.getbyte() - third = self.getbyte() + second = self.getbytecode() + third = self.getbytecode() opType = second >> 5 if opType == 0: # selfsend @@ -294,7 +294,7 @@ self._sendSuperSelector(selector, argcount, interp) def secondExtendedSendBytecode(self, interp): - descriptor = self.getbyte() + descriptor = self.getbytecode() selector = self.w_method().getliteralsymbol(descriptor & 63) argcount = descriptor >> 6 self._sendSelfSelector(selector, argcount, interp) @@ -323,10 +323,10 @@ self.jumpConditional(interp.FALSE,self.shortJumpPosition()) def longUnconditionalJump(self, interp): - self.jump((((self.currentBytecode & 7) - 4) << 8) + self.getbyte()) + self.jump((((self.currentBytecode & 7) - 4) << 8) + self.getbytecode()) def longJumpPosition(self): - return ((self.currentBytecode & 3) << 8) + self.getbyte() + return ((self.currentBytecode & 3) << 8) + self.getbytecode() def longJumpIfTrue(self, interp): self.jumpConditional(interp.TRUE,self.longJumpPosition()) Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Mon Oct 29 12:06:21 2007 @@ -413,14 +413,14 @@ def w_method(self): return self.w_home._w_method - def getbyte(self): + def getbytecode(self): bytecode = self.w_method().bytes[self.pc] currentBytecode = ord(bytecode) self.pc = self.pc + 1 return currentBytecode def getNextBytecode(self): - self.currentBytecode = self.getbyte() + self.currentBytecode = self.getbytecode() return self.currentBytecode # ______________________________________________________________________ Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Mon Oct 29 12:06:21 2007 @@ -332,6 +332,10 @@ @expose_primitive(STRING_AT, unwrap_spec=[object, index1_0]) def func(interp, w_obj, n0): assert_valid_index(n0, w_obj) + # XXX I am not sure this is correct, but it un-breaks translation: + # make sure that getbyte is only performed on W_BytesObjects + if not isinstance(w_obj, model.W_BytesObject): + raise PrimitiveFailedError byte = w_obj.getbyte(n0) return objtable.CharacterTable[byte] @@ -449,6 +453,8 @@ header = unwrap_int(w_header) literalcount = ((header >> 10) & 255) + 1 w_method = w_class.as_class_get_shadow().new(literalcount) + # XXX not sure this is correct + assert isinstance(w_method, model.W_MethodContext) w_method.literals[constants.METHOD_HEADER_INDEX] = w_header for i in range(0,literalcount): w_method.literals[i+1] = objtable.w_nil Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Mon Oct 29 12:06:21 2007 @@ -120,12 +120,11 @@ if w_cls == classtable.w_BlockContext: return model.W_BlockContext(None, None, 0, 0) elif w_cls == classtable.w_MethodContext: + assert 0, "this seems nonsense" return model.W_MethodContext(None, None, [], extrasize) if self.instance_kind == POINTERS: return model.W_PointersObject(w_cls, self.instance_size+extrasize) - elif self.instance_kind == COMPILED_METHOD: - return model.W_CompiledMethod(extrasize+constants.LITERAL_START, w_compiledin = w_cls) elif self.instance_kind == WORDS: return model.W_WordsObject(w_cls, extrasize) elif self.instance_kind == BYTES: From cfbolz at codespeak.net Mon Oct 29 12:11:51 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 29 Oct 2007 12:11:51 +0100 (CET) Subject: [pypy-svn] r48145 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071029111151.60A46810C@code0.codespeak.net> Author: cfbolz Date: Mon Oct 29 12:11:51 2007 New Revision: 48145 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_model.py Log: python strings are _not_ mutable. Of course there never was a test, so nobody noticed this nonsense. Have I noticed that I am grumpy? Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Mon Oct 29 12:11:51 2007 @@ -368,8 +368,9 @@ else: self.setbyte(index0, chr(unwrap_int(w_value))) - def setbyte(self, index0, chr): - self.bytes[index0] = chr + def setbyte(self, index0, character): + self.bytes = (self.bytes[:index0] + character + + self.bytes[index0 + 1:]) class W_ContextPart(W_AbstractObjectWithIdentityHash): Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Mon Oct 29 12:11:51 2007 @@ -70,6 +70,11 @@ classshadow = w_class.as_class_get_shadow() assert classshadow.lookup("foo").w_compiledin is w_super +def test_w_compiledin(): + w_method = model.W_CompiledMethod(0, "abc") + w_method.setbyte(0, "c") + assert w_method.bytes == "cbc" + def test_hashes(): w_five = model.W_SmallInteger(5) assert w_five.gethash() == 5 From rxe at codespeak.net Mon Oct 29 12:13:12 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 29 Oct 2007 12:13:12 +0100 (CET) Subject: [pypy-svn] r48146 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071029111312.1EA928120@code0.codespeak.net> Author: rxe Date: Mon Oct 29 12:13:11 2007 New Revision: 48146 Modified: pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/test/test_lltype.py Log: test and fix for negative infinity floats, found with targetjsstandalone Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Mon Oct 29 12:13:11 2007 @@ -465,11 +465,9 @@ return str(ord(value)) def repr_float(self, type_, value): - repr = "%f" % value - # llvm requires a . when using e notation - if "e" in repr and "." not in repr: - repr = repr.replace("e", ".0e") - elif repr in ["inf", "nan"]: + from pypy.rlib.rarithmetic import isinf, isnan + + if isinf(value) or isnan(value): # Need hex repr import struct packed = struct.pack("d", value) @@ -477,6 +475,13 @@ packed = packed[::-1] repr = "0x" + "".join([("%02x" % ord(ii)) for ii in packed]) + else: + repr = "%f" % value + + # llvm requires a . when using e notation + if "e" in repr and "." not in repr: + repr = repr.replace("e", ".0e") + return repr def repr_address(self, type_, value): Modified: pypy/dist/pypy/translator/llvm/test/test_lltype.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lltype.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lltype.py Mon Oct 29 12:13:11 2007 @@ -216,8 +216,10 @@ f = compile_function(struct_opaque, []) assert f() == struct_opaque() -def test_floats(): #note: this is known to fail with llvm1.6 and llvm1.7cvs when not using gcc +def test_floats(): " test pbc of floats " + from pypy.rlib.rarithmetic import INFINITY, NAN + if sys.maxint != 2**31-1: py.test.skip("WIP on 64 bit architectures") F = GcStruct("f", @@ -226,19 +228,23 @@ ('f3', Float), ('f4', Float), ('f5', Float), + ('f6', Float), ) floats = malloc(F) floats.f1 = 1.25 floats.f2 = 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.252984 floats.f3 = float(29050000000000000000000000000000000000000000000000000000000000000000) - floats.f4 = 1e300 * 1e300 - nan = floats.f5 = floats.f4/floats.f4 + floats.f4 = INFINITY + nan = floats.f5 = NAN + floats.f6 = -INFINITY + def floats_fn(): res = floats.f1 == 1.25 res += floats.f2 > 1e100 res += floats.f3 > 1e50 res += floats.f4 > 1e200 - res += floats.f5 == nan + res += floats.f5 == NAN + res += floats.f4 < -1e200 return res f = compile_function(floats_fn, []) assert f() == floats_fn() From cfbolz at codespeak.net Mon Oct 29 12:20:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 29 Oct 2007 12:20:44 +0100 (CET) Subject: [pypy-svn] r48147 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071029112044.1C0DC8120@code0.codespeak.net> Author: cfbolz Date: Mon Oct 29 12:20:43 2007 New Revision: 48147 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/test/test_model.py Log: use consistent interface for the various setbyte versions. Why aren't those all returning characters? Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Mon Oct 29 12:20:43 2007 @@ -366,9 +366,10 @@ # of memory as smalltalk expects but wrapped in py-os raise NotImplementedError() else: - self.setbyte(index0, chr(unwrap_int(w_value))) + self.setbyte(index0, unwrap_int(w_value)) - def setbyte(self, index0, character): + def setbyte(self, index0, byte): + character = chr(byte) self.bytes = (self.bytes[:index0] + character + self.bytes[index0 + 1:]) Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Mon Oct 29 12:20:43 2007 @@ -72,7 +72,7 @@ def test_w_compiledin(): w_method = model.W_CompiledMethod(0, "abc") - w_method.setbyte(0, "c") + w_method.setbyte(0, ord("c")) assert w_method.bytes == "cbc" def test_hashes(): From rxe at codespeak.net Mon Oct 29 12:30:34 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 29 Oct 2007 12:30:34 +0100 (CET) Subject: [pypy-svn] r48149 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071029113034.3C2A6810F@code0.codespeak.net> Author: rxe Date: Mon Oct 29 12:30:33 2007 New Revision: 48149 Modified: pypy/dist/pypy/translator/llvm/test/test_extfunc.py Log: whatever it was, seems to have fixed itself Modified: pypy/dist/pypy/translator/llvm/test/test_extfunc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_extfunc.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_extfunc.py Mon Oct 29 12:30:33 2007 @@ -22,8 +22,6 @@ assert abs(f()-fn()) < 10.0 def test_math_frexp(): - py.test.skip("what the hell is going on!") - if sys.maxint != 2**31-1: py.test.skip("WIP on 64 bit architectures") @@ -36,8 +34,6 @@ assert res == fn(10.123) def test_math_modf(): - py.test.skip("what the hell is going on!") - from math import modf def fn(x): res = modf(x) From cfbolz at codespeak.net Mon Oct 29 12:37:58 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 29 Oct 2007 12:37:58 +0100 (CET) Subject: [pypy-svn] r48150 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071029113758.D7DCA8120@code0.codespeak.net> Author: cfbolz Date: Mon Oct 29 12:37:58 2007 New Revision: 48150 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: translation works again Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Mon Oct 29 12:37:58 2007 @@ -37,7 +37,9 @@ raise PrimitiveFailedError() def assert_valid_index(n0, w_obj): - assert_bounds(n0, 0, w_obj.size()) + if not 0 <= n0 < w_obj.size(): + raise PrimitiveFailedError() + return n0 # ___________________________________________________________________________ # Primitive table: it is filled in at initialization time with the @@ -311,12 +313,12 @@ @expose_primitive(AT, unwrap_spec=[object, index1_0]) def func(interp, w_obj, n0): - assert_valid_index(n0, w_obj) + n0 = assert_valid_index(n0, w_obj) return w_obj.at0(n0) @expose_primitive(AT_PUT, unwrap_spec=[object, index1_0, object]) def func(interp, w_obj, n0, w_val): - assert_valid_index(n0, w_obj) + n0 = assert_valid_index(n0, w_obj) try: w_obj.atput0(n0, w_val) return w_val @@ -331,7 +333,7 @@ @expose_primitive(STRING_AT, unwrap_spec=[object, index1_0]) def func(interp, w_obj, n0): - assert_valid_index(n0, w_obj) + n0 = assert_valid_index(n0, w_obj) # XXX I am not sure this is correct, but it un-breaks translation: # make sure that getbyte is only performed on W_BytesObjects if not isinstance(w_obj, model.W_BytesObject): @@ -341,7 +343,7 @@ @expose_primitive(STRING_AT_PUT, unwrap_spec=[object, index1_0, object]) def func(interp, w_obj, n0, w_val): - assert_valid_index(n0, w_obj) + n0 = assert_valid_index(n0, w_obj) if w_val.getclass() is not classtable.w_Character: raise PrimitiveFailedError() w_obj.setbyte(n0, objtable.ord_w_char(w_val)) @@ -454,11 +456,11 @@ literalcount = ((header >> 10) & 255) + 1 w_method = w_class.as_class_get_shadow().new(literalcount) # XXX not sure this is correct - assert isinstance(w_method, model.W_MethodContext) + assert isinstance(w_method, model.W_CompiledMethod) w_method.literals[constants.METHOD_HEADER_INDEX] = w_header for i in range(0,literalcount): w_method.literals[i+1] = objtable.w_nil - w_method.bytes = [None] * bytecount + w_method.bytes = "\x00" * bytecount return w_method # ___________________________________________________________________________ From rxe at codespeak.net Mon Oct 29 13:20:56 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 29 Oct 2007 13:20:56 +0100 (CET) Subject: [pypy-svn] r48151 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071029122056.2BAB2813D@code0.codespeak.net> Author: rxe Date: Mon Oct 29 13:20:54 2007 New Revision: 48151 Added: pypy/dist/pypy/translator/llvm/typedefnode.py Modified: pypy/dist/pypy/translator/llvm/arraynode.py pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/extfuncnode.py pypy/dist/pypy/translator/llvm/funcnode.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/node.py pypy/dist/pypy/translator/llvm/opaquenode.py pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/structnode.py pypy/dist/pypy/translator/llvm/test/test_symbolic.py Log: first cut at refactoring the llvm node/database mess Modified: pypy/dist/pypy/translator/llvm/arraynode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/arraynode.py (original) +++ pypy/dist/pypy/translator/llvm/arraynode.py Mon Oct 29 13:20:54 2007 @@ -1,77 +1,18 @@ from pypy.rpython.lltypesystem import lltype -from pypy.translator.llvm.log import log -from pypy.translator.llvm.node import LLVMNode, ConstantLLVMNode -log = log.structnode +from pypy.translator.llvm.node import ConstantNode -class ArrayTypeNode(LLVMNode): - __slots__ = "db array arraytype ref".split() - - def __init__(self, db, array): - assert isinstance(array, lltype.Array) - self.db = db - self.array = array - self.arraytype = arraytype = array.OF - name = "" - if isinstance(arraytype, lltype.Ptr): - name += "ptr_" - arraytype = arraytype.TO - if hasattr(arraytype, "_name"): - name += arraytype._name - else: - name += str(arraytype) - - self.ref = self.make_ref('%arraytype_', name) - - def __str__(self): - return "" % self.ref - - def setup(self): - self.db.prepare_type(self.arraytype) - - # ______________________________________________________________________ - # entry points from genllvm - # - def writedatatypedecl(self, codewriter): - if self.array._hints.get("nolength", False): - codewriter.arraynolendef(self.ref, - self.db.repr_type(self.arraytype)) - else: - codewriter.arraydef(self.ref, - self.db.get_machine_word(), - self.db.repr_type(self.arraytype)) - -class VoidArrayTypeNode(LLVMNode): - __slots__ = "db array ref".split() - - def __init__(self, db, array): - assert isinstance(array, lltype.Array) - self.db = db - self.array = array - self.ref = "%arraytype_Void" - - def writedatatypedecl(self, codewriter): - assert not self.array._hints.get("nolength", False) - codewriter.typedef(self.ref, "{ %s }" % self.db.get_machine_word()) - -class ArrayNode(ConstantLLVMNode): - """ An arraynode. Elements can be - a primitive, - a struct, - pointer to struct/array - """ - __slots__ = "db value arraytype ref".split() +class ArrayNode(ConstantNode): + __slots__ = "db value arraytype".split() + prefix = '%arrayinstance' def __init__(self, db, value): assert isinstance(lltype.typeOf(value), lltype.Array) self.db = db self.value = value self.arraytype = lltype.typeOf(value).OF - prefix = '%arrayinstance' - name = '' #str(value).split()[1] - self.ref = self.make_ref(prefix, name) - def __str__(self): - return "" % (self.ref,) + name = '' #str(value).split()[1] + self.make_name(name) def setup(self): for item in self.value.items: @@ -102,28 +43,29 @@ typeval = self.db.repr_type(lltype.typeOf(self.value)) p, c = lltype.parentlink(self.value) if p is None: - ref = self.ref + ref = self.name else: ref = self.db.get_childref(p, c) ref = "cast(%s* %s to %s*)" % (self.get_typerepr(), ref, typeval) + print 'KKKKKKKKKK', ref return ref def get_pbcref(self, toptr): - ref = self.ref p, c = lltype.parentlink(self.value) assert p is None, "child PBC arrays are NOT needed by rtyper" fromptr = "%s*" % self.get_typerepr() - ref = "cast(%s %s to %s)" % (fromptr, ref, toptr) + ref = "cast(%s %s to %s)" % (fromptr, self.name, toptr) + print 'XXXXXXXXX', ref return ref def get_childref(self, index): return "getelementptr(%s* %s, int 0, uint 1, int %s)" % ( self.get_typerepr(), - self.ref, + self.name, index) def constantvalue(self): @@ -149,7 +91,7 @@ def get_childref(self, index): return "getelementptr(%s* %s, int 0, int %s)" %( self.get_typerepr(), - self.ref, + self.name, index) def constantvalue(self): @@ -187,16 +129,16 @@ r = 'c"%s"' % "".join(s) return item_length, r -class VoidArrayNode(ConstantLLVMNode): - __slots__ = "db value ref".split() +class VoidArrayNode(ConstantNode): + __slots__ = "db value".split() + prefix = '%voidarrayinstance' def __init__(self, db, value): assert isinstance(lltype.typeOf(value), lltype.Array) self.db = db self.value = value - prefix = '%arrayinstance' name = '' #str(value).split()[1] - self.ref = self.make_ref(prefix, name) + self.make_name(name) def constantvalue(self): return "{ %s } {%s %s}" % (self.db.get_machine_word(), Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Mon Oct 29 13:20:54 2007 @@ -72,22 +72,6 @@ def typedef(self, name, type_): self._append("%s = type %s" % (name, type_)) - def structdef(self, name, typereprs): - self.typedef(name, "{ %s }" % ", ".join(typereprs)) - - def arraydef(self, name, lentype, typerepr): - self.typedef(name, "{ %s, [0 x %s] }" % (lentype, typerepr)) - - def arraynolendef(self, name, typerepr): - self.typedef(name, "[0 x %s]" % (typerepr)) - - def fixedarraydef(self, name, arraylen, typerepr): - self.typedef(name, "[%s x %s]" % (arraylen, typerepr)) - - def funcdef(self, name, rettyperepr, argtypereprs): - self.typedef(name, "%s (%s)" % (rettyperepr, - ", ".join(argtypereprs))) - def declare(self, decl, cconv=None): if cconv is None: cconv = self.cconv Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Mon Oct 29 13:20:54 2007 @@ -2,15 +2,17 @@ import sys from pypy.translator.llvm.log import log -from pypy.translator.llvm.funcnode import FuncNode, FuncTypeNode + +from pypy.translator.llvm.typedefnode import create_typedef_node + +from pypy.translator.llvm.funcnode import FuncImplNode from pypy.translator.llvm.extfuncnode import ExternalFuncNode, SimplerExternalFuncNode +from pypy.translator.llvm.opaquenode import OpaqueNode, ExtOpaqueNode from pypy.translator.llvm.structnode import StructNode, StructVarsizeNode, \ - StructTypeNode, StructVarsizeTypeNode, getindexhelper, \ - FixedSizeArrayTypeNode, FixedSizeArrayNode + getindexhelper, FixedSizeArrayNode from pypy.translator.llvm.arraynode import ArrayNode, StrArrayNode, \ - VoidArrayNode, ArrayNoLengthNode, ArrayTypeNode, VoidArrayTypeNode -from pypy.translator.llvm.opaquenode import OpaqueNode, ExtOpaqueNode, \ - OpaqueTypeNode, ExtOpaqueTypeNode + VoidArrayNode, ArrayNoLengthNode + from pypy.rpython.lltypesystem import lltype, llmemory from pypy.objspace.flow.model import Constant, Variable from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic @@ -30,7 +32,12 @@ self.helper2ptr = {} self.primitives = Primitives(self) - + + # keep ordered list for when we write + self.funcnodes = [] + self.typedefnodes = [] + self.containernodes = [] + #_______debuggging______________________________________ def dump_pbcs(self): @@ -52,7 +59,7 @@ "pbcref -> %s \n" % (v, k, ref, pbc_ref) return r - #_______setting up and preperation______________________________ + #_______setting up and preparation______________________________ def create_constant_node(self, type_, value): node = None @@ -65,7 +72,7 @@ elif getattr(value, 'external', None) == 'C': node = SimplerExternalFuncNode(self, value) else: - node = FuncNode(self, value) + node = FuncImplNode(self, value) elif isinstance(type_, lltype.FixedSizeArray): node = FixedSizeArrayNode(self, value) @@ -114,37 +121,14 @@ return if isinstance(type_, lltype.Primitive): - pass + return - elif isinstance(type_, lltype.Ptr): + if isinstance(type_, lltype.Ptr): self.prepare_type(type_.TO) - - elif isinstance(type_, lltype.FixedSizeArray): - self.addpending(type_, FixedSizeArrayTypeNode(self, type_)) - - elif isinstance(type_, lltype.Struct): - if type_._arrayfld: - self.addpending(type_, StructVarsizeTypeNode(self, type_)) - else: - self.addpending(type_, StructTypeNode(self, type_)) - - elif isinstance(type_, lltype.FuncType): - self.addpending(type_, FuncTypeNode(self, type_)) - - elif isinstance(type_, lltype.Array): - if type_.OF is lltype.Void: - self.addpending(type_, VoidArrayTypeNode(self, type_)) - else: - self.addpending(type_, ArrayTypeNode(self, type_)) - - elif isinstance(type_, lltype.OpaqueType): - if hasattr(type_, '_exttypeinfo'): - self.addpending(type_, ExtOpaqueTypeNode(self, type_)) - else: - self.addpending(type_, OpaqueTypeNode(self, type_)) - else: - assert False, "need to prepare typerepr %s %s" % (type_, type(type_)) + node = create_typedef_node(self, type_) + self.addpending(type_, node) + self.typedefnodes.append(node) def prepare_type_multi(self, types): for type_ in types: @@ -248,6 +232,9 @@ def getnodes(self): return self.obj2node.itervalues() + + def gettypedefnodes(self): + return self.typedefnodes # __________________________________________________________ # Representing variables and constants in LLVM source code @@ -261,7 +248,7 @@ if isinstance(arg.value._obj, int): rt = self.repr_type(arg.concretetype) v = repr(arg.value._obj) - return 'cast (int %s to %s)'%(v, rt) + return 'cast (int %s to %s)' % (v, rt) elif not arg.value: return 'null' else: @@ -312,7 +299,7 @@ return None, "%s null" % toptr if isinstance(value, int): - return None, '%s cast (int %s to %s)'%(toptr, value, toptr) + return None, '%s cast (int %s to %s)' % (toptr, value, toptr) node = self.obj2node[value] ref = node.get_pbcref(toptr) @@ -334,14 +321,6 @@ self._tmpcount += 1 return "%tmp_" + str(count) - def repr_name(self, obj): - " simply returns a reference to constant value " - return self.obj2node[obj].ref - - def repr_value(self, value): - # XXX Testing - return self.obj2node[value].get_ref() - # __________________________________________________________ # Other helpers @@ -521,6 +500,8 @@ repr = '1' elif value is jit._we_are_jitted: repr = '0' + elif value is objectmodel.running_on_llinterp: + repr = '0' else: raise NotImplementedError("CDefinedIntSymbolic: %r" % (value,)) else: Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Mon Oct 29 13:20:54 2007 @@ -195,6 +195,7 @@ c_name = c_name[1:] ccode.append("void raise%s(char *);\n" % c_name) else: + # XXX we really shouldnt do this predeclarefn(c_name, db.obj2node[obj._obj].ref) elif type(c_name) is str and type(obj) is int: ccode.append("#define\t%s\t%d\n" % (c_name, obj)) Modified: pypy/dist/pypy/translator/llvm/extfuncnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/extfuncnode.py (original) +++ pypy/dist/pypy/translator/llvm/extfuncnode.py Mon Oct 29 13:20:54 2007 @@ -1,4 +1,4 @@ -from pypy.translator.llvm.node import ConstantLLVMNode +from pypy.translator.llvm.node import FuncNode from pypy.translator.llvm.log import log from pypy.translator.c.extfunc import EXTERNALS from pypy.rpython.lltypesystem import lltype @@ -14,24 +14,19 @@ # signature of external functions differ from C's implementation ext_func_sigs = { - "%LL_os_isatty" : ExtFuncSig("int", None), "%LL_stack_too_big" : ExtFuncSig("int", None), - "%LL_os_lseek" : ExtFuncSig("int", None), - "%LL_thread_acquirelock" : ExtFuncSig("int", [None, "int"]), - "%LL_thread_start" : ExtFuncSig(None, ["sbyte*", "sbyte*"]), } if maxint != 2**31-1: - ext_func_sigs["%LL_os_write"] = ExtFuncSig(None, ["int", None]) ext_func_sigs["%LL_math_ldexp"] = ExtFuncSig(None, [None, "int"]) -class SimplerExternalFuncNode(ConstantLLVMNode): +class SimplerExternalFuncNode(FuncNode): def __init__(self, db, value): self.db = db self.value = value - self.ref = "%" + value._name + self.name = "%" + value._name def external_c_source(self): # return a list of unique includes and sources in C @@ -62,7 +57,7 @@ def writedecl(self, codewriter): codewriter.declare(self.getdecl()) -class ExternalFuncNode(ConstantLLVMNode): +class ExternalFuncNode(FuncNode): def __init__(self, db, value, extname=None): self.db = db @@ -76,15 +71,12 @@ else: mapped_name = EXTERNALS[self.callable] - self.ref = self.make_ref("%", mapped_name) + self.make_name(mapped_name) def setup(self): self.db.prepare_type(self.value._TYPE.RESULT) self.db.prepare_type_multi(self.value._TYPE._trueargs()) - def __str__(self): - return "" % self.ref - def _get_wrapper(self): wrapper = ext_func_sigs.get(self.ref, None) Modified: pypy/dist/pypy/translator/llvm/funcnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/funcnode.py (original) +++ pypy/dist/pypy/translator/llvm/funcnode.py Mon Oct 29 13:20:54 2007 @@ -1,49 +1,26 @@ from pypy.objspace.flow.model import Block, Constant, Link from pypy.objspace.flow.model import mkentrymap, c_last_exception from pypy.rpython.lltypesystem import lltype -from pypy.translator.llvm.node import LLVMNode, ConstantLLVMNode +from pypy.translator.llvm.node import FuncNode from pypy.translator.llvm.opwriter import OpWriter from pypy.translator.llvm.log import log from pypy.translator.unsimplify import remove_double_links, no_links_to_startblock log = log.funcnode -class FuncTypeNode(LLVMNode): - __slots__ = "db type_ ref".split() - - def __init__(self, db, type_): - self.db = db - assert isinstance(type_, lltype.FuncType) - self.type_ = type_ - self.ref = self.make_ref('%functiontype', '') - - def __str__(self): - return "" % self.ref - - def setup(self): - self.db.prepare_type(self.type_.RESULT) - self.db.prepare_type_multi(self.type_._trueargs()) - - def writedatatypedecl(self, codewriter): - returntype = self.db.repr_type(self.type_.RESULT) - inputargtypes = [self.db.repr_type(a) for a in self.type_._trueargs()] - codewriter.funcdef(self.ref, returntype, inputargtypes) - class BranchException(Exception): pass - -class FuncNode(ConstantLLVMNode): - __slots__ = "db value ref graph block_to_name bad_switch_block".split() +class FuncImplNode(FuncNode): + prefix = '%pypy_' + __slots__ = "db value graph block_to_name bad_switch_block".split() def __init__(self, db, value): self.db = db self.value = value - self.ref = self.make_ref('%pypy_', value.graph.name) self.graph = value.graph self.bad_switch_block = False - def __str__(self): - return "" %(self.ref,) + self.make_name(value.graph.name) def setup(self): assert self.graph, "cannot traverse" @@ -92,9 +69,6 @@ codewriter._indent('call void %abort()') codewriter._indent('unreachable') codewriter.closefunc() - - def writeglobalconstants(self, codewriter): - pass # ______________________________________________________________________ # writing helpers for entry points Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Mon Oct 29 13:20:54 2007 @@ -12,7 +12,7 @@ from pypy.translator.llvm import extfuncnode from pypy.translator.llvm.module.support import \ extdeclarations, extfunctions, extfunctions_standalone, write_raise_exc -from pypy.translator.llvm.node import LLVMNode +from pypy.translator.llvm.node import Node from pypy.translator.llvm.externs2ll import setup_externs, generate_llfile from pypy.translator.llvm.gc import GcPolicy from pypy.translator.llvm.log import log @@ -25,7 +25,7 @@ def __init__(self, translator, standalone): # reset counters - LLVMNode.nodename_count = {} + Node.nodename_count = {} self.standalone = standalone self.translator = translator @@ -115,15 +115,17 @@ self._checkpoint('write externs type declarations') # write node type declarations - for typ_decl in self.db.getnodes(): - typ_decl.writedatatypedecl(codewriter) + for typ_decl in self.db.gettypedefnodes(): + typ_decl.writetypedef(codewriter) self._checkpoint('write data type declarations') codewriter.header_comment("Global Data") # write pbcs - for typ_decl in self.db.getnodes(): - typ_decl.writeglobalconstants(codewriter) + for node in self.db.getnodes(): + # XXX tmp + if hasattr(node, "writeglobalconstants"): + node.writeglobalconstants(codewriter) self._checkpoint('write global constants') codewriter.header_comment("Function Prototypes") @@ -132,8 +134,9 @@ codewriter.write_lines(self._set_wordsize(extdeclarations)) # write node protos - for typ_decl in self.db.getnodes(): - typ_decl.writedecl(codewriter) + for node in self.db.getnodes(): + if hasattr(node, 'writedecl'): + node.writedecl(codewriter) self._checkpoint('write function prototypes') @@ -156,8 +159,10 @@ codewriter.write_lines(llvm_implcode(self.entrynode)) # write all node implementations - for typ_decl in self.db.getnodes(): - typ_decl.writeimpl(codewriter) + for node in self.db.getnodes(): + if hasattr(node, 'writeimpl'): + node.writeimpl(codewriter) + self._checkpoint('write node implementations') # write entry point if there is one Modified: pypy/dist/pypy/translator/llvm/node.py ============================================================================== --- pypy/dist/pypy/translator/llvm/node.py (original) +++ pypy/dist/pypy/translator/llvm/node.py Mon Oct 29 13:20:54 2007 @@ -1,71 +1,71 @@ from pypy.rpython.lltypesystem import lltype -class LLVMNode(object): - __slots__ = "".split() +class Node(object): + __slots__ = "name".split() + prefix = '%' nodename_count = {} - def make_name(self, name): + def make_name(self, name=''): " helper for creating names" + name = self.prefix + name if name in self.nodename_count: postfix = '_%d' % self.nodename_count[name] self.nodename_count[name] += 1 else: postfix = '' self.nodename_count[name] = 1 - name += postfix + if " " in name or "<" in name: name = '"%s"' % name - return name - def make_ref(self, prefix, name): - return self.make_name(prefix + name) + self.name = name def setup(self): pass - # ______________________________________________________________________ + def post_setup_transform(self): + pass def external_c_source(self): - # return a list of unique includes and sources in C + " return a list of unique includes and sources in C " return [], [] - - # __________________ before "implementation" ____________________ - - - def post_setup_transform(self): - pass - def writedatatypedecl(self, codewriter): - """ write out declare names of data types - (structs/arrays/function pointers) - """ + def writesetupcode(self, codewriter): + " pre entry-point setup " + pass - def writeglobalconstants(self, codewriter): - """ write out global values. """ + @property + def ref(self): + return self.name - def writedecl(self, codewriter): - """ write function forward declarations. """ + def __str__(self): + return "<%s %r>" % (self.__class__.__name__, self.ref) - def writecomments(self, codewriter): - """ write operations strings for debugging purposes. """ +class FuncNode(Node): - # __________________ after "implementation" ____________________ - def writeimpl(self, codewriter): - """ write function implementations. """ + # XXX proof that the whole llvm is hanging on a bunch of loose stitches + def get_ref(self): + return self.ref - # ______________________________________________________________________ - # pre entry-point setup + # XXX proof that the whole llvm is hanging on a bunch of loose stitches + def get_pbcref(self, _): + return self.ref - def writesetupcode(self, codewriter): + def writedecl(self, codewriter): + " write function forward declarations " pass -class ConstantLLVMNode(LLVMNode): + def writeimpl(self, codewriter): + """ write function implementations """ + pass + +class ConstantNode(Node): __slots__ = "".split() def get_ref(self): - """ Returns a reference as used for operations in blocks for pbc. """ + # XXX tmp return self.ref def get_childref(self, index): @@ -76,13 +76,13 @@ """ Returns a reference as a pointer used per pbc. """ return self.ref - def constantvalue(self): - """ Returns the constant representation for this node. """ - raise AttributeError("Must be implemented in subclass") - # ______________________________________________________________________ # entry points from genllvm + def constantvalue(self): + """ Returns the constant representation for this node. """ + pass + def writeglobalconstants(self, codewriter): p, c = lltype.parentlink(self.value) if p is None: Modified: pypy/dist/pypy/translator/llvm/opaquenode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opaquenode.py (original) +++ pypy/dist/pypy/translator/llvm/opaquenode.py Mon Oct 29 13:20:54 2007 @@ -1,48 +1,26 @@ -from pypy.translator.llvm.node import LLVMNode, ConstantLLVMNode +from pypy.translator.llvm.node import ConstantNode from pypy.rpython.lltypesystem import lltype -class OpaqueTypeNode(LLVMNode): - - def __init__(self, db, opaquetype): - assert isinstance(opaquetype, lltype.OpaqueType) - self.db = db - self.opaquetype = opaquetype - self.ref = "%%RPyOpaque_%s" % (opaquetype.tag) - - def __str__(self): - return "" %(self.ref,) - - # ______________________________________________________________________ - # main entry points from genllvm - - def writedatatypedecl(self, codewriter): - codewriter.typedef(self.ref, "opaque*") - -class ExtOpaqueTypeNode(OpaqueTypeNode): - def writedatatypedecl(self, codewriter): - pass - -class OpaqueNode(ConstantLLVMNode): +class OpaqueNode(ConstantNode): def __init__(self, db, value): self.db = db self.value = value - self.ref = "null" - # ______________________________________________________________________ - # main entry points from genllvm - + self.name = "null" + def writeglobalconstants(self, codewriter): # XXX Dummy - not sure what what we want pass -class ExtOpaqueNode(ConstantLLVMNode): +class ExtOpaqueNode(ConstantNode): + prefix = '%opaqueinstance_' def __init__(self, db, value): self.db = db self.value = value - prefix = '%opaqueinstance_' - name = str(value).split()[1] - self.ref = self.make_ref(prefix, name) self._get_ref_cache = None + name = str(value).split()[1] + self.make_name(name) + # ______________________________________________________________________ # main entry points from genllvm @@ -52,7 +30,7 @@ return self._get_ref_cache p, c = lltype.parentlink(self.value) if p is None: - ref = self.ref + ref = self.name else: ref = self.db.get_childref(p, c) self._get_ref_cache = ref Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Mon Oct 29 13:20:54 2007 @@ -595,3 +595,8 @@ self.codewriter.cast(opr.retref, 'bool', 'false', opr.rettype) + def debug_llinterpcall(self, opr): + self.codewriter.call(None, "void", "%abort", [], []) + # cheat llvm + self.codewriter.cast(opr.retref, opr.rettype, 'null', opr.rettype) + Modified: pypy/dist/pypy/translator/llvm/structnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/structnode.py (original) +++ pypy/dist/pypy/translator/llvm/structnode.py Mon Oct 29 13:20:54 2007 @@ -1,5 +1,4 @@ -from pypy.translator.llvm.log import log -from pypy.translator.llvm.node import LLVMNode, ConstantLLVMNode +from pypy.translator.llvm.node import ConstantNode from pypy.rpython.lltypesystem import lltype def getindexhelper(name, struct): @@ -11,71 +10,14 @@ except ValueError: index = -1 return index - -log = log.structnode - -class StructTypeNode(LLVMNode): - __slots__ = "db struct ref name".split() - prefix = '%structtype_' - - def __init__(self, db, struct): - assert isinstance(struct, lltype.Struct) - self.db = db - self.struct = struct - name = self.struct._name - self.ref = self.make_ref(self.prefix, name) - self.name = self.ref[len(self.prefix):] - def __str__(self): - return "" %(self.ref,) - - def _fields(self): - return [getattr(self.struct, name) - for name in self.struct._names_without_voids()] - - def setup(self): - # Recurse - for field in self._fields(): - self.db.prepare_type(field) - - # ______________________________________________________________________ - # main entry points from genllvm - - def writedatatypedecl(self, codewriter): - fields_types = [self.db.repr_type(f) for f in self._fields()] - codewriter.structdef(self.ref, fields_types) - -class FixedSizeArrayTypeNode(StructTypeNode): - prefix = '%fixarray_' - - def __str__(self): - return "" % self.ref - - def setup(self): - fields = self._fields() - if fields: - self.db.prepare_type(fields[0]) - - def writedatatypedecl(self, codewriter): - codewriter.fixedarraydef(self.ref, - self.struct.length, - self.db.repr_type(self.struct.OF)) - - -class StructVarsizeTypeNode(StructTypeNode): - def __init__(self, db, struct): - super(StructVarsizeTypeNode, self).__init__(db, struct) - - def __str__(self): - return "" %(self.ref,) - -class StructNode(ConstantLLVMNode): +class StructNode(ConstantNode): """ A struct constant. Can simply contain a primitive, a struct, pointer to struct/array """ - __slots__ = "db value structtype ref _get_ref_cache _get_types".split() + __slots__ = "db value structtype _get_ref_cache _get_types".split() prefix = '%structinstance_' @@ -84,12 +26,9 @@ self.value = value self.structtype = self.value._TYPE name = str(value).split()[1] - self.ref = self.make_ref(self.prefix, name) self._get_ref_cache = None self._get_types = self._compute_types() - - def __str__(self): - return "" % (self.ref,) + self.make_name(name) def _compute_types(self): return [(name, self.structtype._flds[name]) @@ -137,7 +76,7 @@ # return self._get_ref_cache p, c = lltype.parentlink(self.value) if p is None: - ref = self.ref + ref = self.name else: ref = self.db.get_childref(p, c) #XXXself._get_ref_cache = ref @@ -162,9 +101,6 @@ self.array = struct self.arraytype = self.structtype.OF - def __str__(self): - return "" % (self.ref,) - def constantvalue(self): """ Returns the constant representation for this node. """ values = self._getvalues() @@ -174,7 +110,7 @@ def get_ref(self): p, c = lltype.parentlink(self.value) if p is None: - ref = self.ref + ref = self.name else: ref = self.db.get_childref(p, c) if isinstance(self.value, lltype._subarray): @@ -212,9 +148,6 @@ a series of embedded structs, which has as its last element an array. """ - def __str__(self): - return "" % (self.ref,) - def _getvalues(self): values = [] for name, T in self._get_types[:-1]: @@ -278,7 +211,7 @@ def get_pbcref(self, toptr): """ Returns a reference as used per pbc. """ - ref = self.ref + ref = self.name p, c = lltype.parentlink(self.value) assert p is None, "child varsize struct are NOT needed by rtyper" fromptr = "%s*" % self.get_typerepr() Modified: pypy/dist/pypy/translator/llvm/test/test_symbolic.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_symbolic.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_symbolic.py Mon Oct 29 13:20:54 2007 @@ -74,6 +74,7 @@ assert res == 1234501234 def test_sizeof_constsize_struct(): + py.test.skip("failing in genc too") # _not_ a GcStruct, since we want to raw_malloc it STRUCT = lltype.Struct("s", ("x", lltype.Signed), ("y", lltype.Signed)) STRUCTPTR = lltype.Ptr(STRUCT) Added: pypy/dist/pypy/translator/llvm/typedefnode.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/typedefnode.py Mon Oct 29 13:20:54 2007 @@ -0,0 +1,155 @@ +from pypy.rpython.lltypesystem import lltype +from pypy.translator.llvm.node import Node + +class TypeDefNode(Node): + __slots__ = "".split() + + def writetypedef(self, codewriter): + " write out the type definition " + +class ArrayTypeNode(TypeDefNode): + __slots__ = "db ARRAY ARRAYTYPE".split() + prefix = '%arraytype_' + + def __init__(self, db, ARRAY): + assert isinstance(ARRAY, lltype.Array) + self.db = db + self.ARRAY = ARRAY + + name = "" + T = ARRAY.OF + if isinstance(T, lltype.Ptr): + name += "ptr_" + T = T.TO + if hasattr(T, "_name"): + name += T._name + else: + name += str(T) + + self.make_name(name) + + def setup(self): + self.db.prepare_type(self.ARRAY.OF) + + def writetypedef(self, codewriter): + if self.ARRAY._hints.get("nolength", False): + codewriter.typedef(self.ref, + "[0 x %s]" % self.db.repr_type(self.ARRAY.OF)) + else: + codewriter.typedef(self.ref, + "{ %s, [0 x %s] }" % (self.db.get_machine_word(), + self.db.repr_type(self.ARRAY.OF))) + +class VoidArrayTypeNode(TypeDefNode): + " void arrays dont have any real elements " + __slots__ = "db ARRAY".split() + prefix = '%voidarraytype_' + + def __init__(self, db, ARRAY): + assert isinstance(ARRAY, lltype.Array) + assert not ARRAY._hints.get("nolength", False) + + self.db = db + self.ARRAY = ARRAY + self.make_name() + + def writetypedef(self, codewriter): + codewriter.typedef(self.ref, "{ %s }" % self.db.get_machine_word()) + +class StructTypeNode(TypeDefNode): + __slots__ = "db STRUCT".split() + prefix = '%structtype_' + + def __init__(self, db, STRUCT): + assert isinstance(STRUCT, lltype.Struct) + self.db = db + self.STRUCT = STRUCT + self.make_name(self.STRUCT._name) + + def _fields(self): + return [getattr(self.STRUCT, name) + for name in self.STRUCT._names_without_voids()] + + def setup(self): + for F in self._fields(): + self.db.prepare_type(F) + + def writetypedef(self, codewriter): + fields_types = [self.db.repr_type(F) for F in self._fields()] + codewriter.typedef(self.ref, + "{ %s }" % ", ".join(fields_types)) + +class FixedSizeArrayTypeNode(StructTypeNode): + prefix = '%fixarray_' + + def setup(self): + FIELDS = self._fields() + if FIELDS: + self.db.prepare_type(FIELDS[0]) + + def writetypedef(self, codewriter): + codewriter.typedef(self.ref, + "[%s x %s]" % (self.STRUCT.length, + self.db.repr_type(self.STRUCT.OF))) + +class FuncTypeNode(TypeDefNode): + __slots__ = "db T".split() + prefix = '%functiontype' + + def __init__(self, db, T): + assert isinstance(T, lltype.FuncType) + + self.db = db + self.T = T + self.make_name() + + def setup(self): + self.db.prepare_type(self.T.RESULT) + self.db.prepare_type_multi(self.T._trueargs()) + + def writetypedef(self, codewriter): + returntype = self.db.repr_type(self.T.RESULT) + inputargtypes = [self.db.repr_type(a) for a in self.T._trueargs()] + codewriter.typedef(self.ref, + "%s (%s)" % (returntype, + ", ".join(inputargtypes))) + +class OpaqueTypeNode(TypeDefNode): + def __init__(self, db, T): + assert isinstance(T, lltype.OpaqueType) + self.db = db + self.T = T + self.make_name("RPyOpaque_%s" % (T.tag)) + + def writetypedef(self, codewriter): + codewriter.typedef(self.ref, "opaque*") + +class ExtOpaqueTypeNode(OpaqueTypeNode): + def writetypedef(self, codewriter): + pass + +def create_typedef_node(db, TYPE): + if isinstance(TYPE, lltype.FixedSizeArray): + return FixedSizeArrayTypeNode(db, TYPE) + + elif isinstance(TYPE, lltype.Struct): + return StructTypeNode(db, TYPE) + + elif isinstance(TYPE, lltype.FuncType): + return FuncTypeNode(db, TYPE) + + elif isinstance(TYPE, lltype.Array): + if TYPE.OF is lltype.Void: + return VoidArrayTypeNode(db, TYPE) + else: + return ArrayTypeNode(db, TYPE) + + elif isinstance(TYPE, lltype.OpaqueType): + if hasattr(TYPE, '_exttypeinfo'): + return ExtOpaqueTypeNode(db, TYPE) + else: + return OpaqueTypeNode(db, TYPE) + + else: + assert False, "create_typedef_node %s %s" % (TYPE, type(TYPE)) + From cfbolz at codespeak.net Mon Oct 29 14:30:15 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 29 Oct 2007 14:30:15 +0100 (CET) Subject: [pypy-svn] r48152 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071029133015.3C68E817B@code0.codespeak.net> Author: cfbolz Date: Mon Oct 29 14:30:13 2007 New Revision: 48152 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/objtable.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_model.py Log: rename getbyte and setbyte to getchar and setchar and make them return characters (instead of ints between 0 and 256). this gets rid of a lot of range checking everywhere. Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Mon Oct 29 14:30:13 2007 @@ -197,16 +197,18 @@ def at0(self, index0): from pypy.lang.smalltalk import objtable - return objtable.wrap_int(self.getbyte(index0)) + return objtable.wrap_int(ord(self.getchar(index0))) def atput0(self, index0, w_value): - self.setbyte(index0, unwrap_int(w_value)) + # XXX use to-be-written unwrap_char + self.setchar(index0, chr(unwrap_int(w_value))) - def getbyte(self, n): - return ord(self.bytes[n]) - - def setbyte(self, n, byte): - self.bytes[n] = chr(byte) + def getchar(self, n0): + return self.bytes[n0] + + def setchar(self, n0, character): + assert len(character) == 1 + self.bytes[n0] = character def size(self): return len(self.bytes) @@ -366,10 +368,10 @@ # of memory as smalltalk expects but wrapped in py-os raise NotImplementedError() else: - self.setbyte(index0, unwrap_int(w_value)) + # XXX use to-be-written unwrap_char + self.setchar(index0, chr(unwrap_int(w_value))) - def setbyte(self, index0, byte): - character = chr(byte) + def setchar(self, index0, character): self.bytes = (self.bytes[:index0] + character + self.bytes[index0 + 1:]) Modified: pypy/dist/pypy/lang/smalltalk/objtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/objtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/objtable.py Mon Oct 29 14:30:13 2007 @@ -13,10 +13,10 @@ def wrap_float(i): return model.W_Float(i) -def wrap_string(str): - w_inst = ct.w_String.as_class_get_shadow().new(len(str)) - for i in range(len(str)): - w_inst.setbyte(i, ord(str[i])) +def wrap_string(string): + w_inst = ct.w_String.as_class_get_shadow().new(len(string)) + for i in range(len(string)): + w_inst.setchar(i, string[i]) return w_inst def wrap_char(c): Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Mon Oct 29 14:30:13 2007 @@ -29,7 +29,7 @@ elif isinstance(w_obj, model.W_WordsObject): return objtable.wrap_int(w_obj.getword(idx)) elif isinstance(w_obj, model.W_BytesObject): - return objtable.wrap_int(w_obj.getbyte(idx)) + return objtable.wrap_int(w_obj.getchar(idx)) raise PrimitiveFailedError() def assert_bounds(n0, minimum, maximum): @@ -39,6 +39,8 @@ def assert_valid_index(n0, w_obj): if not 0 <= n0 < w_obj.size(): raise PrimitiveFailedError() + # return the index, since from here on the annotator knows that + # n0 cannot be negative return n0 # ___________________________________________________________________________ @@ -59,6 +61,7 @@ # indicates that what is pushed is an index1, but it is unwrapped and # converted to an index0 index1_0 = object() +char = object() def expose_primitive(code, unwrap_spec=None, no_result=False): # some serious magic, don't look @@ -87,8 +90,6 @@ interp.w_active_context.push(w_result) return w_result else: - for spec in unwrap_spec: - assert spec in (int, float, object, index1_0, str) len_unwrap_spec = len(unwrap_spec) assert (len_unwrap_spec == len(inspect.getargspec(func)[0]) + 1, "wrong number of arguments") @@ -102,19 +103,22 @@ args = () for i, spec in unrolling_unwrap_spec: index = -len_unwrap_spec + i - arg = frame.stack[index] + w_arg = frame.stack[index] if spec is int: - args += (unwrap_int(arg), ) + args += (unwrap_int(w_arg), ) elif spec is index1_0: - args += (unwrap_int(arg)-1, ) + args += (unwrap_int(w_arg)-1, ) elif spec is float: - args += (unwrap_float(arg), ) + args += (unwrap_float(w_arg), ) elif spec is object: - args += (arg, ) + args += (w_arg, ) elif spec is str: - args += (arg.as_string(), ) + args += (w_arg.as_string(), ) + elif spec is char: + args += (unwrap_char(w_arg), ) else: - assert 0, "this should never happen" + raise NotImplementedError( + "unknown unwrap_spec %s" % (spec, )) w_result = func(interp, *args) frame.pop_n(len_unwrap_spec) # only if no exception occurs! if not no_result: @@ -133,6 +137,12 @@ return w_value.value raise PrimitiveFailedError() +def unwrap_char(w_char): + if w_char.getclass() is not classtable.w_Character: + raise PrimitiveFailedError() + return chr(objtable.ord_w_char(w_char)) + + def wrap_int(value): if value > constants.TAGGED_MAXINT: raise PrimitiveFailedError() @@ -338,15 +348,13 @@ # make sure that getbyte is only performed on W_BytesObjects if not isinstance(w_obj, model.W_BytesObject): raise PrimitiveFailedError - byte = w_obj.getbyte(n0) - return objtable.CharacterTable[byte] + return objtable.wrap_char(w_obj.getchar(n0)) @expose_primitive(STRING_AT_PUT, unwrap_spec=[object, index1_0, object]) def func(interp, w_obj, n0, w_val): + val = unwrap_char(w_val) n0 = assert_valid_index(n0, w_obj) - if w_val.getclass() is not classtable.w_Character: - raise PrimitiveFailedError() - w_obj.setbyte(n0, objtable.ord_w_char(w_val)) + w_obj.setchar(n0, val) return w_val # ___________________________________________________________________________ Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Mon Oct 29 14:30:13 2007 @@ -28,11 +28,11 @@ assert w_bytes.getclass() is w_class assert w_bytes.size() == 20 assert w_class.as_class_get_shadow().instsize() == 0 - assert w_bytes.getbyte(3) == 00 - w_bytes.setbyte(3, 0xAA) - assert w_bytes.getbyte(3) == 0xAA - assert w_bytes.getbyte(0) == 0x00 - py.test.raises(IndexError, lambda: w_bytes.getbyte(20)) + assert w_bytes.getchar(3) == "\x00" + w_bytes.setchar(3, "\xAA") + assert w_bytes.getchar(3) == "\xAA" + assert w_bytes.getchar(0) == "\x00" + py.test.raises(IndexError, lambda: w_bytes.getchar(20)) def test_word_object(): w_class = mockclass(0, format=shadow.WORDS) @@ -70,9 +70,9 @@ classshadow = w_class.as_class_get_shadow() assert classshadow.lookup("foo").w_compiledin is w_super -def test_w_compiledin(): +def test_compiledmethod_setchar(): w_method = model.W_CompiledMethod(0, "abc") - w_method.setbyte(0, ord("c")) + w_method.setchar(0, "c") assert w_method.bytes == "cbc" def test_hashes(): From rxe at codespeak.net Mon Oct 29 15:00:09 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 29 Oct 2007 15:00:09 +0100 (CET) Subject: [pypy-svn] r48153 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071029140009.9C0B08119@code0.codespeak.net> Author: rxe Date: Mon Oct 29 15:00:06 2007 New Revision: 48153 Modified: pypy/dist/pypy/translator/llvm/test/test_rffi.py Log: monkey patch code for errno Modified: pypy/dist/pypy/translator/llvm/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rffi.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rffi.py Mon Oct 29 15:00:06 2007 @@ -5,13 +5,34 @@ from pypy.tool.udir import udir from pypy.rlib.rarithmetic import r_uint -from pypy.rpython.lltypesystem.rffi import * from pypy.rpython.lltypesystem.lltype import Signed, Ptr, Char, malloc from pypy.rpython.lltypesystem import lltype from pypy.translator.llvm.test.runtest import * -py.test.skip("rffi not there yet for llvm") +monkey_patch = False +if monkey_patch: + from pypy.rpython.lltypesystem.rffi import llexternal + c_source = py.code.Source(""" + int get_errno() { + return errno; + } + """) + get_errno = llexternal('get_errno', [], lltype.Signed, sources=[c_source]) + + c_source = py.code.Source(""" + void set_errno(int _errno) { + errno = _errno; + } + """) + set_errno = llexternal('set_errno', [lltype.Signed], lltype.Void, sources=[c_source]) + import pypy.rpython.lltypesystem.rffi + pypy.rpython.lltypesystem.rffi.get_errno = get_errno + pypy.rpython.lltypesystem.rffi.set_errno = set_errno +else: + py.test.skip("rffi not there yet for llvm") + +from pypy.rpython.lltypesystem.rffi import * def test_basic(): c_source = py.code.Source(""" From cfbolz at codespeak.net Mon Oct 29 15:37:14 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 29 Oct 2007 15:37:14 +0100 (CET) Subject: [pypy-svn] r48154 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071029143714.13C1D815E@code0.codespeak.net> Author: cfbolz Date: Mon Oct 29 15:37:13 2007 New Revision: 48154 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: get rid of a huge amount of whitespace at the end of lines Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Mon Oct 29 15:37:13 2007 @@ -1,7 +1,7 @@ import py import os -from pypy.lang.smalltalk import model -from pypy.lang.smalltalk import objtable +from pypy.lang.smalltalk import model +from pypy.lang.smalltalk import objtable from pypy.rlib import objectmodel from pypy.lang.smalltalk.tool.bitmanipulation import splitter @@ -17,7 +17,7 @@ first = ord(b[3]) # little endian if first & 0x80 != 0: first = first - 0x100 - return first << 24 | ord(b[2]) << 16 | ord(b[1]) << 8 | ord(b[0]) + return first << 24 | ord(b[2]) << 16 | ord(b[1]) << 8 | ord(b[0]) # ____________________________________________________________ @@ -25,7 +25,7 @@ # Reads an image file and creates all model objects class Stream(object): - """ Simple input stream """ + """ Simple input stream """ def __init__(self, inputfile): try: self.data = inputfile.read() @@ -36,44 +36,44 @@ self.count = 0 def peek(self): - if self.pos >= len(self.data): + if self.pos >= len(self.data): raise IndexError if self.swap: return swapped_chrs2int( self.data[self.pos:self.pos+4] ) - else: + else: return chrs2int( self.data[self.pos:self.pos+4] ) - + def next(self): integer = self.peek() self.pos += 4 self.count += 4 - return integer - + return integer + def reset_count(self): - self.count = 0 - + self.count = 0 + def skipbytes(self, jump): assert jump > 0 assert (self.pos + jump) <= len(self.data) - self.pos += jump - self.count += jump - + self.pos += jump + self.count += jump + def close(self): - pass # already closed - - - + pass # already closed + + + class CorruptImageError(Exception): - pass + pass # ____________________________________________________________ - + class ImageReader(object): def __init__(self, stream): self.stream = stream self.chunks = {} self.chunklist = [] - + def initialize(self): self.read_header() self.read_body() @@ -84,17 +84,17 @@ def read_header(self): version = self.stream.peek() - if version != 0x1966: + if version != 0x1966: self.stream.swap = True version = self.stream.peek() if version != 0x1966: raise CorruptImageError - version = self.stream.next() - #------ + version = self.stream.next() + #------ headersize = self.stream.next() self.endofmemory = self.stream.next() # endofmemory = bodysize - self.oldbaseaddress = self.stream.next() - self.specialobjectspointer = self.stream.next() + self.oldbaseaddress = self.stream.next() + self.specialobjectspointer = self.stream.next() lasthash = self.stream.next() savedwindowssize = self.stream.next() fullscreenflag = self.stream.next() @@ -109,19 +109,19 @@ if len(self.chunklist) % 1000 == 0: os.write(2,'#') self.chunklist.append(chunk) self.chunks[pos + self.oldbaseaddress] = chunk - self.stream.close() + self.stream.close() self.swap = self.stream.swap #save for later self.stream = None return self.chunklist # return for testing def init_g_objects(self): for chunk in self.chunks.itervalues(): - chunk.as_g_object(self) # initialized g_object + chunk.as_g_object(self) # initialized g_object def init_w_objects(self): self.assign_prebuilt_constants() for chunk in self.chunks.itervalues(): - chunk.g_object.init_w_object() + chunk.g_object.init_w_object() def assign_prebuilt_constants(self): from pypy.lang.smalltalk import classtable, constants, objtable @@ -143,20 +143,20 @@ def fillin_w_objects(self): for chunk in self.chunks.itervalues(): chunk.g_object.fillin_w_object() - + def init_compactclassesarray(self): """ (CompiledMethod Symbol Array PseudoContext LargePositiveInteger nil MethodDictionary Association Point Rectangle nil TranslatedMethod BlockContext MethodContext nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil ) """ - special = self.chunks[self.specialobjectspointer] + special = self.chunks[self.specialobjectspointer] assert special.size > 24 #at least assert special.format == 2 chunk = self.chunks[special.data[COMPACT_CLASSES_ARRAY]] assert len(chunk.data) == 31 assert chunk.format == 2 - self.compactclasses = [self.chunks[pointer] for pointer in chunk.data] - + self.compactclasses = [self.chunks[pointer] for pointer in chunk.data] + def read_object(self): kind = self.stream.peek() & 3 # 2 bits - if kind == 0: # 00 bits + if kind == 0: # 00 bits chunk, pos = self.read_3wordobjectheader() elif kind == 1: # 01 bits chunk, pos = self.read_2wordobjectheader() @@ -165,10 +165,10 @@ else: # 10 bits raise CorruptImageError("Unused block not allowed in image") size = chunk.size - chunk.data = [self.stream.next() - for _ in range(size - 1)] #size-1, excluding header - return chunk, pos - + chunk.data = [self.stream.next() + for _ in range(size - 1)] #size-1, excluding header + return chunk, pos + def read_1wordobjectheader(self): kind, size, format, classid, idhash = ( splitter[2,6,4,5,12](self.stream.next())) @@ -195,16 +195,16 @@ # ____________________________________________________________ class SqueakImage(object): - + def from_reader(self, reader): self.special_objects = [g_object.w_object for g_object in reader.chunks[reader.specialobjectspointer] .g_object.pointers] self.objects = [chunk.g_object.w_object for chunk in reader.chunklist] - + def special(self, index): - return self.special_objects[index] - + return self.special_objects[index] + COMPACT_CLASSES_ARRAY = 28 # ____________________________________________________________ @@ -217,55 +217,55 @@ """ def __init__(self): self.owner = None - + def isinitialized(self): - return self.owner is not None - + return self.owner is not None + def initialize_int(self, value, reader): self.owner = reader self.value = value self.size = -1 self.w_object = objtable.wrap_int(value) - + def initialize(self, chunk, reader): self.owner = reader self.size = chunk.size - self.hash12 = chunk.hash12 + self.hash12 = chunk.hash12 self.format = chunk.format self.init_class(chunk) self.init_data(chunk) # for pointers self.chunk = chunk # for bytes, words and compiledmethod self.w_object = None - - def init_class(self, chunk): + + def init_class(self, chunk): if chunk.iscompact(): self.g_class = self.owner.compactclasses[chunk.classid - 1].g_object # Smalltalk is 1-based indexed else: self.g_class = self.owner.chunks[chunk.classid].g_object - def init_data(self, chunk): + def init_data(self, chunk): if not self.ispointers(): return - self.pointers = [self.decode_pointer(pointer) + self.pointers = [self.decode_pointer(pointer) for pointer in chunk.data] assert None not in self.pointers - + def decode_pointer(self, pointer): if (pointer & 1) == 1: small_int = GenericObject() - small_int.initialize_int(pointer >> 1, self.owner) + small_int.initialize_int(pointer >> 1, self.owner) return small_int else: return self.owner.chunks[pointer].g_object - + def isbytes(self): return 8 <= self.format <= 11 - + def iswords(self): return self.format == 6 - + def ispointers(self): - return self.format < 5 #TODO, what about compiled methods? + return self.format < 5 #TODO, what about compiled methods? def init_w_object(self): """ 0 no fields @@ -283,13 +283,13 @@ # of literal oops specified in method header, followed by indexable bytes (same interpretation of low 2 bits as above) """ - if self.w_object is None: - if self.format < 5: + if self.w_object is None: + if self.format < 5: # XXX self.format == 4 is weak self.w_object = objectmodel.instantiate(model.W_PointersObject) elif self.format == 5: raise CorruptImageError("Unknown format 5") - elif self.format == 6: + elif self.format == 6: self.w_object = objectmodel.instantiate(model.W_WordsObject) elif self.format == 7: raise CorruptImageError("Unknown format 7, no 64-bit support yet :-)") @@ -298,19 +298,19 @@ elif 12 <= self.format <= 15: self.w_object = objectmodel.instantiate(model.W_CompiledMethod) else: - assert 0, "not reachable" + assert 0, "not reachable" return self.w_object - + def fillin_w_object(self): # below we are using an RPython idiom to 'cast' self.w_object # and pass the casted reference to the fillin_* methods - casted = self.w_object + casted = self.w_object if isinstance(casted, model.W_PointersObject): self.fillin_pointersobject(casted) elif isinstance(casted, model.W_WordsObject): self.fillin_wordsobject(casted) elif isinstance(casted, model.W_BytesObject): - self.fillin_bytesobject(casted) + self.fillin_bytesobject(casted) elif isinstance(casted, model.W_CompiledMethod): self.fillin_compiledmethod(casted) else: @@ -325,7 +325,7 @@ w_pointersobject.hash = self.chunk.hash12 if w_pointersobject._shadow is not None: w_pointersobject._shadow.invalidate() - + def fillin_wordsobject(self, w_wordsobject): w_wordsobject.words = self.chunk.data w_wordsobject.w_class = self.g_class.w_object @@ -335,31 +335,29 @@ w_bytesobject.w_class = self.g_class.w_object w_bytesobject.bytes = self.get_bytes() w_bytesobject.hash = self.chunk.hash12 # XXX check this - def get_bytes(self): bytes = [] if self.owner.swap: for each in self.chunk.data: bytes.append(chr((each >> 0) & 0xff)) - bytes.append(chr((each >> 8) & 0xff)) - bytes.append(chr((each >> 16) & 0xff)) + bytes.append(chr((each >> 8) & 0xff)) + bytes.append(chr((each >> 16) & 0xff)) bytes.append(chr((each >> 24) & 0xff)) - else: + else: for each in self.chunk.data: bytes.append(chr((each >> 24) & 0xff)) - bytes.append(chr((each >> 16) & 0xff)) - bytes.append(chr((each >> 8) & 0xff)) + bytes.append(chr((each >> 16) & 0xff)) + bytes.append(chr((each >> 8) & 0xff)) bytes.append(chr((each >> 0) & 0xff)) #strange, for example range(4)[:0] returns [] instead of [0,1,2,3]! #hence what we have to write list[:-odd] as list[:len(list)-odd] instead :( stop = len(bytes)-(self.format & 3) assert stop >= 0 return bytes[:stop] # omit odd bytes - - + def fillin_compiledmethod(self, w_compiledmethod): header = self.chunk.data[0] - #---!!!---- 1 tagged pointer! + #---!!!---- 1 tagged pointer! #(index 0) 9 bits: main part of primitive number (#primitive) #(index 9) 8 bits: number of literals (#numLiterals) #(index 17) 1 bit: whether a large frame size is needed (#frameSize) @@ -372,7 +370,7 @@ primitive = primitive + (highbit << 10) ##XXX todo, check this literals = [self.decode_pointer(pointer).w_object for pointer in self.chunk.data[:literalsize+1]] - bbytes = self.get_bytes()[(literalsize + 1)*4:] + bbytes = self.get_bytes()[(literalsize + 1)*4:] # XXX assert mirrorcache.get_or_build(self.g_class.w_object) is # ct.m_CompiledMethod w_compiledmethod.__init__( @@ -391,10 +389,10 @@ self.hash12 = hash12 self.data = None self.g_object = GenericObject() - + def __eq__(self, other): "(for testing)" - return (self.__class__ is other.__class__ and + return (self.__class__ is other.__class__ and self.format == other.format and self.classid == other.classid and self.hash12 == other.hash12 and @@ -403,13 +401,13 @@ def __ne__(self, other): "(for testing)" return not self == other - + def as_g_object(self, reader): if not self.g_object.isinitialized(): self.g_object.initialize(self, reader) - return self.g_object - + return self.g_object + def iscompact(self): - return 0 < self.classid < 32 - - + return 0 < self.classid < 32 + + From santagada at codespeak.net Mon Oct 29 15:37:35 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 29 Oct 2007 15:37:35 +0100 (CET) Subject: [pypy-svn] r48155 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071029143735.D5F688164@code0.codespeak.net> Author: santagada Date: Mon Oct 29 15:37:35 2007 New Revision: 48155 Added: pypy/dist/pypy/translator/llvm/test/test_rlist.py Modified: pypy/dist/pypy/translator/llvm/test/runtest.py Log: begginings of a LLVMTest class, still lots to go Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Mon Oct 29 15:37:35 2007 @@ -1,6 +1,7 @@ import py py.test.skip("llvm is a state of flux") +from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin from pypy.translator.llvm.buildllvm import llvm_is_on_path, llvm_version, gcc_version from pypy.translator.llvm.genllvm import GenLLVM @@ -107,3 +108,64 @@ " returns compiled function " return compile_test(function, annotation, isolate_hint=isolate_hint, **kwds)[1] +# XXX Work in progress, this was mostly copied from JsTest +class LLVMTest(BaseRtypingTest, LLRtypeMixin): + def _compile(self, _fn, args, policy=None): + # argnames = _fn.func_code.co_varnames[:_fn.func_code.co_argcount] + # func_name = _fn.func_name + # if func_name == '': + # func_name = 'func' + # source = py.code.Source(""" + # def %s(): + # from pypy.rlib.nonconst import NonConstant + # res = _fn(%s) + # if isinstance(res, type(None)): + # return None + # else: + # return str(res)""" + # % (func_name, ",".join(["%s=NonConstant(%r)" % (name, i) for + # name, i in zip(argnames, args)]))) + # exec source.compile() in locals() + return compile_function(_fn, []) + + def interpret(self, fn, args, policy=None): + f = self._compile(fn, args) + res = f(*args) + return res + + def interpret_raises(self, exception, fn, args): + #import exceptions # needed by eval + #try: + #import pdb; pdb.set_trace() + try: + res = self.interpret(fn, args) + except JSException, e: + s = e.args[0] + assert s.startswith('uncaught exception:') + assert re.search(exception.__name__, s) + else: + raise AssertionError("Did not raise, returned %s" % res) + #except ExceptionWrapper, ex: + # assert issubclass(eval(ex.class_name), exception) + #else: + # assert False, 'function did raise no exception at all' + # + # def ll_to_string(self, s): + # return str(s) + # + # def ll_to_list(self, l): + # return l + # + # def ll_unpack_tuple(self, t, length): + # assert len(t) == length + # return tuple(t) + # + # def class_name(self, value): + # return value[:-8].split('.')[-1] + # + # def is_of_instance_type(self, val): + # m = re.match("^<.* object>$", val) + # return bool(m) + # + # def read_attr(self, obj, name): + # py.test.skip('read_attr not supported on genjs tests') Added: pypy/dist/pypy/translator/llvm/test/test_rlist.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_rlist.py Mon Oct 29 15:37:35 2007 @@ -0,0 +1,8 @@ +import py +from pypy.rpython.test.test_rlist import BaseTestRlist +from pypy.translator.llvm.test.runtest import LLVMTest + +# ====> ../../../rpython/test/test_rlist.py + +class TestLLVMList(LLVMTest, BaseTestRlist): + pass \ No newline at end of file From cfbolz at codespeak.net Mon Oct 29 15:46:04 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 29 Oct 2007 15:46:04 +0100 (CET) Subject: [pypy-svn] r48156 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071029144604.916668164@code0.codespeak.net> Author: cfbolz Date: Mon Oct 29 15:46:04 2007 New Revision: 48156 Modified: pypy/dist/pypy/lang/smalltalk/objtable.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_model.py Log: get rid of the ugly "import ... as abbr" Modified: pypy/dist/pypy/lang/smalltalk/objtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/objtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/objtable.py Mon Oct 29 15:46:04 2007 @@ -1,4 +1,4 @@ -import pypy.lang.smalltalk.classtable as ct +from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import constants from pypy.lang.smalltalk import model @@ -14,7 +14,7 @@ return model.W_Float(i) def wrap_string(string): - w_inst = ct.w_String.as_class_get_shadow().new(len(string)) + w_inst = classtable.w_String.as_class_get_shadow().new(len(string)) for i in range(len(string)): w_inst.setchar(i, string[i]) return w_inst @@ -23,9 +23,9 @@ return CharacterTable[ord(c)] def ord_w_char(w_c): - assert w_c.getclass() is ct.w_Character + assert w_c.getclass() is classtable.w_Character w_ord = w_c.fetch(constants.CHARACTER_VALUE_INDEX) - assert w_ord.getclass() is ct.w_SmallInteger + assert w_ord.getclass() is classtable.w_SmallInteger assert isinstance(w_ord, model.W_SmallInteger) return w_ord.value @@ -41,10 +41,10 @@ a wrapped smalltalk array """ lstlen = len(lit) - res = ct.w_Array.as_class_get_shadow().new(lstlen) + res = classtable.w_Array.as_class_get_shadow().new(lstlen) for i in range(lstlen): res.storevarpointer(i, fakeliteral(lit[i])) - return res + return res # ___________________________________________________________________________ # Global Data @@ -52,14 +52,14 @@ def wrap_char_table(): global CharacterTable def bld_char(i): - w_cinst = ct.w_Character.as_class_get_shadow().new() + w_cinst = classtable.w_Character.as_class_get_shadow().new() w_cinst.store(constants.CHARACTER_VALUE_INDEX, wrap_int(i)) return w_cinst CharacterTable = [bld_char(i) for i in range(256)] wrap_char_table() -w_true = ct.classtable['w_True'].as_class_get_shadow().new() -w_false = ct.classtable['w_False'].as_class_get_shadow().new() +w_true = classtable.classtable['w_True'].as_class_get_shadow().new() +w_false = classtable.classtable['w_False'].as_class_get_shadow().new() w_mone = wrap_int(-1) w_zero = wrap_int(0) w_one = wrap_int(1) @@ -69,7 +69,7 @@ # initialize their fields to nil, we have to create it in the model # package, and then patch up its fields here: w_nil = model.w_nil -w_nil.w_class = ct.classtable['w_UndefinedObject'] +w_nil.w_class = classtable.classtable['w_UndefinedObject'] objtable = {} Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Mon Oct 29 15:46:04 2007 @@ -1,10 +1,9 @@ import py from pypy.lang.smalltalk import model, interpreter, primitives, shadow -from pypy.lang.smalltalk import objtable +from pypy.lang.smalltalk import objtable, classtable from pypy.lang.smalltalk.objtable import wrap_int, wrap_char -import pypy.lang.smalltalk.classtable as ct -mockclass = ct.bootstrap_class +mockclass = classtable.bootstrap_class # expose the bytecode's values as global constants. # Bytecodes that have a whole range are exposed as global functions: @@ -57,7 +56,7 @@ return wrap_int(lit) elif isinstance(lit, list): lstlen = len(lit) - res = ct.w_Array.as_class_get_shadow().new(lstlen) + res = classtable.w_Array.as_class_get_shadow().new(lstlen) for i in range(lstlen): res.storevarpointer(i, fakeliteral(lit[i])) return res @@ -271,7 +270,7 @@ interp = new_interpreter(pushConstantOneBytecode + bytecodePrimClass) interp.step() interp.step() - assert interp.w_active_context.pop() == ct.w_SmallInteger + assert interp.w_active_context.pop() == classtable.w_SmallInteger assert interp.w_active_context.stack == [] def test_bytecodePrimSubtract(): @@ -421,7 +420,7 @@ assert primitives.unwrap_int(w_result) == 42 run_with_faked_methods( - [[ct.w_SmallInteger, primitives.SUBTRACT, + [[classtable.w_SmallInteger, primitives.SUBTRACT, 1, "sub"]], test) @@ -652,7 +651,7 @@ 176, 125, 33, 34, 240, 124 ], fakeliterals("value:value:", wrap_int(3), wrap_int(4))).value == 7 run_with_faked_methods( - [[ct.w_BlockContext, primitives.PRIMITIVE_VALUE, + [[classtable.w_BlockContext, primitives.PRIMITIVE_VALUE, 2, "value:value:"]], test) @@ -690,7 +689,7 @@ fakeliterals("valueWithArguments:", [3, 2])).value == 1 run_with_faked_methods( - [[ct.w_BlockContext, primitives.PRIMITIVE_VALUE_WITH_ARGS, + [[classtable.w_BlockContext, primitives.PRIMITIVE_VALUE_WITH_ARGS, 1, "valueWithArguments:"]], test) @@ -701,7 +700,7 @@ [ 32, 118, 192, 124], fakeliterals("a")) == wrap_char("a") run_with_faked_methods( - [[ct.w_String, primitives.STRING_AT, 1, "at:"]], + [[classtable.w_String, primitives.STRING_AT, 1, "at:"]], test) def test_bc_primBytecodeAtPut_string(): @@ -711,7 +710,7 @@ [ 32, 118, 33, 193, 124 ], fakeliterals("a", wrap_char("b"))) == wrap_char("b") run_with_faked_methods( - [[ct.w_String, primitives.STRING_AT_PUT, 2, "at:put:"]], + [[classtable.w_String, primitives.STRING_AT_PUT, 2, "at:put:"]], test) def test_bc_primBytecodeAt_with_instvars(): @@ -766,6 +765,6 @@ assert interpret_bc( [112, 119, 224, 124], oal, receiver=prim_meth).value == 3 run_with_faked_methods( - [[ct.w_CompiledMethod, primitives.OBJECT_AT, 1, "objectAt:"], - [ct.w_CompiledMethod, primitives.OBJECT_AT_PUT, 2, "objectAt:put:"]], + [[classtable.w_CompiledMethod, primitives.OBJECT_AT, 1, "objectAt:"], + [classtable.w_CompiledMethod, primitives.OBJECT_AT_PUT, 2, "objectAt:put:"]], test) Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Mon Oct 29 15:46:04 2007 @@ -1,9 +1,9 @@ import py from pypy.lang.smalltalk import model, shadow, objtable from pypy.lang.smalltalk.shadow import MethodNotFound -import pypy.lang.smalltalk.classtable as ct +from pypy.lang.smalltalk import classtable -mockclass = ct.bootstrap_class +mockclass = classtable.bootstrap_class def test_new(): w_mycls = mockclass(0) From cfbolz at codespeak.net Mon Oct 29 15:56:42 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 29 Oct 2007 15:56:42 +0100 (CET) Subject: [pypy-svn] r48157 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071029145642.88002816A@code0.codespeak.net> Author: cfbolz Date: Mon Oct 29 15:56:37 2007 New Revision: 48157 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/objtable.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: w_mone -> w_minus_one Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Mon Oct 29 15:56:37 2007 @@ -20,7 +20,7 @@ TRUE = objtable.w_true FALSE = objtable.w_false NIL = objtable.w_nil - MONE = objtable.w_mone + MINUS_ONE = objtable.w_minus_one ZERO = objtable.w_zero ONE = objtable.w_one TWO = objtable.w_two @@ -123,7 +123,7 @@ self.push(interp.NIL) def pushConstantMinusOneBytecode(self, interp): - self.push(interp.MONE) + self.push(interp.MINUS_ONE) def pushConstantZeroBytecode(self, interp): self.push(interp.ZERO) Modified: pypy/dist/pypy/lang/smalltalk/objtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/objtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/objtable.py Mon Oct 29 15:56:37 2007 @@ -53,17 +53,18 @@ global CharacterTable def bld_char(i): w_cinst = classtable.w_Character.as_class_get_shadow().new() - w_cinst.store(constants.CHARACTER_VALUE_INDEX, wrap_int(i)) + w_cinst.store(constants.CHARACTER_VALUE_INDEX, + model.W_SmallInteger(i)) return w_cinst CharacterTable = [bld_char(i) for i in range(256)] wrap_char_table() w_true = classtable.classtable['w_True'].as_class_get_shadow().new() w_false = classtable.classtable['w_False'].as_class_get_shadow().new() -w_mone = wrap_int(-1) -w_zero = wrap_int(0) -w_one = wrap_int(1) -w_two = wrap_int(2) +w_minus_one = model.W_SmallInteger(-1) +w_zero = model.W_SmallInteger(0) +w_one = model.W_SmallInteger(1) +w_two = model.W_SmallInteger(2) # Very special nil hack: in order to allow W_PointersObject's to # initialize their fields to nil, we have to create it in the model Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Mon Oct 29 15:56:37 2007 @@ -625,7 +625,7 @@ (PUSH_TRUE, objtable.w_true), (PUSH_FALSE, objtable.w_false), (PUSH_NIL, objtable.w_nil), - (PUSH_MINUS_ONE, objtable.w_mone), + (PUSH_MINUS_ONE, objtable.w_minus_one), (PUSH_ZERO, objtable.w_zero), (PUSH_ONE, objtable.w_one), (PUSH_TWO, objtable.w_two), Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Mon Oct 29 15:56:37 2007 @@ -147,7 +147,7 @@ w_doesnot = image.special(constants.SO_DOES_NOT_UNDERSTAND) assert repr(w_doesnot.shadow_of_my_class()) == "" assert repr(objtable.w_nil.shadow_of_my_class()) == "" - assert repr(objtable.w_mone.shadow_of_my_class()) == "" + assert repr(objtable.w_minus_one.shadow_of_my_class()) == "" assert repr(objtable.w_zero.shadow_of_my_class()) == "" assert repr(objtable.w_one.shadow_of_my_class()) == "" assert repr(objtable.w_two.shadow_of_my_class()) == "" Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Mon Oct 29 15:56:37 2007 @@ -291,7 +291,7 @@ (primitives.PUSH_TRUE, objtable.w_true), (primitives.PUSH_FALSE, objtable.w_false), (primitives.PUSH_NIL, objtable.w_nil), - (primitives.PUSH_MINUS_ONE, objtable.w_mone), + (primitives.PUSH_MINUS_ONE, objtable.w_minus_one), (primitives.PUSH_ZERO, objtable.w_zero), (primitives.PUSH_ONE, objtable.w_one), (primitives.PUSH_TWO, objtable.w_two), From santagada at codespeak.net Mon Oct 29 16:03:03 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 29 Oct 2007 16:03:03 +0100 (CET) Subject: [pypy-svn] r48158 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071029150303.2429F816A@code0.codespeak.net> Author: santagada Date: Mon Oct 29 16:03:00 2007 New Revision: 48158 Modified: pypy/dist/pypy/translator/llvm/test/runtest.py pypy/dist/pypy/translator/llvm/test/test_rlist.py Log: run things like jstest... and removed some commented out code. Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Mon Oct 29 16:03:00 2007 @@ -111,22 +111,22 @@ # XXX Work in progress, this was mostly copied from JsTest class LLVMTest(BaseRtypingTest, LLRtypeMixin): def _compile(self, _fn, args, policy=None): - # argnames = _fn.func_code.co_varnames[:_fn.func_code.co_argcount] - # func_name = _fn.func_name - # if func_name == '': - # func_name = 'func' - # source = py.code.Source(""" - # def %s(): - # from pypy.rlib.nonconst import NonConstant - # res = _fn(%s) - # if isinstance(res, type(None)): - # return None - # else: - # return str(res)""" - # % (func_name, ",".join(["%s=NonConstant(%r)" % (name, i) for - # name, i in zip(argnames, args)]))) - # exec source.compile() in locals() - return compile_function(_fn, []) + argnames = _fn.func_code.co_varnames[:_fn.func_code.co_argcount] + func_name = _fn.func_name + if func_name == '': + func_name = 'func' + source = py.code.Source(""" + def %s(): + from pypy.rlib.nonconst import NonConstant + res = _fn(%s) + if isinstance(res, type(None)): + return None + else: + return str(res)""" + % (func_name, ",".join(["%s=NonConstant(%r)" % (name, i) for + name, i in zip(argnames, args)]))) + exec source.compile() in locals() + return compile_function(locals()[func_name], []) def interpret(self, fn, args, policy=None): f = self._compile(fn, args) @@ -139,33 +139,12 @@ #import pdb; pdb.set_trace() try: res = self.interpret(fn, args) - except JSException, e: - s = e.args[0] - assert s.startswith('uncaught exception:') - assert re.search(exception.__name__, s) + except Exception, e: + assert issubclass(eval(ex.class_name), exception) else: raise AssertionError("Did not raise, returned %s" % res) #except ExceptionWrapper, ex: # assert issubclass(eval(ex.class_name), exception) #else: # assert False, 'function did raise no exception at all' - # - # def ll_to_string(self, s): - # return str(s) - # - # def ll_to_list(self, l): - # return l - # - # def ll_unpack_tuple(self, t, length): - # assert len(t) == length - # return tuple(t) - # - # def class_name(self, value): - # return value[:-8].split('.')[-1] - # - # def is_of_instance_type(self, val): - # m = re.match("^<.* object>$", val) - # return bool(m) - # - # def read_attr(self, obj, name): - # py.test.skip('read_attr not supported on genjs tests') + Modified: pypy/dist/pypy/translator/llvm/test/test_rlist.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rlist.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rlist.py Mon Oct 29 16:03:00 2007 @@ -5,4 +5,4 @@ # ====> ../../../rpython/test/test_rlist.py class TestLLVMList(LLVMTest, BaseTestRlist): - pass \ No newline at end of file + pass From santagada at codespeak.net Mon Oct 29 16:35:23 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 29 Oct 2007 16:35:23 +0100 (CET) Subject: [pypy-svn] r48159 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071029153523.C31718182@code0.codespeak.net> Author: santagada Date: Mon Oct 29 16:35:23 2007 New Revision: 48159 Modified: pypy/dist/pypy/translator/llvm/test/runtest.py Log: now mostly copied from cli, need to fix it Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Mon Oct 29 16:35:23 2007 @@ -4,6 +4,8 @@ from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin from pypy.translator.llvm.buildllvm import llvm_is_on_path, llvm_version, gcc_version from pypy.translator.llvm.genllvm import GenLLVM +from pypy.annotation.model import lltype_to_annotation +from pypy.rpython.lltypesystem.lltype import typeOf optimize_tests = False MINIMUM_LLVM_VERSION = 1.9 @@ -108,43 +110,84 @@ " returns compiled function " return compile_test(function, annotation, isolate_hint=isolate_hint, **kwds)[1] -# XXX Work in progress, this was mostly copied from JsTest +# XXX Work in progress, this was mostly copied from cli +class InstanceWrapper: + def __init__(self, class_name): + self.class_name = class_name + +class ExceptionWrapper: + def __init__(self, class_name): + self.class_name = class_name + + def __repr__(self): + return 'ExceptionWrapper(%s)' % repr(self.class_name) + + class LLVMTest(BaseRtypingTest, LLRtypeMixin): - def _compile(self, _fn, args, policy=None): - argnames = _fn.func_code.co_varnames[:_fn.func_code.co_argcount] - func_name = _fn.func_name - if func_name == '': - func_name = 'func' - source = py.code.Source(""" - def %s(): - from pypy.rlib.nonconst import NonConstant - res = _fn(%s) - if isinstance(res, type(None)): - return None - else: - return str(res)""" - % (func_name, ",".join(["%s=NonConstant(%r)" % (name, i) for - name, i in zip(argnames, args)]))) - exec source.compile() in locals() - return compile_function(locals()[func_name], []) + def __init__(self): + self._func = None + self._ann = None + self._llvm_func = None + + def _compile(self, fn, args, ann=None): + if ann is None: + ann = [lltype_to_annotation(typeOf(x)) for x in args] + if self._func is fn and self._ann == ann: + return self._llvm_func + else: + self._llvm_func = compile_function(fn, ann) + self._func = fn + self._ann = ann + return self._llvm_func + + def _skip_win(self, reason): + if platform.system() == 'Windows': + py.test.skip('Windows --> %s' % reason) + + def _skip_powerpc(self, reason): + if platform.processor() == 'powerpc': + py.test.skip('PowerPC --> %s' % reason) + + def _skip_llinterpreter(self, reason, skipLL=True, skipOO=True): + pass - def interpret(self, fn, args, policy=None): - f = self._compile(fn, args) + def interpret(self, fn, args, annotation=None): + f = self._compile(fn, args, annotation) res = f(*args) + if isinstance(res, ExceptionWrapper): + raise res return res def interpret_raises(self, exception, fn, args): - #import exceptions # needed by eval - #try: - #import pdb; pdb.set_trace() + import exceptions # needed by eval try: - res = self.interpret(fn, args) - except Exception, e: + self.interpret(fn, args) + except ExceptionWrapper, ex: assert issubclass(eval(ex.class_name), exception) else: - raise AssertionError("Did not raise, returned %s" % res) - #except ExceptionWrapper, ex: - # assert issubclass(eval(ex.class_name), exception) - #else: - # assert False, 'function did raise no exception at all' + assert False, 'function did raise no exception at all' + + def float_eq(self, x, y): + diff = abs(x-y) + return diff/x < 10**-FLOAT_PRECISION + + def is_of_type(self, x, type_): + return True # we can't really test the type + + def ll_to_string(self, s): + return s + + def ll_to_list(self, l): + return l + + def ll_to_tuple(self, t): + return t + + def class_name(self, value): + return value.class_name.split(".")[-1] + + def is_of_instance_type(self, val): + return isinstance(val, InstanceWrapper) + def read_attr(self, obj, name): + py.test.skip('read_attr not supported on llvm tests') From cfbolz at codespeak.net Mon Oct 29 16:43:04 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 29 Oct 2007 16:43:04 +0100 (CET) Subject: [pypy-svn] r48160 - in pypy/dist/pypy: lang/smalltalk lang/smalltalk/test translator/goal Message-ID: <20071029154304.AEA118176@code0.codespeak.net> Author: cfbolz Date: Mon Oct 29 16:43:04 2007 New Revision: 48160 Added: pypy/dist/pypy/lang/smalltalk/error.py pypy/dist/pypy/lang/smalltalk/utility.py Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/objtable.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/shadow.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py pypy/dist/pypy/lang/smalltalk/test/test_shadow.py pypy/dist/pypy/translator/goal/targetfibsmalltalk.py pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py Log: put all the (un)wrap_* functions into a utility module Added: pypy/dist/pypy/lang/smalltalk/error.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/error.py Mon Oct 29 16:43:04 2007 @@ -0,0 +1,17 @@ +# some exception classes for the Smalltalk VM + +class SmalltalkException(Exception): + """Base class for Smalltalk exception hierarchy""" + +class PrimitiveFailedError(SmalltalkException): + pass + +class PrimitiveNotYetWrittenError(PrimitiveFailedError): + pass + +class UnwrappingError(PrimitiveFailedError): + pass + +class WrappingError(PrimitiveFailedError): + pass + Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Mon Oct 29 16:43:04 2007 @@ -56,14 +56,6 @@ return False return self.value == other.value -class UnwrappingError(Exception): - pass - -def unwrap_int(w_value): - if isinstance(w_value, W_SmallInteger): - return w_value.value - raise UnwrappingError("expected a W_SmallInteger, got %s" % (w_value,)) - class W_Float(W_Object): def __init__(self, value): self.value = value @@ -196,12 +188,12 @@ self.bytes = ['\x00'] * size def at0(self, index0): - from pypy.lang.smalltalk import objtable - return objtable.wrap_int(ord(self.getchar(index0))) + from pypy.lang.smalltalk import utility + return utility.wrap_int(ord(self.getchar(index0))) def atput0(self, index0, w_value): - # XXX use to-be-written unwrap_char - self.setchar(index0, chr(unwrap_int(w_value))) + from pypy.lang.smalltalk import utility + self.setchar(index0, chr(utility.unwrap_int(w_value))) def getchar(self, n0): return self.bytes[n0] @@ -241,11 +233,12 @@ self.words = [0] * size def at0(self, index0): - from pypy.lang.smalltalk import objtable - return objtable.wrap_int(self.getword(index0)) + from pypy.lang.smalltalk import utility + return utility.wrap_int(self.getword(index0)) def atput0(self, index0, w_value): - self.setword(index0, unwrap_int(w_value)) + from pypy.lang.smalltalk import utility + self.setword(index0, utility.unwrap_int(w_value)) def getword(self, n): return self.words[n] @@ -353,15 +346,16 @@ def at0(self, index0): # XXX - from pypy.lang.smalltalk import objtable + from pypy.lang.smalltalk import utility index0 = index0 - self.staticsize() if index0 < 0: # XXX Do something useful with this.... we are not a block # of memory as smalltalk expects but wrapped in py-os raise NotImplementedError() - return objtable.wrap_int(ord(self.bytes[index0])) + return utility.wrap_int(ord(self.bytes[index0])) def atput0(self, index0, w_value): + from pypy.lang.smalltalk import utility index0 = index0 - self.staticsize() if index0 < 0: # XXX Do something useful with this.... we are not a block @@ -369,7 +363,7 @@ raise NotImplementedError() else: # XXX use to-be-written unwrap_char - self.setchar(index0, chr(unwrap_int(w_value))) + self.setchar(index0, chr(utility.unwrap_int(w_value))) def setchar(self, index0, character): self.bytes = (self.bytes[:index0] + character + @@ -394,16 +388,16 @@ # Imitate the primitive accessors def fetch(self, index): - from pypy.lang.smalltalk import objtable + from pypy.lang.smalltalk import utility, objtable if index == constants.CTXPART_SENDER_INDEX: if self.w_sender: return self.w_sender else: return objtable.w_nil elif index == constants.CTXPART_PC_INDEX: - return objtable.wrap_int(self.pc) + return utility.wrap_int(self.pc) elif index == constants.CTXPART_STACKP_INDEX: - return objtable.wrap_int(len(self.stack)) + return utility.wrap_int(len(self.stack)) # Invalid! raise IndexError @@ -488,11 +482,11 @@ return w_BlockContext def fetch(self, index): - from pypy.lang.smalltalk import objtable + from pypy.lang.smalltalk import utility if index == constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX: - return objtable.wrap_int(self.argcnt) + return utility.wrap_int(self.argcnt) elif index == constants.BLKCTX_INITIAL_IP_INDEX: - return objtable.wrap_int(self.initialip) + return utility.wrap_int(self.initialip) elif index == constants.BLKCTX_HOME_INDEX: return self.w_home elif index >= constants.BLKCTX_TEMP_FRAME_START: @@ -504,10 +498,11 @@ def store(self, index, value): # THIS IS ALL UNTESTED CODE and we're a bit unhappy about it # because it crashd the translation N+4 times :-( + from pypy.lang.smalltalk import utility if index == constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX: - self.argcnt = unwrap_int(value) + self.argcnt = utility.unwrap_int(value) elif index == constants.BLKCTX_INITIAL_IP_INDEX: - self.pc = unwrap_int(value) + self.pc = utility.unwrap_int(value) elif index == constants.BLKCTX_HOME_INDEX: assert isinstance(value, W_MethodContext) self.w_home = value Modified: pypy/dist/pypy/lang/smalltalk/objtable.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/objtable.py (original) +++ pypy/dist/pypy/lang/smalltalk/objtable.py Mon Oct 29 16:43:04 2007 @@ -3,50 +3,6 @@ from pypy.lang.smalltalk import model # ___________________________________________________________________________ -# Utility Methods - -def wrap_int(i): - if i <= 0x3FFFFFFF and i >= -0x40000000: - return model.W_SmallInteger(i) - raise NotImplementedError - -def wrap_float(i): - return model.W_Float(i) - -def wrap_string(string): - w_inst = classtable.w_String.as_class_get_shadow().new(len(string)) - for i in range(len(string)): - w_inst.setchar(i, string[i]) - return w_inst - -def wrap_char(c): - return CharacterTable[ord(c)] - -def ord_w_char(w_c): - assert w_c.getclass() is classtable.w_Character - w_ord = w_c.fetch(constants.CHARACTER_VALUE_INDEX) - assert w_ord.getclass() is classtable.w_SmallInteger - assert isinstance(w_ord, model.W_SmallInteger) - return w_ord.value - -def wrap_bool(bool): - if bool: - return w_true - else: - return w_false - -def wrap_list(lst_w_obj): - """ - Converts a Python list of wrapper objects into - a wrapped smalltalk array - """ - lstlen = len(lit) - res = classtable.w_Array.as_class_get_shadow().new(lstlen) - for i in range(lstlen): - res.storevarpointer(i, fakeliteral(lit[i])) - return res - -# ___________________________________________________________________________ # Global Data def wrap_char_table(): Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Mon Oct 29 16:43:04 2007 @@ -1,18 +1,14 @@ import inspect import math import operator -from pypy.lang.smalltalk import model, shadow +from pypy.lang.smalltalk import model, shadow, utility from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import objtable from pypy.lang.smalltalk import constants +from pypy.lang.smalltalk.error import PrimitiveFailedError, \ + PrimitiveNotYetWrittenError from pypy.rlib import rarithmetic -class PrimitiveFailedError(Exception): - pass - -class PrimitiveNotYetWrittenError(PrimitiveFailedError): - pass - def unwrap_float(w_v): if isinstance(w_v, model.W_Float): return w_v.value elif isinstance(w_v, model.W_SmallInteger): return float(w_v.value) @@ -27,9 +23,9 @@ if isinstance(w_obj, model.W_PointersObject): return w_obj.fetch(idx) elif isinstance(w_obj, model.W_WordsObject): - return objtable.wrap_int(w_obj.getword(idx)) + return utility.wrap_int(w_obj.getword(idx)) elif isinstance(w_obj, model.W_BytesObject): - return objtable.wrap_int(w_obj.getchar(idx)) + return utility.wrap_int(w_obj.getchar(idx)) raise PrimitiveFailedError() def assert_bounds(n0, minimum, maximum): @@ -105,9 +101,9 @@ index = -len_unwrap_spec + i w_arg = frame.stack[index] if spec is int: - args += (unwrap_int(w_arg), ) + args += (utility.unwrap_int(w_arg), ) elif spec is index1_0: - args += (unwrap_int(w_arg)-1, ) + args += (utility.unwrap_int(w_arg)-1, ) elif spec is float: args += (unwrap_float(w_arg), ) elif spec is object: @@ -132,24 +128,7 @@ # ___________________________________________________________________________ # SmallInteger Primitives -def unwrap_int(w_value): - if isinstance(w_value, model.W_SmallInteger): - return w_value.value - raise PrimitiveFailedError() - -def unwrap_char(w_char): - if w_char.getclass() is not classtable.w_Character: - raise PrimitiveFailedError() - return chr(objtable.ord_w_char(w_char)) - -def wrap_int(value): - if value > constants.TAGGED_MAXINT: - raise PrimitiveFailedError() - if value < constants.TAGGED_MININT: - raise PrimitiveFailedError() - return objtable.wrap_int(value) - ADD = 1 SUBTRACT = 2 MULTIPLY = 9 @@ -175,7 +154,7 @@ res = rarithmetic.ovfcheck(op(receiver, argument)) except OverflowError: raise PrimitiveFailedError() - return wrap_int(res) + return utility.wrap_int(res) make_func(op) bitwise_binary_ops = { @@ -188,7 +167,7 @@ @expose_primitive(code, unwrap_spec=[int, int]) def func(interp, receiver, argument): res = op(receiver, argument) - return wrap_int(res) + return utility.wrap_int(res) make_func(op) # #/ -- return the result of a division, only succeed if the division is exact @@ -198,28 +177,28 @@ raise PrimitiveFailedError() if receiver % argument != 0: raise PrimitiveFailedError() - return wrap_int(receiver // argument) + return utility.wrap_int(receiver // argument) # #\\ -- return the remainder of a division @expose_primitive(MOD, unwrap_spec=[int, int]) def func(interp, receiver, argument): if argument == 0: raise PrimitiveFailedError() - return wrap_int(receiver % argument) + return utility.wrap_int(receiver % argument) # #// -- return the result of a division, rounded towards negative zero @expose_primitive(DIV, unwrap_spec=[int, int]) def func(interp, receiver, argument): if argument == 0: raise PrimitiveFailedError() - return wrap_int(receiver // argument) + return utility.wrap_int(receiver // argument) # #// -- return the result of a division, rounded towards negative infinity @expose_primitive(QUO, unwrap_spec=[int, int]) def func(interp, receiver, argument): if argument == 0: raise PrimitiveFailedError() - return wrap_int(receiver // argument) + return utility.wrap_int(receiver // argument) # #bitShift: -- return the shifted value @expose_primitive(BIT_SHIFT, unwrap_spec=[int, int]) @@ -230,11 +209,11 @@ shifted = receiver << argument if (shifted >> argument) != receiver: raise PrimitiveFailedError() - return wrap_int(shifted) + return utility.wrap_int(shifted) # right shift, ok to lose bits else: - return wrap_int(receiver >> -argument) + return utility.wrap_int(receiver >> -argument) # ___________________________________________________________________________ @@ -265,35 +244,35 @@ def make_func(op): @expose_primitive(code, unwrap_spec=[float, float]) def func(interp, v1, v2): - w_res = objtable.wrap_float(op(v1, v2)) + w_res = utility.wrap_float(op(v1, v2)) return w_res make_func(op) @expose_primitive(FLOAT_TRUNCATED, unwrap_spec=[float]) def func(interp, f): - w_res = objtable.wrap_int(int(f)) + w_res = utility.wrap_int(int(f)) return w_res @expose_primitive(FLOAT_TIMES_TWO_POWER, unwrap_spec=[float, int]) def func(interp, rcvr, arg): - w_res = objtable.wrap_float(math.ldexp(rcvr, arg)) + w_res = utility.wrap_float(math.ldexp(rcvr, arg)) return w_res @expose_primitive(FLOAT_SQUARE_ROOT, unwrap_spec=[float]) def func(interp, f): if f < 0.0: raise PrimitiveFailedError - w_res = objtable.wrap_float(math.sqrt(f)) + w_res = utility.wrap_float(math.sqrt(f)) return w_res @expose_primitive(FLOAT_SIN, unwrap_spec=[float]) def func(interp, f): - w_res = objtable.wrap_float(math.sin(f)) + w_res = utility.wrap_float(math.sin(f)) return w_res @expose_primitive(FLOAT_ARCTAN, unwrap_spec=[float]) def func(interp, f): - w_res = objtable.wrap_float(math.atan(f)) + w_res = utility.wrap_float(math.atan(f)) return w_res @expose_primitive(FLOAT_LOG_N, unwrap_spec=[float]) @@ -304,11 +283,11 @@ res = rarithmetic.NAN else: res = math.log(f) - return objtable.wrap_float(res) + return utility.wrap_float(res) @expose_primitive(FLOAT_EXP, unwrap_spec=[float]) def func(interp, f): - w_res = objtable.wrap_float(math.exp(f)) + w_res = utility.wrap_float(math.exp(f)) return w_res @@ -329,17 +308,14 @@ @expose_primitive(AT_PUT, unwrap_spec=[object, index1_0, object]) def func(interp, w_obj, n0, w_val): n0 = assert_valid_index(n0, w_obj) - try: - w_obj.atput0(n0, w_val) - return w_val - except model.UnwrappingError: - raise PrimitiveFailedError() + w_obj.atput0(n0, w_val) + return w_val @expose_primitive(SIZE, unwrap_spec=[object]) def func(interp, w_obj): if not w_obj.shadow_of_my_class().isvariable(): raise PrimitiveFailedError() - return objtable.wrap_int(w_obj.varsize()) + return utility.wrap_int(w_obj.varsize()) @expose_primitive(STRING_AT, unwrap_spec=[object, index1_0]) def func(interp, w_obj, n0): @@ -348,11 +324,11 @@ # make sure that getbyte is only performed on W_BytesObjects if not isinstance(w_obj, model.W_BytesObject): raise PrimitiveFailedError - return objtable.wrap_char(w_obj.getchar(n0)) + return utility.wrap_char(w_obj.getchar(n0)) @expose_primitive(STRING_AT_PUT, unwrap_spec=[object, index1_0, object]) def func(interp, w_obj, n0, w_val): - val = unwrap_char(w_val) + val = utility.unwrap_char(w_val) n0 = assert_valid_index(n0, w_obj) w_obj.setchar(n0, val) return w_val @@ -402,12 +378,11 @@ raise PrimitiveFailedError() return shadow.new() - at expose_primitive(NEW_WITH_ARG, unwrap_spec=[object, object]) -def func(interp, w_cls, w_size): + at expose_primitive(NEW_WITH_ARG, unwrap_spec=[object, int]) +def func(interp, w_cls, size): shadow = w_cls.as_class_get_shadow() if not shadow.isvariable(): raise PrimitiveFailedError() - size = unwrap_int(w_size) return shadow.new(size) @expose_primitive(ARRAY_BECOME_ONE_WAY, unwrap_spec=[object, object]) @@ -437,7 +412,7 @@ def func(interp, w_rcvr): if isinstance(w_rcvr, model.W_SmallInteger): raise PrimitiveFailedError() - return wrap_int(w_rcvr.gethash()) + return utility.wrap_int(w_rcvr.gethash()) @expose_primitive(STORE_STACKP, unwrap_spec=[object, object]) def func(interp, w_obj1, w_obj2): @@ -460,7 +435,8 @@ @expose_primitive(NEW_METHOD, unwrap_spec=[object, int, object]) def func(interp, w_class, bytecount, w_header): - header = unwrap_int(w_header) + # XXX untested + header = utility.unwrap_int(w_header) literalcount = ((header >> 10) & 255) + 1 w_method = w_class.as_class_get_shadow().new(literalcount) # XXX not sure this is correct @@ -483,7 +459,7 @@ @expose_primitive(EQUIVALENT, unwrap_spec=[object, object]) def func(interp, w_arg, w_rcvr): - return objtable.wrap_bool(w_arg.equals(w_rcvr)) + return utility.wrap_bool(w_arg.equals(w_rcvr)) @expose_primitive(CLASS, unwrap_spec=[object]) def func(interp, w_obj): @@ -535,7 +511,7 @@ INC_GC = 131 def fake_bytes_left(): - return wrap_int(2**20) # XXX we don't know how to do this :-( + return utility.wrap_int(2**20) # XXX we don't know how to do this :-( @expose_primitive(INC_GC, unwrap_spec=[object]) @expose_primitive(FULL_GC, unwrap_spec=[object]) @@ -553,13 +529,13 @@ def func(interp, w_arg): import time import math - return wrap_int(int(math.fmod(time.time()*1000, constants.TAGGED_MAXINT/2))) + return utility.wrap_int(int(math.fmod(time.time()*1000, constants.TAGGED_MAXINT/2))) @expose_primitive(SECONDS_CLOCK, unwrap_spec=[object]) def func(interp, w_arg): import time - return wrap_int(0x23910d6c) # HACK: too big for a small int! - #return wrap_int(int(time.time())) + return utility.wrap_int(0x23910d6c) # HACK: too big for a small int! + #return utility.wrap_int(int(time.time())) # ___________________________________________________________________________ # Boolean Primitives @@ -591,7 +567,7 @@ @expose_primitive(code, unwrap_spec=[int, int]) def func(interp, v1, v2): res = op(v1, v2) - w_res = objtable.wrap_bool(res) + w_res = utility.wrap_bool(res) return w_res make_func(op) @@ -600,7 +576,7 @@ @expose_primitive(code+_FLOAT_OFFSET, unwrap_spec=[float, float]) def func(interp, v1, v2): res = op(v1, v2) - w_res = objtable.wrap_bool(res) + w_res = utility.wrap_bool(res) return w_res make_func(op) @@ -650,10 +626,9 @@ PRIMITIVE_SUSPEND = 88 PRIMITIVE_FLUSH_CACHE = 89 - at expose_primitive(PRIMITIVE_BLOCK_COPY, unwrap_spec=[object, object]) -def func(interp, w_context, w_argcnt): + at expose_primitive(PRIMITIVE_BLOCK_COPY, unwrap_spec=[object, int]) +def func(interp, w_context, argcnt): frame = interp.w_active_context - argcnt = unwrap_int(w_argcnt) # From B.B.: If receiver is a MethodContext, then it becomes # the new BlockContext's home context. Otherwise, the home Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Mon Oct 29 16:43:04 2007 @@ -1,5 +1,5 @@ import weakref -from pypy.lang.smalltalk import model, constants +from pypy.lang.smalltalk import model, constants, utility, error class AbstractShadow(object): """A shadow is an optional extra bit of information that @@ -17,12 +17,11 @@ WEAK_POINTERS = 3 COMPILED_METHOD = 4 -unwrap_int = model.unwrap_int -class MethodNotFound(Exception): +class MethodNotFound(error.SmalltalkException): pass -class ClassShadowError(Exception): +class ClassShadowError(error.SmalltalkException): pass class ClassShadow(AbstractShadow): @@ -49,7 +48,8 @@ w_self = self.w_self # read and painfully decode the format - classformat = unwrap_int(w_self.fetch(constants.CLASS_FORMAT_INDEX)) + classformat = utility.unwrap_int( + w_self.fetch(constants.CLASS_FORMAT_INDEX)) # The classformat in Squeak, as an integer value, is: # <2 bits=instSize//64><5 bits=cClass><4 bits=instSpec> # <6 bits=instSize\\64><1 bit=0> Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Mon Oct 29 16:43:04 2007 @@ -1,7 +1,7 @@ import py import os from pypy.lang.smalltalk import model -from pypy.lang.smalltalk import objtable +from pypy.lang.smalltalk import objtable, utility from pypy.rlib import objectmodel from pypy.lang.smalltalk.tool.bitmanipulation import splitter @@ -225,7 +225,7 @@ self.owner = reader self.value = value self.size = -1 - self.w_object = objtable.wrap_int(value) + self.w_object = utility.wrap_int(value) def initialize(self, chunk, reader): self.owner = reader Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Mon Oct 29 16:43:04 2007 @@ -1,7 +1,8 @@ import py from pypy.lang.smalltalk import model, interpreter, primitives, shadow -from pypy.lang.smalltalk import objtable, classtable -from pypy.lang.smalltalk.objtable import wrap_int, wrap_char +from pypy.lang.smalltalk import objtable, classtable, utility +from pypy.lang.smalltalk.utility import wrap_int, wrap_char, wrap_string, \ + unwrap_int mockclass = classtable.bootstrap_class @@ -45,7 +46,7 @@ try: return _cache[s] except KeyError: - result = _cache[s] = objtable.wrap_string(s) + result = _cache[s] = wrap_string(s) return result def fakeliterals(*literals): @@ -209,7 +210,7 @@ def test_pushConstantMinusOneBytecode(): interp = new_interpreter(pushConstantMinusOneBytecode) interp.step() - assert interp.w_active_context.pop() == interp.MONE + assert interp.w_active_context.pop() == interp.MINUS_ONE assert interp.w_active_context.stack == [] def test_pushConstantZeroBytecode(): @@ -403,7 +404,7 @@ interp.w_active_context.push(w_object) interp.w_active_context.push(wrap_int(8)) result = interp.interpret() - assert primitives.unwrap_int(result) == 34 + assert unwrap_int(result) == 34 def test_send_to_primitive(): @@ -417,7 +418,7 @@ assert interp.w_active_context is callerContext assert len(interp.w_active_context.stack) == 1 w_result = interp.w_active_context.pop() - assert primitives.unwrap_int(w_result) == 42 + assert unwrap_int(w_result) == 42 run_with_faked_methods( [[classtable.w_SmallInteger, primitives.SUBTRACT, @@ -720,10 +721,10 @@ w_fakeinst.store(0, wrap_char("a")) # static slot 0: instance variable w_fakeinst.store(1, wrap_char("b")) # varying slot 1 def test(): - assert objtable.ord_w_char(interpret_bc( + assert utility.unwrap_char(interpret_bc( [112, 118, 192, 124], fakeliterals(), - receiver=w_fakeinst)) == ord("b") + receiver=w_fakeinst)) == "b" run_with_faked_methods( [[w_fakeclass, primitives.AT, 1, "at:"]], test) @@ -735,12 +736,12 @@ w_fakeinst.store(0, wrap_char("a")) # static slot 0: instance variable w_fakeinst.store(1, wrap_char("a")) # varying slot 1 def test(): - assert objtable.ord_w_char(interpret_bc( + assert utility.unwrap_char(interpret_bc( [0x70, 0x76, 0x20, 0xc1, 0x7c], fakeliterals(wrap_char("b")), - receiver=w_fakeinst)) == ord("b") - assert objtable.ord_w_char(w_fakeinst.fetch(0)) == ord("a") - assert objtable.ord_w_char(w_fakeinst.fetch(1)) == ord("b") + receiver=w_fakeinst)) == "b" + assert utility.unwrap_char(w_fakeinst.fetch(0)) == "a" + assert utility.unwrap_char(w_fakeinst.fetch(1)) == "b" run_with_faked_methods( [[w_fakeclass, primitives.AT_PUT, 2, "at:put:"]], test) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Mon Oct 29 16:43:04 2007 @@ -2,10 +2,8 @@ import math from pypy.lang.smalltalk.primitives import prim_table, PrimitiveFailedError from pypy.lang.smalltalk import model, shadow -from pypy.lang.smalltalk import interpreter -from pypy.lang.smalltalk import classtable -from pypy.lang.smalltalk import objtable -from pypy.lang.smalltalk import constants +from pypy.lang.smalltalk import interpreter, utility +from pypy.lang.smalltalk import classtable, objtable, constants from pypy.rlib.rarithmetic import INFINITY, NAN, isinf, isnan from pypy.lang.smalltalk import primitives @@ -16,11 +14,11 @@ self.stack = stack def wrap(x): - if isinstance(x, int): return objtable.wrap_int(x) - if isinstance(x, float): return objtable.wrap_float(x) + if isinstance(x, int): return utility.wrap_int(x) + if isinstance(x, float): return utility.wrap_float(x) if isinstance(x, model.W_Object): return x - if isinstance(x, str) and len(x) == 1: return objtable.wrap_char(x) - if isinstance(x, str): return objtable.wrap_string(x) + if isinstance(x, str) and len(x) == 1: return utility.wrap_char(x) + if isinstance(x, str): return utility.wrap_string(x) raise NotImplementedError def mock(stack): Modified: pypy/dist/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Mon Oct 29 16:43:04 2007 @@ -1,5 +1,6 @@ import random from pypy.lang.smalltalk import model, shadow, classtable, constants, objtable +from pypy.lang.smalltalk import utility w_Object = classtable.classtable['w_Object'] w_Metaclass = classtable.classtable['w_Metaclass'] @@ -13,14 +14,14 @@ for i in range(size): w_array.store(i, objtable.w_nil) w_methoddict.store(constants.METHODDICT_NAMES_INDEX+i, objtable.w_nil) - w_tally = objtable.wrap_int(len(methods)) + w_tally = utility.wrap_int(len(methods)) w_methoddict.store(constants.METHODDICT_TALLY_INDEX, w_tally) w_methoddict.store(constants.METHODDICT_VALUES_INDEX, w_array) positions = range(size) random.shuffle(positions) for selector, w_compiledmethod in methods.items(): pos = positions.pop() - w_selector = objtable.wrap_string(selector) + w_selector = utility.wrap_string(selector) w_methoddict.store(constants.METHODDICT_NAMES_INDEX+pos, w_selector) w_array.store(pos, w_compiledmethod) #print w_methoddict._vars @@ -37,9 +38,9 @@ w_class = model.W_PointersObject(w_classofclass, size) w_class.store(constants.CLASS_SUPERCLASS_INDEX, w_superclass) w_class.store(constants.CLASS_METHODDICT_INDEX, w_methoddict) - w_class.store(constants.CLASS_FORMAT_INDEX, objtable.wrap_int(format)) + w_class.store(constants.CLASS_FORMAT_INDEX, utility.wrap_int(format)) if name is not None: - w_class.store(constants.CLASS_NAME_INDEX, objtable.wrap_string(name)) + w_class.store(constants.CLASS_NAME_INDEX, utility.wrap_string(name)) return w_class def basicshape(name, format, kind, varsized, instsize): Added: pypy/dist/pypy/lang/smalltalk/utility.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/smalltalk/utility.py Mon Oct 29 16:43:04 2007 @@ -0,0 +1,68 @@ +from pypy.lang.smalltalk.error import UnwrappingError, WrappingError +from pypy.lang.smalltalk import model + + +# ____________________________________________________________ +# unwrapping utilities + +def unwrap_int(w_value): + if isinstance(w_value, model.W_SmallInteger): + return w_value.value + raise UnwrappingError("expected a W_SmallInteger, got %s" % (w_value,)) + +def unwrap_char(w_char): + from pypy.lang.smalltalk import classtable, objtable, constants + w_class = w_char.getclass() + if w_class is not classtable.w_Character: + raise UnwrappingError("expected character, got %s" % (w_class, )) + w_ord = w_char.fetch(constants.CHARACTER_VALUE_INDEX) + w_class = w_ord.getclass() + if w_class is not classtable.w_SmallInteger: + raise UnwrappingError("expected smallint from character, got %s" % (w_class, )) + + assert isinstance(w_ord, model.W_SmallInteger) + return chr(w_ord.value) + +# ____________________________________________________________ +# wrapping utilities + +def wrap_int(i): + from pypy.lang.smalltalk import constants + if i <= constants.TAGGED_MAXINT and i >= constants.TAGGED_MININT: + return model.W_SmallInteger(i) + raise WrappingError("integer too large to fit into a tagged pointer") + +def wrap_float(i): + return model.W_Float(i) + +def wrap_string(string): + from pypy.lang.smalltalk import classtable + w_inst = classtable.w_String.as_class_get_shadow().new(len(string)) + for i in range(len(string)): + w_inst.setchar(i, string[i]) + return w_inst + +def wrap_char(c): + from pypy.lang.smalltalk.objtable import CharacterTable + return CharacterTable[ord(c)] + +def wrap_bool(bool): + from pypy.lang.smalltalk import objtable + if bool: + return objtable.w_true + else: + return objtable.w_false + +def wrap_list(lst_w_obj): + from pypy.lang.smalltalk import classtable + """ + Converts a Python list of wrapper objects into + a wrapped smalltalk array + """ + lstlen = len(lit) + res = classtable.w_Array.as_class_get_shadow().new(lstlen) + for i in range(lstlen): + res.storevarpointer(i, fakeliteral(lit[i])) + return res + + Modified: pypy/dist/pypy/translator/goal/targetfibsmalltalk.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetfibsmalltalk.py (original) +++ pypy/dist/pypy/translator/goal/targetfibsmalltalk.py Mon Oct 29 16:43:04 2007 @@ -1,6 +1,6 @@ from pypy.lang.smalltalk import model, interpreter, primitives, shadow from pypy.lang.smalltalk import objtable -from pypy.lang.smalltalk.objtable import wrap_int +from pypy.lang.smalltalk.utility import wrap_int, unwrap_int from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk.test.test_interpreter import * @@ -30,7 +30,7 @@ interp.w_active_context.push(w_object) interp.w_active_context.push(wrap_int(8)) result = interp.interpret() - assert primitives.unwrap_int(result) == 34 + assert unwrap_int(result) == 34 print "check_me() ok" check_me() @@ -45,7 +45,7 @@ interp.w_active_context.push(w_object) interp.w_active_context.push(wrap_int(n)) result = interp.interpret() - print primitives.unwrap_int(result) + print unwrap_int(result) return 0 # _____ Define and setup target ___ Modified: pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py (original) +++ pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py Mon Oct 29 16:43:04 2007 @@ -3,7 +3,6 @@ import os from pypy.lang.smalltalk import model, interpreter, primitives, shadow from pypy.lang.smalltalk import objtable -from pypy.lang.smalltalk.objtable import wrap_int from pypy.lang.smalltalk import classtable # from pypy.lang.smalltalk.test.test_interpreter import * from pypy.lang.smalltalk import squeakimage @@ -69,8 +68,8 @@ reader.initialize() image = squeakimage.SqueakImage() image.from_reader(reader) - interp = tinyBenchmarks(image) - run_benchmarks(interp) + #interp = tinyBenchmarks(image) + #run_benchmarks(interp) return 0 # _____ Define and setup target ___ From cfbolz at codespeak.net Mon Oct 29 16:47:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 29 Oct 2007 16:47:22 +0100 (CET) Subject: [pypy-svn] r48161 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071029154722.338CF817B@code0.codespeak.net> Author: cfbolz Date: Mon Oct 29 16:47:21 2007 New Revision: 48161 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/utility.py Log: move unwrap_float, kill unused w_subscript Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Mon Oct 29 16:47:21 2007 @@ -9,24 +9,6 @@ PrimitiveNotYetWrittenError from pypy.rlib import rarithmetic -def unwrap_float(w_v): - if isinstance(w_v, model.W_Float): return w_v.value - elif isinstance(w_v, model.W_SmallInteger): return float(w_v.value) - raise PrimitiveFailedError() - -def w_subscript(w_obj, idx): - """ - Rather cryptically named function which retrieves an indexed field - from the object, wrapping as necessary depending on the format of - the object so that the result can be returned. - """ - if isinstance(w_obj, model.W_PointersObject): - return w_obj.fetch(idx) - elif isinstance(w_obj, model.W_WordsObject): - return utility.wrap_int(w_obj.getword(idx)) - elif isinstance(w_obj, model.W_BytesObject): - return utility.wrap_int(w_obj.getchar(idx)) - raise PrimitiveFailedError() def assert_bounds(n0, minimum, maximum): if not minimum <= n0 < maximum: @@ -105,7 +87,7 @@ elif spec is index1_0: args += (utility.unwrap_int(w_arg)-1, ) elif spec is float: - args += (unwrap_float(w_arg), ) + args += (utility.unwrap_float(w_arg), ) elif spec is object: args += (w_arg, ) elif spec is str: Modified: pypy/dist/pypy/lang/smalltalk/utility.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/utility.py (original) +++ pypy/dist/pypy/lang/smalltalk/utility.py Mon Oct 29 16:47:21 2007 @@ -23,6 +23,12 @@ assert isinstance(w_ord, model.W_SmallInteger) return chr(w_ord.value) +def unwrap_float(w_v): + from pypy.lang.smalltalk import model + if isinstance(w_v, model.W_Float): return w_v.value + elif isinstance(w_v, model.W_SmallInteger): return float(w_v.value) + raise PrimitiveFailedError() + # ____________________________________________________________ # wrapping utilities From cfbolz at codespeak.net Mon Oct 29 16:52:37 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 29 Oct 2007 16:52:37 +0100 (CET) Subject: [pypy-svn] r48162 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071029155237.84D6C8177@code0.codespeak.net> Author: cfbolz Date: Mon Oct 29 16:52:37 2007 New Revision: 48162 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py Log: shadow --> s_class in some functions to not shadow (pun not intended) the global module name Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Mon Oct 29 16:52:37 2007 @@ -355,17 +355,17 @@ @expose_primitive(NEW, unwrap_spec=[object]) def func(interp, w_cls): - shadow = w_cls.as_class_get_shadow() - if shadow.isvariable(): + s_class = w_cls.as_class_get_shadow() + if s_class.isvariable(): raise PrimitiveFailedError() - return shadow.new() + return s_class.new() @expose_primitive(NEW_WITH_ARG, unwrap_spec=[object, int]) def func(interp, w_cls, size): - shadow = w_cls.as_class_get_shadow() - if not shadow.isvariable(): + s_class = w_cls.as_class_get_shadow() + if not s_class.isvariable(): raise PrimitiveFailedError() - return shadow.new(size) + return s_class.new(size) @expose_primitive(ARRAY_BECOME_ONE_WAY, unwrap_spec=[object, object]) def func(interp, w_obj1, w_obj2): @@ -374,8 +374,8 @@ @expose_primitive(INST_VAR_AT, unwrap_spec=[object, index1_0]) def func(interp, w_rcvr, n0): "Fetches a fixed field from the object, and fails otherwise" - shadow = w_rcvr.shadow_of_my_class() - assert_bounds(n0, 0, shadow.instsize()) + s_class = w_rcvr.shadow_of_my_class() + assert_bounds(n0, 0, s_class.instsize()) # only pointers have non-0 size assert isinstance(w_rcvr, model.W_PointersObject) return w_rcvr.fetch(n0) @@ -383,8 +383,8 @@ @expose_primitive(INST_VAR_AT_PUT, unwrap_spec=[object, index1_0, object]) def func(interp, w_rcvr, n0, w_value): "Stores a value into a fixed field from the object, and fails otherwise" - shadow = w_rcvr.shadow_of_my_class() - assert_bounds(n0, 0, shadow.instsize()) + s_class = w_rcvr.shadow_of_my_class() + assert_bounds(n0, 0, s_class.instsize()) # only pointers have non-0 size assert isinstance(w_rcvr, model.W_PointersObject) w_rcvr.store(n0, w_value) From rxe at codespeak.net Mon Oct 29 17:04:56 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 29 Oct 2007 17:04:56 +0100 (CET) Subject: [pypy-svn] r48164 - in pypy/dist/pypy/translator/llvm: . module Message-ID: <20071029160456.2E9E9817B@code0.codespeak.net> Author: rxe Date: Mon Oct 29 17:04:54 2007 New Revision: 48164 Modified: pypy/dist/pypy/translator/llvm/buildllvm.py pypy/dist/pypy/translator/llvm/exception.py pypy/dist/pypy/translator/llvm/module/excsupport.py pypy/dist/pypy/translator/llvm/module/support.py Log: attempt to handle void/bool return types Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Mon Oct 29 17:04:54 2007 @@ -44,7 +44,10 @@ result = %(name)s(*args) if %(name)s_raised(): raise LLVMException("Exception raised") - return result + if %(rt_isbool)s: + return bool(result) + else: + return result """ import ctypes @@ -59,15 +62,19 @@ lltype.Signed: "ctypes.c_int", lltype.Unsigned: "ctypes.c_uint", lltype.SignedLongLong: "ctypes.c_longlong", - lltype.UnsignedLongLong: "ctypes.c_ulonglong" + lltype.UnsignedLongLong: "ctypes.c_ulonglong", + lltype.Void: None # XXX why no ctypes.c_void ? } name = genllvm.entrynode.ref.strip("%") g = genllvm.entrynode.graph - returntype = TO_CTYPES[g.returnblock.inputargs[0].concretetype] + rt = g.returnblock.inputargs[0].concretetype + returntype = TO_CTYPES[rt] inputargtypes = [TO_CTYPES[a.concretetype] for a in g.startblock.inputargs] args = '[%s]' % ", ".join(inputargtypes) + + rt_isbool = rt is lltype.Bool modfilename.write(template % locals()) return modfilename.purebasename Modified: pypy/dist/pypy/translator/llvm/exception.py ============================================================================== --- pypy/dist/pypy/translator/llvm/exception.py (original) +++ pypy/dist/pypy/translator/llvm/exception.py Mon Oct 29 17:04:54 2007 @@ -1,3 +1,5 @@ +# XXX this is just so we can compile modules - need to put somewhere else + def _noresult(returntype): r = returntype.strip() @@ -13,8 +15,15 @@ def llvm_implcode(entrynode): from pypy.translator.llvm.codewriter import DEFAULT_CCONV as cconv - from pypy.translator.llvm.module.excsupport import exctransform_code + from pypy.translator.llvm.module.excsupport import entrycode, voidentrycode, raisedcode returntype, entrypointname = entrynode.getdecl().split('%', 1) noresult = _noresult(returntype) - return exctransform_code % locals() + + code = raisedcode % locals() + if returntype == "void": + code += voidentrycode % locals() + else: + code += entrycode % locals() + + return code Modified: pypy/dist/pypy/translator/llvm/module/excsupport.py ============================================================================== --- pypy/dist/pypy/translator/llvm/module/excsupport.py (original) +++ pypy/dist/pypy/translator/llvm/module/excsupport.py Mon Oct 29 17:04:54 2007 @@ -1,5 +1,5 @@ -exctransform_code = ''' +entrycode = ''' ccc %(returntype)s%%__entrypoint__%(entrypointname)s { store %%RPYTHON_EXCEPTION_VTABLE* null, %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type %%result = call %(cconv)s %(returntype)s%%%(entrypointname)s @@ -13,7 +13,25 @@ exception: ret %(noresult)s } +''' +voidentrycode = ''' +ccc %(returntype)s%%__entrypoint__%(entrypointname)s { + store %%RPYTHON_EXCEPTION_VTABLE* null, %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type + %%result = call %(cconv)s %(returntype)s%%%(entrypointname)s + %%tmp = load %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type + %%exc = seteq %%RPYTHON_EXCEPTION_VTABLE* %%tmp, null + br bool %%exc, label %%no_exception, label %%exception + +no_exception: + ret %(returntype)s %%result + +exception: + ret %(noresult)s +} +''' + +raisedcode = ''' ;XXX this should use the transformation data that has the same purpose ccc int %%__entrypoint__raised_LLVMException() { %%tmp = load %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type @@ -21,7 +39,4 @@ ret int %%result } -internal fastcc void %%unwind() { - ret void -} ''' Modified: pypy/dist/pypy/translator/llvm/module/support.py ============================================================================== --- pypy/dist/pypy/translator/llvm/module/support.py (original) +++ pypy/dist/pypy/translator/llvm/module/support.py Mon Oct 29 17:04:54 2007 @@ -117,7 +117,6 @@ %%exception_type = load %%RPYTHON_EXCEPTION_VTABLE** %%tmp store %%RPYTHON_EXCEPTION_VTABLE* %%exception_type, %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type store %%RPYTHON_EXCEPTION* %%exception_value, %%RPYTHON_EXCEPTION** %%last_exception_value - call fastcc void %%unwind() ret void } """ % (c_name, exc_repr) From santagada at codespeak.net Mon Oct 29 17:10:05 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 29 Oct 2007 17:10:05 +0100 (CET) Subject: [pypy-svn] r48166 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071029161005.20E8C8192@code0.codespeak.net> Author: santagada Date: Mon Oct 29 17:10:04 2007 New Revision: 48166 Added: pypy/dist/pypy/translator/llvm/test/test_runtest.py Modified: pypy/dist/pypy/translator/llvm/test/runtest.py pypy/dist/pypy/translator/llvm/test/test_rlist.py Log: Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Mon Oct 29 17:10:04 2007 @@ -9,6 +9,7 @@ optimize_tests = False MINIMUM_LLVM_VERSION = 1.9 +FLOAT_PRECISION = 8 ext_modules = [] Modified: pypy/dist/pypy/translator/llvm/test/test_rlist.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rlist.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rlist.py Mon Oct 29 17:10:04 2007 @@ -1,6 +1,6 @@ import py from pypy.rpython.test.test_rlist import BaseTestRlist -from pypy.translator.llvm.test.runtest import LLVMTest +from pypy.translator.llvm.test.runtest import * # ====> ../../../rpython/test/test_rlist.py Added: pypy/dist/pypy/translator/llvm/test/test_runtest.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_runtest.py Mon Oct 29 17:10:04 2007 @@ -0,0 +1,10 @@ +import py +from pypy.translator.oosupport.test_template.runtest import BaseTestRunTest +from pypy.translator.llvm.test.runtest import * + +class TestRunTest(BaseTestRunTest, LLVMTest): + def test_none(self): + def fn(x): + y = 1 + x + return None + assert self.interpret(fn,[1]) == None From rxe at codespeak.net Mon Oct 29 17:13:47 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 29 Oct 2007 17:13:47 +0100 (CET) Subject: [pypy-svn] r48167 - in pypy/dist/pypy/translator/llvm: . module test Message-ID: <20071029161347.382EE817F@code0.codespeak.net> Author: rxe Date: Mon Oct 29 17:13:46 2007 New Revision: 48167 Modified: pypy/dist/pypy/translator/llvm/exception.py pypy/dist/pypy/translator/llvm/module/excsupport.py pypy/dist/pypy/translator/llvm/test/test_lltype.py Log: maybe a better attemptn Modified: pypy/dist/pypy/translator/llvm/exception.py ============================================================================== --- pypy/dist/pypy/translator/llvm/exception.py (original) +++ pypy/dist/pypy/translator/llvm/exception.py Mon Oct 29 17:13:46 2007 @@ -20,7 +20,7 @@ noresult = _noresult(returntype) code = raisedcode % locals() - if returntype == "void": + if returntype == "void ": code += voidentrycode % locals() else: code += entrycode % locals() Modified: pypy/dist/pypy/translator/llvm/module/excsupport.py ============================================================================== --- pypy/dist/pypy/translator/llvm/module/excsupport.py (original) +++ pypy/dist/pypy/translator/llvm/module/excsupport.py Mon Oct 29 17:13:46 2007 @@ -18,16 +18,8 @@ voidentrycode = ''' ccc %(returntype)s%%__entrypoint__%(entrypointname)s { store %%RPYTHON_EXCEPTION_VTABLE* null, %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type - %%result = call %(cconv)s %(returntype)s%%%(entrypointname)s - %%tmp = load %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type - %%exc = seteq %%RPYTHON_EXCEPTION_VTABLE* %%tmp, null - br bool %%exc, label %%no_exception, label %%exception - -no_exception: - ret %(returntype)s %%result - -exception: - ret %(noresult)s + call %(cconv)s %(returntype)s%%%(entrypointname)s + ret void } ''' Modified: pypy/dist/pypy/translator/llvm/test/test_lltype.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lltype.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lltype.py Mon Oct 29 17:13:46 2007 @@ -748,6 +748,17 @@ assert fn(0) == 0 assert fn(42) == 42 * 2**32 +def test_rettypes(): + ' test returning bool and void types ' + def llf(): + return + fn = compile_function(llf, []) + assert fn() is None + def llf(): + return not(False) + fn = compile_function(llf, []) + assert fn() is True + class TestLowLevelType(object): def getcompiled(self, f, args=[]): return compile_function(f, args) From cfbolz at codespeak.net Mon Oct 29 18:16:17 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 29 Oct 2007 18:16:17 +0100 (CET) Subject: [pypy-svn] r48168 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071029171617.436B28174@code0.codespeak.net> Author: cfbolz Date: Mon Oct 29 18:16:15 2007 New Revision: 48168 Modified: pypy/dist/pypy/lang/smalltalk/model.py Log: fix default argument Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Mon Oct 29 18:16:15 2007 @@ -281,7 +281,7 @@ The trailer has two variant formats. In the first variant, the last byte is at least 252 and the last four bytes represent a source pointer into one of the sources files (see #sourcePointer). In the second variant, the last byte is less than 252, and the last several bytes are a compressed version of the names of the method's temporary variables. The number of bytes used for this purpose is the value of the last byte in the method. """ - def __init__(self, literalsize, bytes=[], argsize=0, + def __init__(self, literalsize, bytes="", argsize=0, tempsize=0, primitive=0, w_compiledin=None): self.literals = [None] * literalsize self.w_compiledin = w_compiledin From cfbolz at codespeak.net Mon Oct 29 18:32:29 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 29 Oct 2007 18:32:29 +0100 (CET) Subject: [pypy-svn] r48169 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071029173229.EA9498179@code0.codespeak.net> Author: cfbolz Date: Mon Oct 29 18:32:29 2007 New Revision: 48169 Modified: pypy/dist/pypy/lang/smalltalk/utility.py Log: raise correct error Modified: pypy/dist/pypy/lang/smalltalk/utility.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/utility.py (original) +++ pypy/dist/pypy/lang/smalltalk/utility.py Mon Oct 29 18:32:29 2007 @@ -27,7 +27,7 @@ from pypy.lang.smalltalk import model if isinstance(w_v, model.W_Float): return w_v.value elif isinstance(w_v, model.W_SmallInteger): return float(w_v.value) - raise PrimitiveFailedError() + raise UnwrappingError() # ____________________________________________________________ # wrapping utilities From rxe at codespeak.net Mon Oct 29 22:59:41 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 29 Oct 2007 22:59:41 +0100 (CET) Subject: [pypy-svn] r48182 - in pypy/dist/pypy/translator/llvm: . module Message-ID: <20071029215941.80B12817B@code0.codespeak.net> Author: rxe Date: Mon Oct 29 22:59:38 2007 New Revision: 48182 Added: pypy/dist/pypy/translator/llvm/modwrapper.py - copied, changed from r48167, pypy/dist/pypy/translator/llvm/exception.py Removed: pypy/dist/pypy/translator/llvm/exception.py Modified: pypy/dist/pypy/translator/llvm/buildllvm.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/module/excsupport.py Log: whack until llvm backend returns more rich types. now passing 48/57 test_rlist. Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Mon Oct 29 22:59:38 2007 @@ -3,80 +3,9 @@ import py -from pypy.translator.llvm.log import log from pypy.translator.tool import stdoutcapture - -def write_ctypes_module(genllvm, dllname): - """ use ctypes to create a temporary module """ - - template = """ -import ctypes -from os.path import join, dirname, realpath -_c = ctypes.CDLL(join(dirname(realpath(__file__)), "%(dllname)s")) - -_setup = False - -class LLVMException(Exception): - pass - -%(name)s = _c.__entrypoint__%(name)s -%(name)s.argtypes = %(args)s -%(name)s.restype = %(returntype)s - -%(name)s_raised = _c.__entrypoint__raised_LLVMException -%(name)s_raised.argtypes = [] -%(name)s_raised.restype = ctypes.c_int - -GC_get_heap_size_wrapper = _c.GC_get_heap_size -GC_get_heap_size_wrapper.argtypes = [] -GC_get_heap_size_wrapper.restype = ctypes.c_int - -startup_code = _c.ctypes_RPython_StartupCode -startup_code.argtypes = [] -startup_code.restype = ctypes.c_int - -def %(name)s_wrapper(*args): - global _setup - if not _setup: - if not startup_code(): - raise LLVMException("Failed to startup") - _setup = True - result = %(name)s(*args) - if %(name)s_raised(): - raise LLVMException("Exception raised") - if %(rt_isbool)s: - return bool(result) - else: - return result -""" - - import ctypes - from pypy.rpython.lltypesystem import lltype - - basename = genllvm.filename.purebasename + '_wrapper.py' - modfilename = genllvm.filename.new(basename = basename) - - TO_CTYPES = {lltype.Bool: "ctypes.c_int", - lltype.Float: "ctypes.c_double", - lltype.Char: "ctypes.c_char", - lltype.Signed: "ctypes.c_int", - lltype.Unsigned: "ctypes.c_uint", - lltype.SignedLongLong: "ctypes.c_longlong", - lltype.UnsignedLongLong: "ctypes.c_ulonglong", - lltype.Void: None # XXX why no ctypes.c_void ? - } - - name = genllvm.entrynode.ref.strip("%") - - g = genllvm.entrynode.graph - rt = g.returnblock.inputargs[0].concretetype - returntype = TO_CTYPES[rt] - inputargtypes = [TO_CTYPES[a.concretetype] for a in g.startblock.inputargs] - args = '[%s]' % ", ".join(inputargtypes) - - rt_isbool = rt is lltype.Bool - modfilename.write(template % locals()) - return modfilename.purebasename +from pypy.translator.llvm.log import log +from pypy.translator.llvm.modwrapper import write_ctypes_module def llvm_is_on_path(): if py.path.local.sysfind("llvm-as") is None or \ Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Mon Oct 29 22:59:38 2007 @@ -154,9 +154,10 @@ self._checkpoint('write support implentations') - # write exception implementaions - from pypy.translator.llvm.exception import llvm_implcode - codewriter.write_lines(llvm_implcode(self.entrynode)) + # write wrapper code + if not self.standalone: + from pypy.translator.llvm.modwrapper import llvm_implcode + codewriter.write_lines(llvm_implcode(self.entrynode)) # write all node implementations for node in self.db.getnodes(): Modified: pypy/dist/pypy/translator/llvm/module/excsupport.py ============================================================================== --- pypy/dist/pypy/translator/llvm/module/excsupport.py (original) +++ pypy/dist/pypy/translator/llvm/module/excsupport.py Mon Oct 29 22:59:38 2007 @@ -1,22 +1,14 @@ entrycode = ''' -ccc %(returntype)s%%__entrypoint__%(entrypointname)s { +ccc %(returntype)s %%__entrypoint__%(entrypointname)s { store %%RPYTHON_EXCEPTION_VTABLE* null, %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type %%result = call %(cconv)s %(returntype)s%%%(entrypointname)s - %%tmp = load %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type - %%exc = seteq %%RPYTHON_EXCEPTION_VTABLE* %%tmp, null - br bool %%exc, label %%no_exception, label %%exception - -no_exception: ret %(returntype)s %%result - -exception: - ret %(noresult)s } ''' voidentrycode = ''' -ccc %(returntype)s%%__entrypoint__%(entrypointname)s { +ccc %(returntype)s %%__entrypoint__%(entrypointname)s { store %%RPYTHON_EXCEPTION_VTABLE* null, %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type call %(cconv)s %(returntype)s%%%(entrypointname)s ret void From rxe at codespeak.net Tue Oct 30 00:35:30 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 30 Oct 2007 00:35:30 +0100 (CET) Subject: [pypy-svn] r48184 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071029233530.651FF816D@code0.codespeak.net> Author: rxe Date: Tue Oct 30 00:35:29 2007 New Revision: 48184 Modified: pypy/dist/pypy/translator/llvm/arraynode.py pypy/dist/pypy/translator/llvm/test/test_lltype.py Log: an epiphany: an array is an array Modified: pypy/dist/pypy/translator/llvm/arraynode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/arraynode.py (original) +++ pypy/dist/pypy/translator/llvm/arraynode.py Tue Oct 30 00:35:29 2007 @@ -50,7 +50,6 @@ ref = "cast(%s* %s to %s*)" % (self.get_typerepr(), ref, typeval) - print 'KKKKKKKKKK', ref return ref def get_pbcref(self, toptr): @@ -59,7 +58,6 @@ fromptr = "%s*" % self.get_typerepr() ref = "cast(%s %s to %s)" % (fromptr, self.name, toptr) - print 'XXXXXXXXX', ref return ref def get_childref(self, index): @@ -86,7 +84,7 @@ def get_typerepr(self): arraylen = self.get_arrayvalue()[0] typeval = self.db.repr_type(self.arraytype) - return "{ [%s x %s] }" % (arraylen, typeval) + return "[%s x %s]" % (arraylen, typeval) def get_childref(self, index): return "getelementptr(%s* %s, int 0, int %s)" %( @@ -97,12 +95,7 @@ def constantvalue(self): physicallen, arrayrepr = self.get_arrayvalue() typeval = self.db.repr_type(self.arraytype) - - value = "[%s x %s] %s" % (physicallen, - typeval, - arrayrepr) - - s = "%s {%s}" % (self.get_typerepr(), value) + s = "%s %s" % (self.get_typerepr(), arrayrepr) return s class StrArrayNode(ArrayNode): Modified: pypy/dist/pypy/translator/llvm/test/test_lltype.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lltype.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lltype.py Tue Oct 30 00:35:29 2007 @@ -374,7 +374,8 @@ def test_direct_arrayitems(): for a in [malloc(GcArray(Signed), 5), - malloc(FixedSizeArray(Signed, 5), immortal=True)]: + malloc(FixedSizeArray(Signed, 5), immortal=True), + malloc(Array(Signed, hints={'nolength': True}), 5, immortal=True)]: a[0] = 0 a[1] = 10 a[2] = 20 @@ -763,41 +764,6 @@ def getcompiled(self, f, args=[]): return compile_function(f, args) - def test_direct_arrayitems(self): - py.test.skip("nolength ???") - for a in [malloc(GcArray(Signed), 5), - malloc(FixedSizeArray(Signed, 5), immortal=True), - malloc(Array(Signed, hints={'nolength': True}), 5, immortal=True), - ]: - a[0] = 0 - a[1] = 10 - a[2] = 20 - a[3] = 30 - a[4] = 40 - b0 = direct_arrayitems(a) - b1 = direct_ptradd(b0, 1) - b2 = direct_ptradd(b1, 1) - def llf(n): - b0 = direct_arrayitems(a) - b3 = direct_ptradd(direct_ptradd(b0, 5), -2) - saved = a[n] - a[n] = 1000 - try: - return b0[0] + b3[-2] + b2[1] + b1[3] - finally: - a[n] = saved - fn = self.getcompiled(llf, [int]) - res = fn(0) - assert res == 1000 + 10 + 30 + 40 - res = fn(1) - assert res == 0 + 1000 + 30 + 40 - res = fn(2) - assert res == 0 + 10 + 30 + 40 - res = fn(3) - assert res == 0 + 10 + 1000 + 40 - res = fn(4) - assert res == 0 + 10 + 30 + 1000 - def test_arithmetic_cornercases(self): py.test.skip("pyobject in this test - but why ???") import operator, sys From tismer at codespeak.net Tue Oct 30 01:12:03 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Tue, 30 Oct 2007 01:12:03 +0100 (CET) Subject: [pypy-svn] r48185 - pypy/dist/pypy/translator/goal/test2 Message-ID: <20071030001203.3947C8181@code0.codespeak.net> Author: tismer Date: Tue Oct 30 01:12:01 2007 New Revision: 48185 Modified: pypy/dist/pypy/translator/goal/test2/test_app_main.py Log: three tests need to be re-designed for windows. close_fds is not supported on Windows platforms Modified: pypy/dist/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/dist/pypy/translator/goal/test2/test_app_main.py Tue Oct 30 01:12:01 2007 @@ -213,6 +213,8 @@ child.sendline('X') def test_options_i_m(self): + if sys.platform == "win32": + skip("close_fds is not supported on Windows platforms") p = os.path.join(autopath.this_dir, 'mymodule.py') p = os.path.abspath(p) child = self.spawn(['-i', @@ -234,6 +236,8 @@ child.expect('False') def test_options_u_i(self): + if sys.platform == "win32": + skip("close_fds is not supported on Windows platforms") import subprocess, select, os python = sys.executable pipe = subprocess.Popen([python, app_main, "-u", "-i"], @@ -247,6 +251,8 @@ assert data.startswith('Python') def test_options_u_PYTHONINSPECT(self): + if sys.platform == "win32": + skip("close_fds is not supported on Windows platforms") import subprocess, select, os python = sys.executable pipe = subprocess.Popen([python, app_main, "-u"], From santagada at codespeak.net Tue Oct 30 02:59:45 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Tue, 30 Oct 2007 02:59:45 +0100 (CET) Subject: [pypy-svn] r48186 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071030015945.6B21B8107@code0.codespeak.net> Author: santagada Date: Tue Oct 30 02:59:42 2007 New Revision: 48186 Added: pypy/dist/pypy/translator/llvm/test/test_rbool.py pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py pypy/dist/pypy/translator/llvm/test/test_rfloat.py pypy/dist/pypy/translator/llvm/test/test_rint.py pypy/dist/pypy/translator/llvm/test/test_rpbc.py pypy/dist/pypy/translator/llvm/test/test_rstr.py pypy/dist/pypy/translator/llvm/test/test_rtuple.py Modified: pypy/dist/pypy/translator/llvm/test/test_rlist.py Log: some more rpython tests, mostly passing Added: pypy/dist/pypy/translator/llvm/test/test_rbool.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_rbool.py Tue Oct 30 02:59:42 2007 @@ -0,0 +1,6 @@ +import py +from pypy.rpython.test.test_rbool import BaseTestRbool +from pypy.translator.llvm.test.runtest import * + +class TestLLVMBool(LLVMTest, BaseTestRbool): + pass Added: pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py Tue Oct 30 02:59:42 2007 @@ -0,0 +1,6 @@ +import py +from pypy.rpython.test.test_rbuiltin import BaseTestRbuiltin +from pypy.translator.llvm.test.runtest import * + +class TestLLVMBuiltin(LLVMTest, BaseTestRbuiltin): + pass Added: pypy/dist/pypy/translator/llvm/test/test_rfloat.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_rfloat.py Tue Oct 30 02:59:42 2007 @@ -0,0 +1,6 @@ +import py +from pypy.rpython.test.test_rfloat import BaseTestRfloat +from pypy.translator.llvm.test.runtest import * + +class TestLLVMFloat(LLVMTest, BaseTestRfloat): + pass Added: pypy/dist/pypy/translator/llvm/test/test_rint.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_rint.py Tue Oct 30 02:59:42 2007 @@ -0,0 +1,6 @@ +import py +from pypy.rpython.test.test_rint import BaseTestRint +from pypy.translator.llvm.test.runtest import * + +class TestLLVMInt(LLVMTest, BaseTestRint): + pass Modified: pypy/dist/pypy/translator/llvm/test/test_rlist.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rlist.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rlist.py Tue Oct 30 02:59:42 2007 @@ -2,7 +2,5 @@ from pypy.rpython.test.test_rlist import BaseTestRlist from pypy.translator.llvm.test.runtest import * -# ====> ../../../rpython/test/test_rlist.py - class TestLLVMList(LLVMTest, BaseTestRlist): pass Added: pypy/dist/pypy/translator/llvm/test/test_rpbc.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_rpbc.py Tue Oct 30 02:59:42 2007 @@ -0,0 +1,6 @@ +import py +from pypy.rpython.test.test_rpbc import BaseTestRPBC +from pypy.translator.llvm.test.runtest import * + +class TestLLVMPBC(LLVMTest, BaseTestRPBC): + pass Added: pypy/dist/pypy/translator/llvm/test/test_rstr.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_rstr.py Tue Oct 30 02:59:42 2007 @@ -0,0 +1,6 @@ +import py +from pypy.rpython.test.test_rstr import BaseTestRstr +from pypy.translator.llvm.test.runtest import * + +class TestLLVMStr(LLVMTest, BaseTestRstr): + pass Added: pypy/dist/pypy/translator/llvm/test/test_rtuple.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_rtuple.py Tue Oct 30 02:59:42 2007 @@ -0,0 +1,6 @@ +import py +from pypy.rpython.test.test_rint import BaseTestRint +from pypy.translator.llvm.test.runtest import * + +class TestLLVMInt(LLVMTest, BaseTestRint): + pass From cfbolz at codespeak.net Tue Oct 30 13:39:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 30 Oct 2007 13:39:55 +0100 (CET) Subject: [pypy-svn] r48194 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071030123955.2F7C781B0@code0.codespeak.net> Author: cfbolz Date: Tue Oct 30 13:39:53 2007 New Revision: 48194 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py Log: rename callPrimitiveAndPush to callPrimitive (the primitive is itself doing the pushing now). Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Tue Oct 30 13:39:53 2007 @@ -334,7 +334,7 @@ def longJumpIfFalse(self, interp): self.jumpConditional(interp.FALSE,self.longJumpPosition()) - def callPrimitiveAndPush(self, primitive, selector, + def callPrimitive(self, primitive, selector, argcount, interp): # XXX XXX REMEMBER TO SWITCH COMMENT AND ACTUAL CODE BEFORE # TESTING ON AN IMAGE, AND TO LEAVE AS IS FOR TESTCODE (and @@ -348,104 +348,104 @@ #self._sendSelfSelector(selector, argcount, interp) def bytecodePrimAdd(self, interp): - self.callPrimitiveAndPush(primitives.ADD, "+", 1, interp) + self.callPrimitive(primitives.ADD, "+", 1, interp) def bytecodePrimSubtract(self, interp): - self.callPrimitiveAndPush(primitives.SUBTRACT, "-", 1, interp) + self.callPrimitive(primitives.SUBTRACT, "-", 1, interp) def bytecodePrimLessThan(self, interp): - self.callPrimitiveAndPush(primitives.LESSTHAN, "<", 1, interp) + self.callPrimitive(primitives.LESSTHAN, "<", 1, interp) def bytecodePrimGreaterThan(self, interp): - self.callPrimitiveAndPush(primitives.GREATERTHAN, ">", 1, interp) + self.callPrimitive(primitives.GREATERTHAN, ">", 1, interp) def bytecodePrimLessOrEqual(self, interp): - self.callPrimitiveAndPush(primitives.LESSOREQUAL, "<=", 1, interp) + self.callPrimitive(primitives.LESSOREQUAL, "<=", 1, interp) def bytecodePrimGreaterOrEqual(self, interp): - self.callPrimitiveAndPush(primitives.GREATEROREQUAL, ">=", 1, interp) + self.callPrimitive(primitives.GREATEROREQUAL, ">=", 1, interp) def bytecodePrimEqual(self, interp): - self.callPrimitiveAndPush(primitives.EQUAL, "=", 1, interp) + self.callPrimitive(primitives.EQUAL, "=", 1, interp) def bytecodePrimNotEqual(self, interp): - self.callPrimitiveAndPush(primitives.NOTEQUAL, "~=", 1, interp) + self.callPrimitive(primitives.NOTEQUAL, "~=", 1, interp) def bytecodePrimMultiply(self, interp): - self.callPrimitiveAndPush(primitives.MULTIPLY, "*", 1, interp) + self.callPrimitive(primitives.MULTIPLY, "*", 1, interp) def bytecodePrimDivide(self, interp): - self.callPrimitiveAndPush(primitives.DIVIDE, "/", 1, interp) + self.callPrimitive(primitives.DIVIDE, "/", 1, interp) def bytecodePrimMod(self, interp): - self.callPrimitiveAndPush(primitives.MOD, "\\\\", 1, interp) + self.callPrimitive(primitives.MOD, "\\\\", 1, interp) def bytecodePrimMakePoint(self, interp): raise MissingBytecode("bytecodePrimMakePoint") def bytecodePrimBitShift(self, interp): - self.callPrimitiveAndPush(primitives.BIT_SHIFT, "bitShift:", 1, interp) + self.callPrimitive(primitives.BIT_SHIFT, "bitShift:", 1, interp) def bytecodePrimDiv(self, interp): - self.callPrimitiveAndPush(primitives.DIV, "//", 1, interp) + self.callPrimitive(primitives.DIV, "//", 1, interp) def bytecodePrimBitAnd(self, interp): - self.callPrimitiveAndPush(primitives.BIT_AND, "bitAnd:", 1, interp) + self.callPrimitive(primitives.BIT_AND, "bitAnd:", 1, interp) def bytecodePrimBitOr(self, interp): - self.callPrimitiveAndPush(primitives.BIT_OR, "bitOr:", 1, interp) + self.callPrimitive(primitives.BIT_OR, "bitOr:", 1, interp) def bytecodePrimAt(self, interp): # n.b.: depending on the type of the receiver, this may invoke # primitives.AT, primitives.STRING_AT, or something else for all # I know. - #self.callPrimitiveAndPush(primitives.AT, "at:", 1, interp) + #self.callPrimitive(primitives.AT, "at:", 1, interp) self._sendSelfSelector("at:", 1, interp) def bytecodePrimAtPut(self, interp): # n.b. as above - #self.callPrimitiveAndPush(primitives.AT_PUT, "at:put:", 2, interp) + #self.callPrimitive(primitives.AT_PUT, "at:put:", 2, interp) self._sendSelfSelector("at:put:", 2, interp) def bytecodePrimSize(self, interp): - self.callPrimitiveAndPush(primitives.SIZE, "size", 0, interp) + self.callPrimitive(primitives.SIZE, "size", 0, interp) def bytecodePrimNext(self, interp): - self.callPrimitiveAndPush(primitives.NEXT, "next", 0, interp) + self.callPrimitive(primitives.NEXT, "next", 0, interp) def bytecodePrimNextPut(self, interp): - self.callPrimitiveAndPush(primitives.NEXT_PUT, "nextPut:", 1, interp) + self.callPrimitive(primitives.NEXT_PUT, "nextPut:", 1, interp) def bytecodePrimAtEnd(self, interp): - self.callPrimitiveAndPush(primitives.AT_END, "atEnd", 0, interp) + self.callPrimitive(primitives.AT_END, "atEnd", 0, interp) def bytecodePrimEquivalent(self, interp): - self.callPrimitiveAndPush(primitives.EQUIVALENT, "==", 1, interp) + self.callPrimitive(primitives.EQUIVALENT, "==", 1, interp) def bytecodePrimClass(self, interp): - self.callPrimitiveAndPush( + self.callPrimitive( primitives.CLASS, "class", 0, interp) def bytecodePrimBlockCopy(self, interp): - self.callPrimitiveAndPush( + self.callPrimitive( primitives.PRIMITIVE_BLOCK_COPY, "blockCopy:", 1, interp) def bytecodePrimValue(self, interp): - self.callPrimitiveAndPush( + self.callPrimitive( primitives.PRIMITIVE_VALUE, "value", 0, interp) def bytecodePrimValueWithArg(self, interp): - self.callPrimitiveAndPush( + self.callPrimitive( primitives.PRIMITIVE_VALUE, "value:", 1, interp) def bytecodePrimDo(self, interp): self._sendSelfSelector("do:", 1, interp) def bytecodePrimNew(self, interp): - self.callPrimitiveAndPush(primitives.NEW, "new", 0, interp) + self.callPrimitive(primitives.NEW, "new", 0, interp) def bytecodePrimNewWithArg(self, interp): - self.callPrimitiveAndPush(primitives.NEW_WITH_ARG, "new:", 1, interp) + self.callPrimitive(primitives.NEW_WITH_ARG, "new:", 1, interp) def bytecodePrimPointX(self, interp): self._sendSelfSelector("x", 0, interp) From cfbolz at codespeak.net Tue Oct 30 13:54:24 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 30 Oct 2007 13:54:24 +0100 (CET) Subject: [pypy-svn] r48195 - pypy/dist/pypy/translator/goal Message-ID: <20071030125424.AF4A981AB@code0.codespeak.net> Author: cfbolz Date: Tue Oct 30 13:54:24 2007 New Revision: 48195 Modified: pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py Log: re-enable running of the benchmark Modified: pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py (original) +++ pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py Tue Oct 30 13:54:24 2007 @@ -68,8 +68,8 @@ reader.initialize() image = squeakimage.SqueakImage() image.from_reader(reader) - #interp = tinyBenchmarks(image) - #run_benchmarks(interp) + interp = tinyBenchmarks(image) + run_benchmarks(interp) return 0 # _____ Define and setup target ___ From arigo at codespeak.net Tue Oct 30 14:42:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 30 Oct 2007 14:42:12 +0100 (CET) Subject: [pypy-svn] r48196 - pypy/dist/pypy/translator/c/src Message-ID: <20071030134212.A704A81B8@code0.codespeak.net> Author: arigo Date: Tue Oct 30 14:42:11 2007 New Revision: 48196 Modified: pypy/dist/pypy/translator/c/src/asm_gcc_x86.h Log: Also use assembler for int_add_nonneg_ovf. Modified: pypy/dist/pypy/translator/c/src/asm_gcc_x86.h ============================================================================== --- pypy/dist/pypy/translator/c/src/asm_gcc_x86.h (original) +++ pypy/dist/pypy/translator/c/src/asm_gcc_x86.h Tue Oct 30 14:42:11 2007 @@ -14,6 +14,9 @@ : "0"(x), "g"(y) /* inputs */ \ : "cc", "memory") /* clobber */ +#undef OP_INT_ADD_NONNEG_OVF +#define OP_INT_ADD_NONNEG_OVF(x,y,r) OP_INT_ADD_OVF(x,y,r) + #undef OP_INT_SUB_OVF #define OP_INT_SUB_OVF(x,y,r) \ asm volatile("subl %2,%0\n\t" \ From arigo at codespeak.net Tue Oct 30 14:44:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 30 Oct 2007 14:44:00 +0100 (CET) Subject: [pypy-svn] r48197 - pypy/dist/pypy/rpython Message-ID: <20071030134400.98BCC81B9@code0.codespeak.net> Author: arigo Date: Tue Oct 30 14:44:00 2007 New Revision: 48197 Modified: pypy/dist/pypy/rpython/llinterp.py Log: flush() the tracer before entering pdb. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Tue Oct 30 14:44:00 2007 @@ -450,6 +450,8 @@ 'unexpected exception when calling') log.ERROR('the external function %r:' % (fptr,)) log.ERROR('%s: %s' % (e.__class__.__name__, e)) + if self.llinterpreter.tracer: + self.llinterpreter.tracer.flush() import sys from pypy.translator.tool.pdbplus import PdbPlusShow PdbPlusShow(None).post_mortem(sys.exc_info()[2]) From arigo at codespeak.net Tue Oct 30 14:46:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 30 Oct 2007 14:46:51 +0100 (CET) Subject: [pypy-svn] r48198 - in pypy/dist/pypy/jit: codegen/i386 codegen/llgraph codegen/test hintannotator hintannotator/test timeshifter timeshifter/test Message-ID: <20071030134651.54C8281B8@code0.codespeak.net> Author: arigo Date: Tue Oct 30 14:46:50 2007 New Revision: 48198 Modified: pypy/dist/pypy/jit/codegen/i386/operation.py pypy/dist/pypy/jit/codegen/llgraph/llimpl.py pypy/dist/pypy/jit/codegen/llgraph/rgenop.py pypy/dist/pypy/jit/codegen/test/rgenop_tests.py pypy/dist/pypy/jit/hintannotator/model.py pypy/dist/pypy/jit/hintannotator/test/test_annotator.py pypy/dist/pypy/jit/timeshifter/hrtyper.py pypy/dist/pypy/jit/timeshifter/rcontainer.py pypy/dist/pypy/jit/timeshifter/rtimeshift.py pypy/dist/pypy/jit/timeshifter/test/test_portal.py pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Log: Attempt to fix the JIT. This adds support for the new interior pointer operations (and for int_add_nonneg_ovf). Modified: pypy/dist/pypy/jit/codegen/i386/operation.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/operation.py (original) +++ pypy/dist/pypy/jit/codegen/i386/operation.py Tue Oct 30 14:46:50 2007 @@ -198,7 +198,7 @@ self.emit(allocator.mc, dstop, op2) class OpIntAdd(BinaryOp): - opname = 'int_add', 'uint_add', 'int_add_ovf' + opname = 'int_add', 'uint_add', 'int_add_ovf', 'int_add_nonneg_ovf' emit = staticmethod(I386CodeBuilder.ADD) commutative = True ccexcflag = Conditions['O'] Modified: pypy/dist/pypy/jit/codegen/llgraph/llimpl.py ============================================================================== --- pypy/dist/pypy/jit/codegen/llgraph/llimpl.py (original) +++ pypy/dist/pypy/jit/codegen/llgraph/llimpl.py Tue Oct 30 14:46:50 2007 @@ -244,6 +244,114 @@ return isinstance(c, flowmodel.Constant) +# ____________________________________________________________ +# Interior access helpers + +class InteriorPtrVariable(object): + def __init__(self, base_and_offsets_gv): + self.base_and_offsets_gv = base_and_offsets_gv + +def gengetsubstruct(block, gv_ptr, gv_PTRTYPE, gv_fieldname): + v_ptr = from_opaque_object(gv_ptr) + # don't generate any operation for an interior getsubstruct, + # but just return a special pseudo-variable + if isinstance(v_ptr, InteriorPtrVariable): + # a nested getsubstruct + v = InteriorPtrVariable(v_ptr.base_and_offsets_gv + [gv_fieldname]) + return to_opaque_object(v) + # in all other cases we need a proper cast + gv_ptr = cast(block, gv_PTRTYPE, gv_ptr) + PTRTYPE = from_opaque_object(gv_PTRTYPE).value + if PTRTYPE.TO._gckind == 'gc': + # reading from a GcStruct requires returning an interior pointer + # pseudo-variable + v = InteriorPtrVariable([gv_ptr, gv_fieldname]) + return to_opaque_object(v) + else: + vars_gv = [gv_ptr, gv_fieldname] + c_fieldname = from_opaque_object(gv_fieldname) + RESULTTYPE = lltype.Ptr(getattr(PTRTYPE.TO, c_fieldname.value)) + return genop(block, "getsubstruct", vars_gv, RESULTTYPE) + +def gengetarraysubstruct(block, gv_ptr, gv_index): + v_ptr = from_opaque_object(gv_ptr) + # don't generate any operation for an interior getarraysubstruct, + # but just return a special pseudo-variable + if isinstance(v_ptr, InteriorPtrVariable): + # a nested getarraysubstruct + v = InteriorPtrVariable(v_ptr.base_and_offsets_gv + [gv_index]) + return to_opaque_object(v) + PTRTYPE = v_ptr.concretetype + if PTRTYPE.TO._gckind == 'gc': + # reading from a GcArray requires returning an interior pointer + # pseudo-variable + v = InteriorPtrVariable([gv_ptr, gv_index]) + return to_opaque_object(v) + else: + vars_gv = [gv_ptr, gv_index] + RESULTTYPE = lltype.Ptr(PTRTYPE.TO.OF) + return genop(block, "getarraysubstruct", vars_gv, RESULTTYPE) + +def gensetfield(block, gv_ptr, gv_PTRTYPE, gv_fieldname, gv_value): + v_ptr = from_opaque_object(gv_ptr) + if isinstance(v_ptr, InteriorPtrVariable): + # this is really a setinteriorfield + vars_gv = v_ptr.base_and_offsets_gv + [gv_fieldname, gv_value] + genop(block, "setinteriorfield", vars_gv, lltype.Void) + else: + # for setfield we need a proper cast (for setinteriorfield, the + # top-level cast was already inserted by gengetsubstruct) + gv_ptr = cast(block, gv_PTRTYPE, gv_ptr) + vars_gv = [gv_ptr, gv_fieldname, gv_value] + genop(block, "setfield", vars_gv, lltype.Void) + +def gengetfield(block, gv_ptr, gv_PTRTYPE, gv_fieldname): + PTRTYPE = from_opaque_object(gv_PTRTYPE).value + c_fieldname = from_opaque_object(gv_fieldname) + RESULTTYPE = getattr(PTRTYPE.TO, c_fieldname.value) + v_ptr = from_opaque_object(gv_ptr) + if isinstance(v_ptr, InteriorPtrVariable): + # this is really a getinteriorfield + vars_gv = v_ptr.base_and_offsets_gv + [gv_fieldname] + return genop(block, "getinteriorfield", vars_gv, RESULTTYPE) + else: + # for getfield we need a proper cast (for getinteriorfield, the + # top-level cast was already inserted by gengetsubstruct) + gv_ptr = cast(block, gv_PTRTYPE, gv_ptr) + vars_gv = [gv_ptr, gv_fieldname] + return genop(block, "getfield", vars_gv, RESULTTYPE) + +def gensetarrayitem(block, gv_ptr, gv_index, gv_value): + v_ptr = from_opaque_object(gv_ptr) + if isinstance(v_ptr, InteriorPtrVariable): + # this is really a setinteriorfield + vars_gv = v_ptr.base_and_offsets_gv + [gv_index, gv_value] + genop(block, "setinteriorfield", vars_gv, lltype.Void) + else: + vars_gv = [gv_ptr, gv_index, gv_value] + genop(block, "setarrayitem", vars_gv, lltype.Void) + +def gengetarrayitem(block, gv_ITEMTYPE, gv_ptr, gv_index): + ITEMTYPE = from_opaque_object(gv_ITEMTYPE).value + v_ptr = from_opaque_object(gv_ptr) + if isinstance(v_ptr, InteriorPtrVariable): + # this is really a getinteriorfield + vars_gv = v_ptr.base_and_offsets_gv + [gv_index] + return genop(block, "getinteriorfield", vars_gv, ITEMTYPE) + else: + vars_gv = [gv_ptr, gv_index] + return genop(block, "getarrayitem", vars_gv, ITEMTYPE) + +def gengetarraysize(block, gv_ptr): + v_ptr = from_opaque_object(gv_ptr) + if isinstance(v_ptr, InteriorPtrVariable): + # this is really a getinteriorarraysize + vars_gv = v_ptr.base_and_offsets_gv + return genop(block, "getinteriorarraysize", vars_gv, lltype.Signed) + else: + vars_gv = [gv_ptr] + return genop(block, "getarraysize", vars_gv, lltype.Signed) + # XXX # temporary interface; it's unclear if genop itself should change to # ease dinstinguishing Void special args from the rest. Or there @@ -503,6 +611,8 @@ consttypeinfo = declareptrtype(flowmodel.Constant, "ConstOrVar") vartypeinfo = declareptrtype(flowmodel.Variable, "ConstOrVar") vartypeinfo.set_lltype(consttypeinfo.get_lltype()) # force same lltype +interiorptrvartypeinfo = declareptrtype(InteriorPtrVariable, "ConstOrVar") +interiorptrvartypeinfo.set_lltype(vartypeinfo.get_lltype()) # force same lltype linktypeinfo = declareptrtype(flowmodel.Link, "Link") graphtypeinfo = declareptrtype(flowmodel.FunctionGraph, "FunctionGraph") @@ -566,6 +676,13 @@ setannotation(geninputarg, s_ConstOrVar) setannotation(getinputarg, s_ConstOrVar) setannotation(genop, s_ConstOrVar) +setannotation(gengetsubstruct, s_ConstOrVar) +setannotation(gengetarraysubstruct, s_ConstOrVar) +setannotation(gensetfield, None) +setannotation(gengetfield, s_ConstOrVar) +setannotation(gensetarrayitem, None) +setannotation(gengetarrayitem, s_ConstOrVar) +setannotation(gengetarraysize, s_ConstOrVar) setannotation(end, None) setannotation(genconst, s_ConstOrVar) setannotation(genzeroconst, s_ConstOrVar) Modified: pypy/dist/pypy/jit/codegen/llgraph/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/llgraph/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/llgraph/rgenop.py Tue Oct 30 14:46:50 2007 @@ -159,53 +159,63 @@ def genop_getfield(self, (gv_name, gv_PTRTYPE, gv_FIELDTYPE), gv_ptr): debug_assert(self.rgenop.currently_writing is self, "genop_getfield: bad currently_writing") - vars_gv = [llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr.v), gv_name.v] - return LLVar(llimpl.genop(self.b, 'getfield', vars_gv, - gv_FIELDTYPE.v)) + return LLVar(llimpl.gengetfield(self.b, gv_ptr.v, + gv_PTRTYPE.v, gv_name.v)) + #vars_gv = [llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr.v), gv_name.v] + #return LLVar(llimpl.genop(self.b, 'getfield', vars_gv, + # gv_FIELDTYPE.v)) def genop_setfield(self, (gv_name, gv_PTRTYPE, gv_FIELDTYPE), gv_ptr, gv_value): debug_assert(self.rgenop.currently_writing is self, "genop_setfield: bad currently_writing") - vars_gv = [llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr.v), - gv_name.v, - llimpl.cast(self.b, gv_FIELDTYPE.v, gv_value.v)] - return LLVar(llimpl.genop(self.b, 'setfield', vars_gv, - gv_Void.v)) + v_value = llimpl.cast(self.b, gv_FIELDTYPE.v, gv_value.v) + llimpl.gensetfield(self.b, gv_ptr.v, gv_PTRTYPE.v, gv_name.v, v_value) + #vars_gv = [llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr.v), + # gv_name.v, + # llimpl.cast(self.b, gv_FIELDTYPE.v, gv_value.v)] + #return LLVar(llimpl.genop(self.b, 'setfield', vars_gv, + # gv_Void.v)) def genop_getsubstruct(self, (gv_name, gv_PTRTYPE, gv_FIELDTYPE), gv_ptr): debug_assert(self.rgenop.currently_writing is self, "genop_getsubstruct: bad currently_writing") - vars_gv = [llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr.v), gv_name.v] - return LLVar(llimpl.genop(self.b, 'getsubstruct', vars_gv, - gv_FIELDTYPE.v)) + return LLVar(llimpl.gengetsubstruct(self.b, gv_ptr.v, + gv_PTRTYPE.v, gv_name.v)) + #vars_gv = [llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr.v), gv_name.v] + #return LLVar(llimpl.genop(self.b, 'getsubstruct', vars_gv, + # gv_FIELDTYPE.v)) def genop_getarrayitem(self, gv_ITEMTYPE, gv_ptr, gv_index): debug_assert(self.rgenop.currently_writing is self, "genop_getarrayitem: bad currently_writing") - vars_gv = [gv_ptr.v, gv_index.v] - return LLVar(llimpl.genop(self.b, 'getarrayitem', vars_gv, - gv_ITEMTYPE.v)) + return LLVar(llimpl.gengetarrayitem(self.b, gv_ITEMTYPE.v, + gv_ptr.v, gv_index.v)) + #vars_gv = [gv_ptr.v, gv_index.v] + #return LLVar(llimpl.genop(self.b, 'getarrayitem', vars_gv, + # gv_ITEMTYPE.v)) def genop_getarraysubstruct(self, gv_ITEMTYPE, gv_ptr, gv_index): debug_assert(self.rgenop.currently_writing is self, "genop_getarraysubstruct: bad currently_writing") - vars_gv = [gv_ptr.v, gv_index.v] - return LLVar(llimpl.genop(self.b, 'getarraysubstruct', vars_gv, - gv_ITEMTYPE.v)) + return LLVar(llimpl.gengetarraysubstruct(self.b, gv_ptr.v, gv_index.v)) + #vars_gv = [gv_ptr.v, gv_index.v] + #return LLVar(llimpl.genop(self.b, 'getarraysubstruct', vars_gv, + # gv_ITEMTYPE.v)) def genop_setarrayitem(self, gv_ITEMTYPE, gv_ptr, gv_index, gv_value): debug_assert(self.rgenop.currently_writing is self, "genop_setarrayitem: bad currently_writing") - vars_gv = [gv_ptr.v, gv_index.v, gv_value.v] - return LLVar(llimpl.genop(self.b, 'setarrayitem', vars_gv, - gv_Void.v)) + llimpl.gensetarrayitem(self.b, gv_ptr.v, gv_index.v, gv_value.v) + #vars_gv = [gv_ptr.v, gv_index.v, gv_value.v] + #llimpl.genop(self.b, 'setarrayitem', vars_gv, gv_Void.v) def genop_getarraysize(self, gv_ITEMTYPE, gv_ptr): debug_assert(self.rgenop.currently_writing is self, "genop_getarraysize: bad currently_writing") - return LLVar(llimpl.genop(self.b, 'getarraysize', [gv_ptr.v], - gv_Signed.v)) + return LLVar(llimpl.gengetarraysize(self.b, gv_ptr.v)) + #return LLVar(llimpl.genop(self.b, 'getarraysize', [gv_ptr.v], + # gv_Signed.v)) def genop_malloc_fixedsize(self, (gv_TYPE, gv_PTRTYPE)): debug_assert(self.rgenop.currently_writing is self, Modified: pypy/dist/pypy/jit/codegen/test/rgenop_tests.py ============================================================================== --- pypy/dist/pypy/jit/codegen/test/rgenop_tests.py (original) +++ pypy/dist/pypy/jit/codegen/test/rgenop_tests.py Tue Oct 30 14:46:50 2007 @@ -2075,3 +2075,40 @@ x |= 1 result = fnptr(x) assert result == expected(x) + + def test_interior_access(self): + # for assembler backends, the 'interior' lloperations can be + # simply expressed as a sequence of genop_getsubstruct and + # genop_getarraysubstruct. So we put magic in the llgraph + # backend to recognize and rebuild the expected 'interior' + # lloperation in the llgraphs. + T = lltype.Struct('T', ('x', lltype.Signed)) + A = lltype.Array(T) + S = lltype.GcStruct('S', ('a', A)) + rgenop = self.RGenOp() + sigtoken = rgenop.sigToken(FUNC) + builder, gv_fn, [gv_x] = rgenop.newgraph(sigtoken, "interior_access") + builder.start_writing() + gv_s = builder.genop_malloc_varsize(rgenop.varsizeAllocToken(S), + rgenop.genconst(5)) + # generate an expanded 'setinteriorfield' + gv_a1 = builder.genop_getsubstruct(rgenop.fieldToken(S, 'a'), gv_s) + gv_t1 = builder.genop_getarraysubstruct(rgenop.arrayToken(A), + gv_a1, rgenop.genconst(3)) + builder.genop_setfield(rgenop.fieldToken(T, 'x'), gv_t1, gv_x) + # generate an expanded 'getinteriorfield' + gv_a1 = builder.genop_getsubstruct(rgenop.fieldToken(S, 'a'), gv_s) + gv_t1 = builder.genop_getarraysubstruct(rgenop.arrayToken(A), + gv_a1, rgenop.genconst(3)) + gv_y = builder.genop_getfield(rgenop.fieldToken(T, 'x'), gv_t1) + # generate an expanded 'getinteriorarraysize' + gv_a1 = builder.genop_getsubstruct(rgenop.fieldToken(S, 'a'), gv_s) + gv_z = builder.genop_getarraysize(rgenop.arrayToken(A), gv_a1) + # return + gv_result = builder.genop2("int_add", gv_y, gv_z) + builder.finish_and_return(sigtoken, gv_result) + builder.end() + + fnptr = self.cast(gv_fn, 1) + result = fnptr(42) + assert result == 47 Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Tue Oct 30 14:46:50 2007 @@ -4,6 +4,7 @@ from pypy.rpython.lltypesystem import lltype, lloperation UNARY_OPERATIONS = """same_as hint getfield setfield getsubstruct getarraysize + getinteriorfield getinteriorarraysize setinteriorfield cast_pointer direct_call indirect_call @@ -24,7 +25,7 @@ BINARY_OPERATIONS = """int_add int_sub int_mul int_mod int_and int_rshift int_lshift int_floordiv int_xor int_or int_add_ovf int_sub_ovf int_mul_ovf int_mod_ovf - int_floordiv_ovf int_lshift_ovf + int_floordiv_ovf int_lshift_ovf int_add_nonneg_ovf uint_add uint_sub uint_mul uint_mod uint_and uint_lshift uint_rshift uint_floordiv char_gt char_lt char_le char_ge char_eq char_ne @@ -346,6 +347,35 @@ FIELD_TYPE = getattr(S, hs_fieldname.const) return SomeLLAbstractVariable(lltype.Ptr(FIELD_TYPE), hs_v1.deepfrozen) + def _getinterior(hs_v1, *offsets_hs): + hs_container = hs_v1 + for hs_offset in offsets_hs: + if hs_offset.concretetype is lltype.Signed: + hs_container = pair(hs_container,hs_offset).getarraysubstruct() + else: + hs_container = hs_container.getsubstruct(hs_offset) + return hs_container + + def getinteriorfield(hs_v1, *offsets_hs): + hs_container = hs_v1._getinterior(*offsets_hs[:-1]) + hs_lastofs = offsets_hs[-1] + if hs_lastofs.concretetype is lltype.Signed: + return pair(hs_container, hs_lastofs).getarrayitem() + else: + return hs_container.getfield(hs_lastofs) + + def getinteriorarraysize(hs_v1, *offsets_hs): + return hs_v1._getinterior(*offsets_hs).getarraysize() + + def setinteriorfield(hs_v1, *offsets_and_val_hs): + hs_inner = hs_v1._getinterior(*offsets_and_val_hs[:-2]) + hs_lastofs = offsets_and_val_hs[-2] + hs_value = offsets_and_val_hs[-1] + if hs_lastofs.concretetype is lltype.Signed: + pair(hs_inner, hs_lastofs).setarrayitem(hs_value) + else: + hs_inner.setfield(hs_lastofs, hs_value) + def cast_pointer(hs_v1): RESTYPE = getbookkeeper().current_op_concretetype() return SomeLLAbstractVariable(RESTYPE, hs_v1.deepfrozen) Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Tue Oct 30 14:46:50 2007 @@ -144,7 +144,9 @@ res = hint(res, variable=True) return res - hs = hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) + # must backendoptimize to remove the mallocs related to the interior ptrs + hs = hannotate(ll_function, [int, int], policy=P_NOVIRTUAL, + backendoptimize=True) assert hs.concretetype == lltype.Signed @@ -428,6 +430,7 @@ assert hs1.contentdef.degenerated def test_degenerated_merge_cross_substructure(): + py.test.skip("no longer a valid test") from pypy.rlib import objectmodel S = lltype.Struct('S', ('n', lltype.Signed)) T = lltype.GcStruct('T', ('s', S), ('s1', S), ('n', lltype.Float)) Modified: pypy/dist/pypy/jit/timeshifter/hrtyper.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/hrtyper.py (original) +++ pypy/dist/pypy/jit/timeshifter/hrtyper.py Tue Oct 30 14:46:50 2007 @@ -880,6 +880,100 @@ ts.s_RedBox) + def _getinteriordesc(self, hop, PTRTYPE, nb_offsets): + path = [] + CONTAINER = PTRTYPE.TO + indices_v = [] + for i in range(1, 1 + nb_offsets): + T = originalconcretetype(hop.args_s[i]) + if T is lltype.Void: + fieldname = hop.args_v[i].value + CONTAINER = getattr(CONTAINER, fieldname) + path.append(fieldname) + else: + assert T is lltype.Signed + CONTAINER = CONTAINER.OF + path.append(None) # placeholder for 'array index' + v_index = hop.inputarg(self.getredrepr(lltype.Signed), arg=i) + indices_v.append(v_index) + if CONTAINER is lltype.Void: # Void field + return None, None + else: + return (rcontainer.InteriorDesc(self, PTRTYPE.TO, tuple(path)), + indices_v) + + def translate_op_getinteriorfield(self, hop): + ts = self + # no virtualizable access read here + PTRTYPE = originalconcretetype(hop.args_s[0]) + assert not PTRTYPE.TO._hints.get('virtualizable', False) + + # non virtual case + interiordesc, indices_v = self._getinteriordesc(hop, PTRTYPE, + hop.nb_args - 1) + if interiordesc is None: # Void field + return None + v_argbox = hop.inputarg(self.getredrepr(PTRTYPE), arg=0) + v_argbox = hop.llops.as_ptrredbox(v_argbox) + c_deepfrozen = inputconst(lltype.Bool, hop.args_s[0].deepfrozen) + c_interiordesc = inputconst(lltype.Void, interiordesc) + s_interiordesc = ts.rtyper.annotator.bookkeeper.immutablevalue( + interiordesc) + v_jitstate = hop.llops.getjitstate() + return hop.llops.genmixlevelhelpercall( + rtimeshift.ll_gengetinteriorfield, + [ts.s_JITState, annmodel.s_Bool, s_interiordesc, ts.s_PtrRedBox] + + [ts.s_RedBox]*len(indices_v), + [v_jitstate , c_deepfrozen , c_interiordesc, v_argbox ] + + indices_v, + ts.s_RedBox) + + def translate_op_setinteriorfield(self, hop): + ts = self + PTRTYPE = originalconcretetype(hop.args_s[0]) + # non virtual case + interiordesc, indices_v = self._getinteriordesc(hop, PTRTYPE, + hop.nb_args - 2) + if interiordesc is None: # Void field + return None + v_destbox = hop.inputarg(self.getredrepr(PTRTYPE), arg=0) + v_valuebox = hop.inputarg(self.getredrepr(interiordesc.VALUETYPE), + arg = hop.nb_args - 1) + v_destbox = hop.llops.as_ptrredbox(v_destbox) + c_interiordesc = inputconst(lltype.Void, interiordesc) + s_interiordesc = ts.rtyper.annotator.bookkeeper.immutablevalue( + interiordesc) + v_jitstate = hop.llops.getjitstate() + return hop.llops.genmixlevelhelpercall( + rtimeshift.ll_gensetinteriorfield, + [ts.s_JITState, s_interiordesc, ts.s_PtrRedBox, ts.s_RedBox] + + [ts.s_RedBox]*len(indices_v), + [v_jitstate, c_interiordesc, v_destbox, v_valuebox] + + indices_v, + annmodel.s_None) + + def translate_op_getinteriorarraysize(self, hop): + ts = self + PTRTYPE = originalconcretetype(hop.args_s[0]) + # non virtual case + interiordesc, indices_v = self._getinteriordesc(hop, PTRTYPE, + hop.nb_args - 1) + assert interiordesc is not None + v_argbox = hop.inputarg(self.getredrepr(PTRTYPE), arg=0) + v_argbox = hop.llops.as_ptrredbox(v_argbox) + c_interiordesc = inputconst(lltype.Void, interiordesc) + s_interiordesc = ts.rtyper.annotator.bookkeeper.immutablevalue( + interiordesc) + v_jitstate = hop.llops.getjitstate() + return hop.llops.genmixlevelhelpercall( + rtimeshift.ll_gengetinteriorarraysize, + [ts.s_JITState, s_interiordesc, ts.s_PtrRedBox] + + [ts.s_RedBox]*len(indices_v), + [v_jitstate, c_interiordesc, v_argbox ] + + indices_v, + ts.s_RedBox) + + def translate_op_cast_pointer(self, hop): FROM_TYPE = originalconcretetype(hop.args_s[0]) [v_argbox] = hop.inputargs(self.getredrepr(FROM_TYPE)) Modified: pypy/dist/pypy/jit/timeshifter/rcontainer.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rcontainer.py (original) +++ pypy/dist/pypy/jit/timeshifter/rcontainer.py Tue Oct 30 14:46:50 2007 @@ -349,6 +349,147 @@ content.content_boxes.append(outsidebox) return vstructbox + +class InteriorDesc(object): + __metaclass__ = cachedtype + + def __init__(self, hrtyper, TOPCONTAINER, path): + self.TOPCONTAINER = TOPCONTAINER + self.path = path + PTRTYPE = lltype.Ptr(TOPCONTAINER) + TYPE = TOPCONTAINER + fielddescs = [] + for offset in path: + LASTCONTAINER = TYPE + if offset is None: # array substruct + fielddescs.append(ArrayFieldDesc(hrtyper, TYPE)) + TYPE = TYPE.OF + else: + fielddescs.append(NamedFieldDesc(hrtyper, lltype.Ptr(TYPE), + offset)) + TYPE = getattr(TYPE, offset) + unroll_path = unrolling_iterable(path) + self.VALUETYPE = TYPE + + if not isinstance(TYPE, lltype.ContainerType): + lastoffset = path[-1] + lastfielddesc = fielddescs[-1] + immutable = LASTCONTAINER._hints.get('immutable', False) + getinterior_initial = make_interior_getter(fielddescs[:-1]) + + def gengetinteriorfield(jitstate, deepfrozen, argbox, *indexboxes): + if (immutable or deepfrozen) and argbox.is_constant(): + ptr = rvalue.ll_getvalue(argbox, PTRTYPE) + if ptr: # else don't constant-fold the segfault... + i = 0 + for offset in unroll_path: + if offset is None: # array substruct + indexbox = indexboxes[i] + i += 1 + if not indexbox.is_constant(): + break # non-constant array index + index = rvalue.ll_getvalue(indexbox, + lltype.Signed) + ptr = ptr[index] + else: + ptr = getattr(ptr, offset) + else: + # constant-folding: success + assert i == len(indexboxes) + return rvalue.ll_fromvalue(jitstate, ptr) + argbox = getinterior_initial(jitstate, argbox, *indexboxes) + if lastoffset is None: # getarrayitem + indexbox = indexboxes[-1] + genvar = jitstate.curbuilder.genop_getarrayitem( + lastfielddesc.arraytoken, + argbox.getgenvar(jitstate), + indexbox.getgenvar(jitstate)) + return lastfielddesc.makebox(jitstate, genvar) + else: # getfield + return argbox.op_getfield(jitstate, lastfielddesc) + + def gensetinteriorfield(jitstate, destbox, valuebox, *indexboxes): + destbox = getinterior_initial(jitstate, destbox, *indexboxes) + if lastoffset is None: # setarrayitem + indexbox = indexboxes[-1] + genvar = jitstate.curbuilder.genop_setarrayitem( + lastfielddesc.arraytoken, + destbox.getgenvar(jitstate), + indexbox.getgenvar(jitstate), + valuebox.getgenvar(jitstate) + ) + else: # setfield + destbox.op_setfield(jitstate, lastfielddesc, valuebox) + + self.gengetinteriorfield = gengetinteriorfield + self.gensetinteriorfield = gensetinteriorfield + + else: + assert isinstance(TYPE, lltype.Array) + arrayfielddesc = ArrayFieldDesc(hrtyper, TYPE) + getinterior_all = make_interior_getter(fielddescs) + + def gengetinteriorarraysize(jitstate, argbox, *indexboxes): + if argbox.is_constant(): + ptr = rvalue.ll_getvalue(argbox, PTRTYPE) + if ptr: # else don't constant-fold the segfault... + i = 0 + for offset in unroll_path: + if offset is None: # array substruct + indexbox = indexboxes[i] + i += 1 + if not indexbox.is_constant(): + break # non-constant array index + index = rvalue.ll_getvalue(indexbox, + lltype.Signed) + ptr = ptr[index] + else: + ptr = getattr(ptr, offset) + else: + # constant-folding: success + assert i == len(indexboxes) + return rvalue.ll_fromvalue(jitstate, len(ptr)) + argbox = getinterior_all(jitstate, argbox, *indexboxes) + genvar = jitstate.curbuilder.genop_getarraysize( + arrayfielddesc.arraytoken, + argbox.getgenvar(jitstate)) + return rvalue.IntRedBox(arrayfielddesc.indexkind, genvar) + + self.gengetinteriorarraysize = gengetinteriorarraysize + + def _freeze_(self): + return True + + +def make_interior_getter(fielddescs, _cache={}): + # returns a 'getinterior(jitstate, argbox, *indexboxes)' function + key = tuple(fielddescs) + try: + return _cache[key] + except KeyError: + unroll_fielddescs = unrolling_iterable([ + (fielddesc, isinstance(fielddesc, ArrayFieldDesc)) + for fielddesc in fielddescs]) + + def getinterior(jitstate, argbox, *indexboxes): + i = 0 + for fielddesc, is_array in unroll_fielddescs: + if is_array: # array substruct + indexbox = indexboxes[i] + i += 1 + genvar = jitstate.curbuilder.genop_getarraysubstruct( + fielddesc.arraytoken, + argbox.getgenvar(jitstate), + indexbox.getgenvar(jitstate)) + argbox = fielddesc.makebox(jitstate, genvar) + else: # getsubstruct + argbox = argbox.op_getsubstruct(jitstate, fielddesc) + assert isinstance(argbox, rvalue.PtrRedBox) + return argbox + + _cache[key] = getinterior + return getinterior + # ____________________________________________________________ # XXX basic field descs for now Modified: pypy/dist/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/rtimeshift.py Tue Oct 30 14:46:50 2007 @@ -11,6 +11,7 @@ FOLDABLE_GREEN_OPS = dict.fromkeys(lloperation.enum_foldable_ops()) FOLDABLE_GREEN_OPS['getfield'] = None FOLDABLE_GREEN_OPS['getarrayitem'] = None +FOLDABLE_GREEN_OPS['getinteriorfield'] = None NULL_OBJECT = base_ptr_lltype()._defl() @@ -211,6 +212,20 @@ argbox.getgenvar(jitstate)) return rvalue.IntRedBox(fielddesc.indexkind, genvar) + +def ll_gengetinteriorfield(jitstate, deepfrozen, interiordesc, + argbox, *indexboxes): + return interiordesc.gengetinteriorfield(jitstate, deepfrozen, + argbox, *indexboxes) + +def ll_gensetinteriorfield(jitstate, interiordesc, destbox, + valuebox, *indexboxes): + interiordesc.gensetinteriorfield(jitstate, destbox, valuebox, *indexboxes) + +def ll_gengetinteriorarraysize(jitstate, interiordesc, argbox, *indexboxes): + return interiordesc.gengetinteriorarraysize(jitstate, argbox, *indexboxes) + + def ll_genptrnonzero(jitstate, argbox, reverse): if argbox.is_constant(): addr = rvalue.ll_getvalue(argbox, llmemory.Address) Modified: pypy/dist/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_portal.py Tue Oct 30 14:46:50 2007 @@ -196,10 +196,16 @@ s = ["aaaaaaaaaab", "aaaa"][gets] return recognizetable(dfatable, s, final_states) - res = self.timeshift_from_portal(main, recognizetable, [0], policy=P_NOVIRTUAL) + # must backendoptimize to remove the mallocs related + # to the interior ptrs + res = self.timeshift_from_portal(main, recognizetable, [0], + policy=P_NOVIRTUAL, + backendoptimize=True) assert res - res = self.timeshift_from_portal(main, recognizetable, [1], policy=P_NOVIRTUAL) + res = self.timeshift_from_portal(main, recognizetable, [1], + policy=P_NOVIRTUAL, + backendoptimize=True) assert not res def test_dfa_compile2(self): @@ -210,11 +216,17 @@ s = ["aaaaaaaaaab", "aaaa"][gets] return recognizeparts(alltrans, final_states, s) - res = self.timeshift_from_portal(main, recognizeparts, [0, 0], policy=P_NOVIRTUAL) + # must backendoptimize to remove the mallocs related + # to the interior ptrs + res = self.timeshift_from_portal(main, recognizeparts, [0, 0], + policy=P_NOVIRTUAL, + backendoptimize=True) assert res # XXX unfortunately we have to create a new version each time - because of pbc - res = self.timeshift_from_portal(main, recognizeparts, [1, 0], policy=P_NOVIRTUAL) + res = self.timeshift_from_portal(main, recognizeparts, [1, 0], + policy=P_NOVIRTUAL, + backendoptimize=True) assert not res def test_dfa_compile3(self): Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Tue Oct 30 14:46:50 2007 @@ -59,8 +59,7 @@ auto_inlining(t, threshold=inline) if backendoptimize: from pypy.translator.backendopt.all import backend_optimizations - if inline is not None: - backend_optimizations(t, inline_threshold=inline) + backend_optimizations(t, inline_threshold=inline or 0) if portal is None: portal = func if hasattr(policy, "seetranslator"): @@ -809,16 +808,14 @@ policy=P_NOVIRTUAL) assert res == 42 self.check_insns({'malloc_varsize': 1, - 'getarraysubstruct': 3, - 'setfield': 2, 'getfield': 1, + 'setinteriorfield': 2, 'getinteriorfield': 1, 'getarraysize': 1, 'int_mul': 1}) res = self.timeshift(ll_function, [21, -21, 1], [], policy=P_NOVIRTUAL) assert res == -42 self.check_insns({'malloc_varsize': 1, - 'getarraysubstruct': 3, - 'setfield': 2, 'getfield': 1, + 'setinteriorfield': 2, 'getinteriorfield': 1, 'getarraysize': 1, 'int_mul': 1}) @@ -835,12 +832,14 @@ res = self.timeshift(ll_function, [21, -21, 0], [], policy=P_NOVIRTUAL) assert res == 42 - self.check_insns(malloc_varsize=1) + self.check_insns(malloc_varsize=1, + getinteriorarraysize=1) res = self.timeshift(ll_function, [21, -21, 1], [], policy=P_NOVIRTUAL) assert res == -42 - self.check_insns(malloc_varsize=1) + self.check_insns(malloc_varsize=1, + getinteriorarraysize=1) def test_array_of_voids(self): A = lltype.GcArray(lltype.Void) From arigo at codespeak.net Tue Oct 30 14:49:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 30 Oct 2007 14:49:52 +0100 (CET) Subject: [pypy-svn] r48199 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071030134952.06D8781B8@code0.codespeak.net> Author: arigo Date: Tue Oct 30 14:49:52 2007 New Revision: 48199 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: Remove tabs. Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Tue Oct 30 14:49:52 2007 @@ -263,20 +263,20 @@ The current format of a CompiledMethod is as follows: - header (4 bytes) - literals (4 bytes each) - bytecodes (variable) - trailer (variable) + header (4 bytes) + literals (4 bytes each) + bytecodes (variable) + trailer (variable) The header is a 30-bit integer with the following format: - (index 0) 9 bits: main part of primitive number (#primitive) - (index 9) 8 bits: number of literals (#numLiterals) - (index 17) 1 bit: whether a large frame size is needed (#frameSize) - (index 18) 6 bits: number of temporary variables (#numTemps) - (index 24) 4 bits: number of arguments to the method (#numArgs) - (index 28) 1 bit: high-bit of primitive number (#primitive) - (index 29) 1 bit: flag bit, ignored by the VM (#flag) + (index 0) 9 bits: main part of primitive number (#primitive) + (index 9) 8 bits: number of literals (#numLiterals) + (index 17) 1 bit: whether a large frame size is needed (#frameSize) + (index 18) 6 bits: number of temporary variables (#numTemps) + (index 24) 4 bits: number of arguments to the method (#numArgs) + (index 28) 1 bit: high-bit of primitive number (#primitive) + (index 29) 1 bit: flag bit, ignored by the VM (#flag) The trailer has two variant formats. In the first variant, the last byte is at least 252 and the last four bytes represent a source pointer into one of the sources files (see #sourcePointer). In the second variant, the last byte is less than 252, and the last several bytes are a compressed version of the names of the method's temporary variables. The number of bytes used for this purpose is the value of the last byte in the method. """ Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Tue Oct 30 14:49:52 2007 @@ -358,13 +358,13 @@ def fillin_compiledmethod(self, w_compiledmethod): header = self.chunk.data[0] #---!!!---- 1 tagged pointer! - #(index 0) 9 bits: main part of primitive number (#primitive) - #(index 9) 8 bits: number of literals (#numLiterals) - #(index 17) 1 bit: whether a large frame size is needed (#frameSize) - #(index 18) 6 bits: number of temporary variables (#numTemps) - #(index 24) 4 bits: number of arguments to the method (#numArgs) - #(index 28) 1 bit: high-bit of primitive number (#primitive) - #(index 29) 1 bit: flag bit, ignored by the VM (#flag) + #(index 0) 9 bits: main part of primitive number (#primitive) + #(index 9) 8 bits: number of literals (#numLiterals) + #(index 17) 1 bit: whether a large frame size is needed (#frameSize) + #(index 18) 6 bits: number of temporary variables (#numTemps) + #(index 24) 4 bits: number of arguments to the method (#numArgs) + #(index 28) 1 bit: high-bit of primitive number (#primitive) + #(index 29) 1 bit: flag bit, ignored by the VM (#flag) _, primitive, literalsize, islarge, tempsize, numargs, highbit = ( splitter[1,9,8,1,6,4,1](header)) primitive = primitive + (highbit << 10) ##XXX todo, check this Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Tue Oct 30 14:49:52 2007 @@ -619,10 +619,10 @@ # tests: bytecodePrimValue & bytecodePrimValueWithArg def test_bc_3_plus_4(): # value0 - # " (self >> #value0) byteCode " - # " (self >> #value0) literals " + # " (self >> #value0) byteCode " + # " (self >> #value0) literals " # - # ^ [ 3 + 4 ] value + # ^ [ 3 + 4 ] value assert interpret_bc( [ 137, 117, 200, 164, 4, 32, 33, 176, 125, 201, 124], fakeliterals(wrap_int(3), wrap_int(4))).value == 7 @@ -630,10 +630,10 @@ def test_bc_x_plus_x_plus_1(): # value1 - # " (self >> #value1) byteCode " - # " (self >> #value1) literals " + # " (self >> #value1) byteCode " + # " (self >> #value1) literals " # - # ^ [ :x | x + x + 1 ] value: 3 + # ^ [ :x | x + x + 1 ] value: 3 assert interpret_bc( [ 137, 118, 200, 164, 7, 104, 16, 16, 176, 118, 176, 125, 32, 202, 124 ], @@ -641,10 +641,10 @@ def test_bc_x_plus_y(): # value2 - # " (self >> #value2) byteCode " - # " (self >> #value2) literals " + # " (self >> #value2) byteCode " + # " (self >> #value2) literals " # - # ^ [ :x :y | x + y ] value: 3 value: 4 + # ^ [ :x :y | x + y ] value: 3 value: 4 def test(): assert interpret_bc( @@ -658,30 +658,30 @@ def test_bc_push_rcvr_in_block(): # value1 - # " (self >> #value1) byteCode " - # " (self >> #value1) literals " + # " (self >> #value1) byteCode " + # " (self >> #value1) literals " # - # ^ [ self ] value + # ^ [ self ] value assert interpret_bc( [ 137, 117, 200, 164, 2, 112, 125, 201, 124 ], fakeliterals(wrap_int(3))) is objtable.w_nil def test_bc_value_return(): # valueReturn - # " (self >> #value1) byteCode " - # " (self >> #value1) literals " + # " (self >> #value1) byteCode " + # " (self >> #value1) literals " # - # [ ^ 1 ] value. ^ 2 + # [ ^ 1 ] value. ^ 2 assert interpret_bc( [ 137, 117, 200, 164, 2, 118, 124, 201, 135, 119, 124 ], fakeliterals()).value == 1 def test_bc_value_with_args(): # valueWithArgs - # " (self >> #value1) byteCode " - # " (self >> #value1) literals " + # " (self >> #value1) byteCode " + # " (self >> #value1) literals " # - # [ :a :b | a - b ] valueWithArguments: #(3 2) + # [ :a :b | a - b ] valueWithArguments: #(3 2) def test(): assert interpret_bc( [ 137, 119, 200, 164, 6, @@ -695,7 +695,7 @@ test) def test_bc_primBytecodeAt_string(): - # ^ 'a' at: 1 + # ^ 'a' at: 1 def test(): assert interpret_bc( [ 32, 118, 192, 124], @@ -705,7 +705,7 @@ test) def test_bc_primBytecodeAtPut_string(): - # ^ 'a' at: 1 put:'b' + # ^ 'a' at: 1 put:'b' def test(): assert interpret_bc( [ 32, 118, 33, 193, 124 ], @@ -715,7 +715,7 @@ test) def test_bc_primBytecodeAt_with_instvars(): - # ^ self at: 1 + # ^ self at: 1 w_fakeclass = mockclass(1, name='fakeclass', varsized=True) w_fakeinst = w_fakeclass.as_class_get_shadow().new(1) w_fakeinst.store(0, wrap_char("a")) # static slot 0: instance variable @@ -730,7 +730,7 @@ test) def test_bc_primBytecodeAtPut_with_instvars(): - # ^ self at: 1 put: #b + # ^ self at: 1 put: #b w_fakeclass = mockclass(1, name='fakeclass', varsized=True) w_fakeinst = w_fakeclass.as_class_get_shadow().new(1) w_fakeinst.store(0, wrap_char("a")) # static slot 0: instance variable @@ -749,7 +749,7 @@ def test_bc_objectAtAndAtPut(): # ^ self objectAt: 1. yields the method header # ^ self objectAt: 2. yields the first literal (22) - # ^ self objectAt: 2 put: 3. changes the first literal to 3 + # ^ self objectAt: 2 put: 3. changes the first literal to 3 # ^ self objectAt: 2. yields the new first literal (3) prim_meth = model.W_CompiledMethod(0, "") prim_meth.literals = fakeliterals(22) Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Tue Oct 30 14:49:52 2007 @@ -229,8 +229,8 @@ py.test.skip("not quite yet") sourcecode = """fib ^self < 2 - ifTrue: [ 1 ] - ifFalse: [ (self - 1) fib + (self - 2) fib ]""" + ifTrue: [ 1 ] + ifFalse: [ (self - 1) fib + (self - 2) fib ]""" perform(w(10).getclass(), "compile:classified:notifying:", w(sourcecode), w('pypy'), w(None)) assert perform(w(10), "fib") == w(89) From arigo at codespeak.net Tue Oct 30 14:50:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 30 Oct 2007 14:50:09 +0100 (CET) Subject: [pypy-svn] r48200 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071030135009.277F981BB@code0.codespeak.net> Author: arigo Date: Tue Oct 30 14:50:08 2007 New Revision: 48200 Modified: pypy/dist/pypy/lang/smalltalk/conftest.py (props changed) pypy/dist/pypy/lang/smalltalk/error.py (props changed) pypy/dist/pypy/lang/smalltalk/utility.py (props changed) Log: fixeol From arigo at codespeak.net Tue Oct 30 14:52:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 30 Oct 2007 14:52:08 +0100 (CET) Subject: [pypy-svn] r48201 - pypy/dist/pypy/annotation Message-ID: <20071030135208.30F7881BE@code0.codespeak.net> Author: arigo Date: Tue Oct 30 14:52:07 2007 New Revision: 48201 Modified: pypy/dist/pypy/annotation/binaryop.py Log: Fix for translator.c.test.test_symbolic: test_sizeof_constsize_struct. Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Tue Oct 30 14:52:07 2007 @@ -337,7 +337,9 @@ # a special case for 'x < 0' or 'x >= 0', # where 0 is a flow graph Constant # (in this case we are sure that it cannot become a r_uint later) - if isinstance(op.args[1], Constant) and op.args[1].value == 0: + if (isinstance(op.args[1], Constant) and + type(op.args[1].value) is int and # filter out Symbolics + op.args[1].value == 0): if int1.nonneg: if opname == 'lt': r.const = False From arigo at codespeak.net Tue Oct 30 16:46:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 30 Oct 2007 16:46:53 +0100 (CET) Subject: [pypy-svn] r48202 - in pypy/dist/pypy/jit/codegen: i386/test llgraph/test test Message-ID: <20071030154653.C6C9981BC@code0.codespeak.net> Author: arigo Date: Tue Oct 30 16:46:51 2007 New Revision: 48202 Modified: pypy/dist/pypy/jit/codegen/i386/test/test_rgenop.py pypy/dist/pypy/jit/codegen/llgraph/test/test_rgenop.py pypy/dist/pypy/jit/codegen/test/rgenop_tests.py Log: Shuffle tests so that some rgenop_tests can use an RGenOpPacked if necessary. Modified: pypy/dist/pypy/jit/codegen/i386/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/i386/test/test_rgenop.py Tue Oct 30 16:46:51 2007 @@ -5,48 +5,7 @@ class TestRI386Genop(AbstractRGenOpTests): RGenOp = RI386GenOp + from pypy.jit.codegen.i386.test.test_operation import RGenOpPacked # for the individual tests see # ====> ../../test/rgenop_tests.py - - def test_array_of_ints(self): - from pypy.jit.codegen.i386.test.test_operation import RGenOpPacked - rgenop = RGenOpPacked() - A = lltype.GcArray(lltype.Signed) - FUNC3 = lltype.FuncType([lltype.Signed]*3, lltype.Signed) - varsizealloctoken = rgenop.varsizeAllocToken(A) - arraytoken = rgenop.arrayToken(A) - signed_kind = rgenop.kindToken(lltype.Signed) - # ------------------------------------------------------------ - builder0, gv_callable, [v0, v1, v2] = rgenop.newgraph( - rgenop.sigToken(FUNC3), 'generated') - builder0.start_writing() - v3 = builder0.genop_malloc_varsize(varsizealloctoken, - rgenop.genconst(2)) - v4 = builder0.genop1('ptr_iszero', v3) - builder1 = builder0.jump_if_false(v4, [v2, v0, v3, v1]) - builder2 = builder0.pause_writing([]) - builder1.start_writing() - builder1.genop_setarrayitem(arraytoken, v3, rgenop.genconst(0), v0) - builder1.genop_setarrayitem(arraytoken, v3, rgenop.genconst(1), v1) - v5 = builder1.genop_getarrayitem(arraytoken, v3, v2) - v6 = builder1.genop_getarraysize(arraytoken, v3) - v7 = builder1.genop2('int_mul', v5, v6) - builder3 = builder1.pause_writing([v7]) - builder3.start_writing() - args_gv = [v7] - label0 = builder3.enter_next_block([signed_kind], args_gv) - [v8] = args_gv - builder4 = builder3.pause_writing([v8]) - builder2.start_writing() - builder2.finish_and_goto([rgenop.genconst(-1)], label0) - builder4.start_writing() - args_gv = [v8] - label1 = builder4.enter_next_block([signed_kind], args_gv) - [v9] = args_gv - builder4.finish_and_return(rgenop.sigToken(FUNC3), v9) - builder0.end() - - fnptr = self.cast(gv_callable, 3) - res = fnptr(21, -21, 0) - assert res == 42 Modified: pypy/dist/pypy/jit/codegen/llgraph/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/llgraph/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/llgraph/test/test_rgenop.py Tue Oct 30 16:46:51 2007 @@ -8,6 +8,7 @@ class TestLLGraphRGenop(AbstractRGenOpTests): RGenOp = RGenOp + RGenOpPacked = RGenOp def setup_method(self, meth): if 'ovfcheck' in meth.__name__: Modified: pypy/dist/pypy/jit/codegen/test/rgenop_tests.py ============================================================================== --- pypy/dist/pypy/jit/codegen/test/rgenop_tests.py (original) +++ pypy/dist/pypy/jit/codegen/test/rgenop_tests.py Tue Oct 30 16:46:51 2007 @@ -1,4 +1,4 @@ -import random, sys +import random, sys, py from pypy.rpython.annlowlevel import MixLevelAnnotatorPolicy, llhelper from pypy.rlib.rarithmetic import intmask, r_uint from pypy.rlib.objectmodel import keepalive_until_here @@ -836,6 +836,7 @@ class AbstractRGenOpTests(test_boehm.AbstractGCTestClass): RGenOp = None + RGenOpPacked = None def compile(self, runner, argtypes): return self.getcompiled(runner, argtypes, @@ -2076,16 +2077,61 @@ result = fnptr(x) assert result == expected(x) + def test_array_of_ints(self): + if self.RGenOpPacked is None: + py.test.skip("requires RGenOpPacked") + rgenop = self.RGenOpPacked() + A = lltype.GcArray(lltype.Signed) + FUNC3 = lltype.FuncType([lltype.Signed]*3, lltype.Signed) + varsizealloctoken = rgenop.varsizeAllocToken(A) + arraytoken = rgenop.arrayToken(A) + signed_kind = rgenop.kindToken(lltype.Signed) + # ------------------------------------------------------------ + builder0, gv_callable, [v0, v1, v2] = rgenop.newgraph( + rgenop.sigToken(FUNC3), 'generated') + builder0.start_writing() + v3 = builder0.genop_malloc_varsize(varsizealloctoken, + rgenop.genconst(2)) + v4 = builder0.genop1('ptr_iszero', v3) + builder1 = builder0.jump_if_false(v4, [v2, v0, v3, v1]) + builder2 = builder0.pause_writing([]) + builder1.start_writing() + builder1.genop_setarrayitem(arraytoken, v3, rgenop.genconst(0), v0) + builder1.genop_setarrayitem(arraytoken, v3, rgenop.genconst(1), v1) + v5 = builder1.genop_getarrayitem(arraytoken, v3, v2) + v6 = builder1.genop_getarraysize(arraytoken, v3) + v7 = builder1.genop2('int_mul', v5, v6) + builder3 = builder1.pause_writing([v7]) + builder3.start_writing() + args_gv = [v7] + label0 = builder3.enter_next_block([signed_kind], args_gv) + [v8] = args_gv + builder4 = builder3.pause_writing([v8]) + builder2.start_writing() + builder2.finish_and_goto([rgenop.genconst(-1)], label0) + builder4.start_writing() + args_gv = [v8] + label1 = builder4.enter_next_block([signed_kind], args_gv) + [v9] = args_gv + builder4.finish_and_return(rgenop.sigToken(FUNC3), v9) + builder0.end() + + fnptr = self.cast(gv_callable, 3) + res = fnptr(21, -21, 0) + assert res == 42 + def test_interior_access(self): # for assembler backends, the 'interior' lloperations can be # simply expressed as a sequence of genop_getsubstruct and # genop_getarraysubstruct. So we put magic in the llgraph # backend to recognize and rebuild the expected 'interior' # lloperation in the llgraphs. + if self.RGenOpPacked is None: + py.test.skip("requires RGenOpPacked") T = lltype.Struct('T', ('x', lltype.Signed)) A = lltype.Array(T) S = lltype.GcStruct('S', ('a', A)) - rgenop = self.RGenOp() + rgenop = self.RGenOpPacked() sigtoken = rgenop.sigToken(FUNC) builder, gv_fn, [gv_x] = rgenop.newgraph(sigtoken, "interior_access") builder.start_writing() From arigo at codespeak.net Tue Oct 30 17:46:10 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 30 Oct 2007 17:46:10 +0100 (CET) Subject: [pypy-svn] r48203 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071030164610.6CF2981C1@code0.codespeak.net> Author: arigo Date: Tue Oct 30 17:46:08 2007 New Revision: 48203 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Log: A conservative fix of the bytecodePrimXxx(): always use _sendSelfSelector() unless it is safe to directly invoke the primitive because the latter checks the class of the receiver. This is mostly what Squeak does too. Also add a call to the FLOAT_XYZ primitive if the XYZ primitive failed (i.e. the arguments were not integers). Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Tue Oct 30 17:46:08 2007 @@ -334,48 +334,80 @@ def longJumpIfFalse(self, interp): self.jumpConditional(interp.FALSE,self.longJumpPosition()) - def callPrimitive(self, primitive, selector, - argcount, interp): - # XXX XXX REMEMBER TO SWITCH COMMENT AND ACTUAL CODE BEFORE - # TESTING ON AN IMAGE, AND TO LEAVE AS IS FOR TESTCODE (and - # vice versa) + def callPrimitive(self, primitive, selector, argcount, interp): + # WARNING: this is used for bytecodes for which it is safe to + # directly call the primitive. In general, it is not safe: for + # example, depending on the type of the receiver, bytecodePrimAt + # may invoke primitives.AT, primitives.STRING_AT, or anything + # else that the user put in a class in an 'at:' method. + # The rule of thumb is that primitives with only int and float + # in their unwrap_spec are safe. try: # note that argcount does not include self - w_result = primitives.prim_table[primitive](interp, argcount) + primitives.prim_table[primitive](interp, argcount) # the primitive pushes the result (if any) onto the stack itself except primitives.PrimitiveFailedError: self._sendSelfSelector(selector, argcount, interp) - #self._sendSelfSelector(selector, argcount, interp) + + def callPrimitive2(self, primitive1, primitive2, + selector, argcount, interp): + # same as callPrimitive(), but tries two primitives before falling + # back to the general case. + try: + primitives.prim_table[primitive1](interp, argcount) + # the primitive pushes the result (if any) onto the stack itself + except primitives.PrimitiveFailedError: + self.callPrimitive(primitive2, selector, argcount, interp) def bytecodePrimAdd(self, interp): - self.callPrimitive(primitives.ADD, "+", 1, interp) + self.callPrimitive2(primitives.ADD, + primitives.FLOAT_ADD, + "+", 1, interp) def bytecodePrimSubtract(self, interp): - self.callPrimitive(primitives.SUBTRACT, "-", 1, interp) + self.callPrimitive2(primitives.SUBTRACT, + primitives.FLOAT_SUBTRACT, + "-", 1, interp) def bytecodePrimLessThan(self, interp): - self.callPrimitive(primitives.LESSTHAN, "<", 1, interp) + self.callPrimitive2(primitives.LESSTHAN, + primitives.FLOAT_LESSTHAN, + "<", 1, interp) def bytecodePrimGreaterThan(self, interp): - self.callPrimitive(primitives.GREATERTHAN, ">", 1, interp) + self.callPrimitive2(primitives.GREATERTHAN, + primitives.FLOAT_GREATERTHAN, + ">", 1, interp) def bytecodePrimLessOrEqual(self, interp): - self.callPrimitive(primitives.LESSOREQUAL, "<=", 1, interp) + self.callPrimitive2(primitives.LESSOREQUAL, + primitives.FLOAT_LESSOREQUAL, + "<=", 1, interp) def bytecodePrimGreaterOrEqual(self, interp): - self.callPrimitive(primitives.GREATEROREQUAL, ">=", 1, interp) + self.callPrimitive2(primitives.GREATEROREQUAL, + primitives.FLOAT_GREATEROREQUAL, + ">=", 1, interp) def bytecodePrimEqual(self, interp): - self.callPrimitive(primitives.EQUAL, "=", 1, interp) + self.callPrimitive2(primitives.EQUAL, + primitives.FLOAT_EQUAL, + "=", 1, interp) def bytecodePrimNotEqual(self, interp): - self.callPrimitive(primitives.NOTEQUAL, "~=", 1, interp) + self.callPrimitive2(primitives.NOTEQUAL, + primitives.FLOAT_NOTEQUAL, + "~=", 1, interp) def bytecodePrimMultiply(self, interp): - self.callPrimitive(primitives.MULTIPLY, "*", 1, interp) + self.callPrimitive2(primitives.MULTIPLY, + primitives.FLOAT_MULTIPLY, + "*", 1, interp) def bytecodePrimDivide(self, interp): - self.callPrimitive(primitives.DIVIDE, "/", 1, interp) + self.callPrimitive2(primitives.DIVIDE, + primitives.FLOAT_DIVIDE, + "/", 1, interp) def bytecodePrimMod(self, interp): self.callPrimitive(primitives.MOD, "\\\\", 1, interp) @@ -399,42 +431,50 @@ # n.b.: depending on the type of the receiver, this may invoke # primitives.AT, primitives.STRING_AT, or something else for all # I know. - #self.callPrimitive(primitives.AT, "at:", 1, interp) self._sendSelfSelector("at:", 1, interp) def bytecodePrimAtPut(self, interp): # n.b. as above - #self.callPrimitive(primitives.AT_PUT, "at:put:", 2, interp) self._sendSelfSelector("at:put:", 2, interp) def bytecodePrimSize(self, interp): - self.callPrimitive(primitives.SIZE, "size", 0, interp) + self._sendSelfSelector("size", 0, interp) def bytecodePrimNext(self, interp): - self.callPrimitive(primitives.NEXT, "next", 0, interp) + self._sendSelfSelector("next", 0, interp) def bytecodePrimNextPut(self, interp): - self.callPrimitive(primitives.NEXT_PUT, "nextPut:", 1, interp) + self._sendSelfSelector("nextPut:", 1, interp) def bytecodePrimAtEnd(self, interp): - self.callPrimitive(primitives.AT_END, "atEnd", 0, interp) + self._sendSelfSelector("atEnd", 0, interp) def bytecodePrimEquivalent(self, interp): - self.callPrimitive(primitives.EQUIVALENT, "==", 1, interp) + # short-circuit: classes cannot override the '==' method, + # which cannot fail + primitives.prim_table[primitives.EQUIVALENT](interp, 1) def bytecodePrimClass(self, interp): - self.callPrimitive( - primitives.CLASS, "class", 0, interp) + # short-circuit: classes cannot override the 'class' method, + # which cannot fail + primitives.prim_table[primitives.CLASS](interp, 0) def bytecodePrimBlockCopy(self, interp): - self.callPrimitive( - primitives.PRIMITIVE_BLOCK_COPY, "blockCopy:", 1, interp) + # the primitive checks the class of the receiver + self.callPrimitive(primitives.PRIMITIVE_BLOCK_COPY, + "blockCopy:", 1, interp) def bytecodePrimValue(self, interp): + # the primitive checks the class of the receiver self.callPrimitive( primitives.PRIMITIVE_VALUE, "value", 0, interp) def bytecodePrimValueWithArg(self, interp): + # the primitive checks the class of the receiver + # Note that the PRIMITIVE_VALUE_WITH_ARGS takes an array of + # arguments but this bytecode is about the one-argument case. + # The PRIMITIVE_VALUE is general enough to take any number of + # arguments from the stack, so it's the one we need to use here. self.callPrimitive( primitives.PRIMITIVE_VALUE, "value:", 1, interp) @@ -442,10 +482,10 @@ self._sendSelfSelector("do:", 1, interp) def bytecodePrimNew(self, interp): - self.callPrimitive(primitives.NEW, "new", 0, interp) + self._sendSelfSelector("new", 0, interp) def bytecodePrimNewWithArg(self, interp): - self.callPrimitive(primitives.NEW_WITH_ARG, "new:", 1, interp) + self._sendSelfSelector("new:", 1, interp) def bytecodePrimPointX(self, interp): self._sendSelfSelector("x", 0, interp) Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Tue Oct 30 17:46:08 2007 @@ -330,21 +330,29 @@ assert interp.w_active_context.stack == [] def test_bytecodePrimNew(): - w_fakeclass = mockclass(1, name='fakeclass', varsized=False) + w_fakeclassclass = mockclass(10, name='fakeclassclass') + w_fakeclass = mockclass(1, name='fakeclass', varsized=False, + w_metaclass=w_fakeclassclass) interp = new_interpreter(bytecodePrimNew) interp.w_active_context.push(w_fakeclass) - interp.step() + run_with_faked_methods( + [[w_fakeclassclass, primitives.NEW, 0, "new"]], + interp.step) w_fakeinst = interp.w_active_context.pop() assert interp.w_active_context.stack == [] assert w_fakeinst.getclass() == w_fakeclass assert w_fakeinst.size() == 1 def test_bytecodePrimNewWithArg(): - w_fakeclass = mockclass(1, name='fakeclass', varsized=True) + w_fakeclassclass = mockclass(10, name='fakeclassclass') + w_fakeclass = mockclass(1, name='fakeclass', varsized=True, + w_metaclass=w_fakeclassclass) interp = new_interpreter(bytecodePrimNewWithArg) interp.w_active_context.push(w_fakeclass) interp.w_active_context.push(interpreter.Interpreter.TWO) - interp.step() + run_with_faked_methods( + [[w_fakeclassclass, primitives.NEW_WITH_ARG, 1, "new:"]], + interp.step) w_fakeinst = interp.w_active_context.pop() assert interp.w_active_context.stack == [] assert w_fakeinst.getclass() == w_fakeclass @@ -355,7 +363,9 @@ w_fakeinst = w_fakeclass.as_class_get_shadow().new(5) interp = new_interpreter(bytecodePrimSize) interp.w_active_context.push(w_fakeinst) - interp.step() + run_with_faked_methods( + [[w_fakeclass, primitives.SIZE, 0, "size"]], + interp.step) assert interp.w_active_context.pop().value == 5 assert interp.w_active_context.stack == [] From arigo at codespeak.net Tue Oct 30 17:48:14 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 30 Oct 2007 17:48:14 +0100 (CET) Subject: [pypy-svn] r48204 - pypy/dist/pypy/translator/goal Message-ID: <20071030164814.DA38781BB@code0.codespeak.net> Author: arigo Date: Tue Oct 30 17:48:14 2007 New Revision: 48204 Modified: pypy/dist/pypy/translator/goal/targettinybenchsmalltalk.py Log: Fix this target. Modified: pypy/dist/pypy/translator/goal/targettinybenchsmalltalk.py ============================================================================== --- pypy/dist/pypy/translator/goal/targettinybenchsmalltalk.py (original) +++ pypy/dist/pypy/translator/goal/targettinybenchsmalltalk.py Tue Oct 30 17:48:14 2007 @@ -1,7 +1,7 @@ import os from pypy.lang.smalltalk import model, interpreter, primitives, shadow from pypy.lang.smalltalk import objtable -from pypy.lang.smalltalk.objtable import wrap_int +from pypy.lang.smalltalk.utility import wrap_int from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk.tool.analyseimage import * From arigo at codespeak.net Tue Oct 30 18:03:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 30 Oct 2007 18:03:24 +0100 (CET) Subject: [pypy-svn] r48205 - pypy/dist/pypy/lang/smalltalk/tool Message-ID: <20071030170324.4D3BE81A7@code0.codespeak.net> Author: arigo Date: Tue Oct 30 18:03:22 2007 New Revision: 48205 Modified: pypy/dist/pypy/lang/smalltalk/tool/bitmanipulation.py Log: A far less terrible hack works too. Modified: pypy/dist/pypy/lang/smalltalk/tool/bitmanipulation.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/tool/bitmanipulation.py (original) +++ pypy/dist/pypy/lang/smalltalk/tool/bitmanipulation.py Tue Oct 30 18:03:22 2007 @@ -1,8 +1,5 @@ from pypy.rlib import unroll -# terrible hack to make the annotator think this is a real dict -__name__ = '__builtin__' - class BitSplitter(dict): def __getitem__(self, lengths): if isinstance(lengths, int): @@ -24,5 +21,11 @@ splitbits.func_name += "_" + "_".join([str(i) for i in lengths]) self[lengths] = splitbits return splitbits - + + def _freeze_(self): + # as this class is not in __builtin__, we need to explicitly tell + # the flow space that the object is frozen and the accesses can + # be constant-folded. + return True + splitter = BitSplitter() From rxe at codespeak.net Tue Oct 30 18:54:09 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 30 Oct 2007 18:54:09 +0100 (CET) Subject: [pypy-svn] r48206 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071030175409.B90AB81C1@code0.codespeak.net> Author: rxe Date: Tue Oct 30 18:54:08 2007 New Revision: 48206 Modified: pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/modwrapper.py pypy/dist/pypy/translator/llvm/test/test_lltype.py Log: painfully support single float Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Tue Oct 30 18:54:08 2007 @@ -349,6 +349,7 @@ self.types = { lltype.Char: "sbyte", lltype.Bool: "bool", + lltype.SingleFloat: "float", lltype.Float: "double", lltype.UniChar: "uint", lltype.Void: "void", @@ -377,6 +378,7 @@ lltype.Signed : self.repr_signed, lltype.UnsignedLongLong : self.repr_default, lltype.Unsigned : self.repr_default, + lltype.SingleFloat: self.repr_singlefloat, lltype.Float : self.repr_float, lltype.Char : self.repr_char, lltype.UniChar : self.repr_unichar, @@ -463,6 +465,26 @@ return repr + def repr_singlefloat(self, type_, value): + from pypy.rlib.rarithmetic import isinf, isnan + + # XXX this doesnt work on 1.9 -> only hex repr supported for single floats + f = float(value) + if isinf(f) or isnan(f): + import struct + packed = value._bytes + if sys.byteorder == 'little': + packed = packed[::-1] + assert len(packed) == 4 + repr = "0x" + "".join([("%02x" % ord(ii)) for ii in packed]) + else: + repr = "%f" % f + + # llvm requires a . when using e notation + if "e" in repr and "." not in repr: + repr = repr.replace("e", ".0e") + return repr + def repr_address(self, type_, value): if not value: return 'null' Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Tue Oct 30 18:54:08 2007 @@ -36,6 +36,7 @@ return code TO_CTYPES = {lltype.Bool: "ctypes.c_int", + lltype.SingleFloat: "ctypes.c_float", lltype.Float: "ctypes.c_double", lltype.Char: "ctypes.c_char", lltype.Signed: "ctypes.c_int", Modified: pypy/dist/pypy/translator/llvm/test/test_lltype.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lltype.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lltype.py Tue Oct 30 18:54:08 2007 @@ -760,6 +760,34 @@ fn = compile_function(llf, []) assert fn() is True +def test_r_singlefloat(): + z = r_singlefloat(0.4) + + def llf(): + return z + + fn = compile_function(llf, []) + res = fn() + assert res != 0.4 # precision lost + assert abs(res - 0.4) < 1E-6 + + def g(n): + if n > 0: + return r_singlefloat(n * 0.1) + else: + return z + + def llf(n): + return float(g(n)) + + fn = compile_function(llf, [int]) + res = fn(21) + assert res != 2.1 # precision lost + assert abs(res - 2.1) < 1E-6 + res = fn(-5) + assert res != 0.4 # precision lost + assert abs(res - 0.4) < 1E-6 + class TestLowLevelType(object): def getcompiled(self, f, args=[]): return compile_function(f, args) @@ -893,28 +921,6 @@ 0, maxlonglong>>4, # rshift )) - def test_r_singlefloat(self): - py.test.skip("singlefloat") - - z = r_singlefloat(0.4) - - def g(n): - if n > 0: - return r_singlefloat(n * 0.1) - else: - return z - - def llf(n): - return float(g(n)) - - fn = self.getcompiled(llf, [int]) - res = fn(21) - assert res != 2.1 # precision lost - assert abs(res - 2.1) < 1E-6 - res = fn(-5) - assert res != 0.4 # precision lost - assert abs(res - 0.4) < 1E-6 - def test_prebuilt_nolength_char_array(self): py.test.skip("fails on the trunk too") for lastchar in ('\x00', 'X'): From rxe at codespeak.net Tue Oct 30 18:55:56 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 30 Oct 2007 18:55:56 +0100 (CET) Subject: [pypy-svn] r48207 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071030175556.6871E81C4@code0.codespeak.net> Author: rxe Date: Tue Oct 30 18:55:55 2007 New Revision: 48207 Modified: pypy/dist/pypy/translator/llvm/buildllvm.py pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/test/runtest.py Log: get rid of some XXXs Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Tue Oct 30 18:55:55 2007 @@ -100,7 +100,7 @@ def cmds_objects(self, base): # XXX why this hack??? - use_gcc = False #self.genllvm.config.translation.llvm_via_c + use_gcc = self.genllvm.config.translation.llvm_via_c if use_gcc: self.cmds.append("llc %s.bc -march=c -f -o %s.c" % (base, base)) self.cmds.append("gcc %s.c -c -O3 -fomit-frame-pointer" % base) Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Tue Oct 30 18:55:55 2007 @@ -167,7 +167,7 @@ includestr += "-I %s " % ii return includestr -def generate_llfile(db, extern_decls, entrynode, c_include, c_sources, standalone): +def generate_llfile(db, extern_decls, entrynode, c_includes, c_sources, standalone): ccode = [] function_names = [] @@ -212,9 +212,14 @@ # ask gcpolicy for any code needed ccode.append('%s\n' % db.gcpolicy.genextern_code()) + ccode.append('\n') + for c_include in c_includes: + ccode.append('#include <%s>\n' % c_include) + for c_source in c_sources: for l in c_source: ccode.append(l) + ccode.append('\n') # append our source file ccode.append(open(get_module_file('genexterns.c')).read()) Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Tue Oct 30 18:55:55 2007 @@ -282,38 +282,6 @@ # ha, ha pass - # XXX not needed anymore - def flavored_malloc(self, opr): - flavor = opr.op.args[0].value - type_ = opr.rettype[:-1] #XXX stripping of * - if flavor == "raw": - self.codewriter.malloc(opr.retref, type_) - elif flavor == "stack": - self.codewriter.alloca(opr.retref, type_) - else: - raise NotImplementedError - - def flavored_malloc_varsize(self, opr): - flavor = opr.op.args[0].value - if flavor == "raw": - arg_type = opr.op.args[1].value - node = self.db.obj2node[arg_type] - self.db.gcpolicy.var_zeromalloc(self.codewriter, opr.retref, - opr.rettype, node, opr.argrefs[2], - atomic=arg_type._is_atomic()) - else: - raise NotImplementedError - - def flavored_free(self, opr): - flavor = opr.op.args[0].value - if flavor == "raw": - self.codewriter.free(opr.argtypes[1], opr.argrefs[1]) - elif flavor == "stack": - self.codewriter.comment('***Skipping free of stack allocated data') - else: - raise NotImplementedError - # XXX /not needed anymore - def call_boehm_gc_alloc(self, opr): word = self.db.get_machine_word() self.codewriter.call(opr.retref, 'sbyte*', '%pypy_malloc', @@ -407,7 +375,8 @@ else: assert isinstance(TYPE, lltype.FixedSizeArray) lengthref = TYPE.length - XXX # no tests for this + XXX # no tests for this - is it valid even since known at compile time ???? + self.codewriter.load(opr.retref, opr.rettype, lengthref) # array | fixedsizearray @@ -428,7 +397,6 @@ # getelementptr gets a pointer to the right type, except the generated code really expected # an array of size 1... so we just cast it - #assert isinstance(opr.op.result.concretetype.TO, lltype.FixedSizeArray) #XXX why? element_type = self.db.repr_type(op.result.concretetype.TO.OF) + '*' self.codewriter.cast(opr.retref, element_type, tmpvar, opr.rettype) @@ -444,7 +412,6 @@ # getelementptr gets a pointer to the right type, except the generated code really expected # an array of size 1... so we just cast it - #assert isinstance(opr.op.result.concretetype.TO, lltype.FixedSizeArray) #XXX why? element_type = self.db.repr_type(opr.op.result.concretetype.TO.OF) + '*' self.codewriter.cast(opr.retref, element_type, tmpvar, opr.rettype) @@ -460,15 +427,6 @@ element_type = self.db.repr_type(opr.op.result.concretetype.TO.OF) + '*' self.codewriter.cast(opr.retref, element_type, tmpvar, opr.rettype) -# + if isinstance(opr.op.result.concretetype, lltype.FixedSizeArray): -# + tmpvar = self._tmp() -# + self.codewriter.getelementptr(tmpvar, arraytype, array, [(self.word, incr)]) -# + element_type = self.db.repr_type(opr.op.result.concretetype.TO.OF) + '*' -# + self.codewriter.cast(opr.retref, element_type, tmpvar, opr.rettype) -# + else: -# + self.codewriter.getelementptr(opr.retref, arraytype, array, [(self.word, incr)], getptr=False) -# + - def adr_delta(self, opr): addr1, addr2 = self._tmp(2) self.codewriter.cast(addr1, opr.argtypes[0], opr.argrefs[0], self.word) Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Tue Oct 30 18:55:55 2007 @@ -8,6 +8,7 @@ from pypy.rpython.lltypesystem.lltype import typeOf optimize_tests = False +native_llvm_backend = True MINIMUM_LLVM_VERSION = 1.9 FLOAT_PRECISION = 8 @@ -72,7 +73,8 @@ 'translation.llvm.isolate': isolate, 'translation.backendopt.none': not optimize, 'translation.gc': 'boehm', - } + 'translation.llvm_via_c' : not native_llvm_backend +} options.update(extra_opts) config.set(**options) driver = TranslationDriver(config=config) From rxe at codespeak.net Tue Oct 30 18:57:12 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 30 Oct 2007 18:57:12 +0100 (CET) Subject: [pypy-svn] r48208 - pypy/dist/pypy/translator/llvm/module Message-ID: <20071030175712.CDE2E81D4@code0.codespeak.net> Author: rxe Date: Tue Oct 30 18:57:12 2007 New Revision: 48208 Modified: pypy/dist/pypy/translator/llvm/module/boehm.h Log: comment this out for now - not sure if it is even possible anymore. no point enlarging our ll files and increasing test times for it Modified: pypy/dist/pypy/translator/llvm/module/boehm.h ============================================================================== --- pypy/dist/pypy/translator/llvm/module/boehm.h (original) +++ pypy/dist/pypy/translator/llvm/module/boehm.h Tue Oct 30 18:57:12 2007 @@ -45,27 +45,27 @@ // fails badly (segfaults) when a reference to the data is kept // around and used (much) later. -#define ringbufsize 1024 -#define ringbufentry_maxsize 16 +/* #define ringbufsize 1024 */ +/* #define ringbufentry_maxsize 16 */ -static char ringbufdata[ringbufsize + ringbufentry_maxsize]; -static long ringbufindex = 0; +/* static char ringbufdata[ringbufsize + ringbufentry_maxsize]; */ +/* static long ringbufindex = 0; */ -char *pypy_malloc_ringbuffer(long size) { - if (size <= ringbufentry_maxsize) { //test expected to be optimized away during compile time - ringbufindex = (ringbufindex + ringbufentry_maxsize) & (ringbufsize - 1); - return &ringbufdata[ringbufindex]; - } else { - return GC_MALLOC(size); - } -} +/* char *pypy_malloc_ringbuffer(long size) { */ +/* if (size <= ringbufentry_maxsize) { //test expected to be optimized away during compile time */ +/* ringbufindex = (ringbufindex + ringbufentry_maxsize) & (ringbufsize - 1); */ +/* return &ringbufdata[ringbufindex]; */ +/* } else { */ +/* return GC_MALLOC(size); */ +/* } */ +/* } */ -char *pypy_malloc_atomic_ringbuffer(long size) { - if (size <= ringbufentry_maxsize) { //test expected to be optimized away during compile time - ringbufindex = (ringbufindex + ringbufentry_maxsize) & (ringbufsize - 1); - return &ringbufdata[ringbufindex]; - } else { - return GC_MALLOC_ATOMIC(size); - } -} +/* char *pypy_malloc_atomic_ringbuffer(long size) { */ +/* if (size <= ringbufentry_maxsize) { //test expected to be optimized away during compile time */ +/* ringbufindex = (ringbufindex + ringbufentry_maxsize) & (ringbufsize - 1); */ +/* return &ringbufdata[ringbufindex]; */ +/* } else { */ +/* return GC_MALLOC_ATOMIC(size); */ +/* } */ +/* } */ From rxe at codespeak.net Tue Oct 30 19:25:06 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 30 Oct 2007 19:25:06 +0100 (CET) Subject: [pypy-svn] r48209 - pypy/dist/pypy/translator/llvm Message-ID: <20071030182506.7586A81D1@code0.codespeak.net> Author: rxe Date: Tue Oct 30 19:25:05 2007 New Revision: 48209 Modified: pypy/dist/pypy/translator/llvm/modwrapper.py Log: missing return statement Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Tue Oct 30 19:25:05 2007 @@ -119,6 +119,9 @@ raise Exception("struct must be of same kind") if F0 not in TO_CTYPES: raise Exception("struct must be of primitve type") + + return to_tuple % (len(fields), TO_CTYPES[F0]) + elif isinstance(T, lltype.Ptr) and isinstance(T.TO, lltype.Array): OF = T.TO.OF if OF not in TO_CTYPES: From cfbolz at codespeak.net Tue Oct 30 19:28:35 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 30 Oct 2007 19:28:35 +0100 (CET) Subject: [pypy-svn] r48210 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071030182835.5461981D1@code0.codespeak.net> Author: cfbolz Date: Tue Oct 30 19:28:34 2007 New Revision: 48210 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/primitives.py Log: turn the primitives into one big switch as well Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Tue Oct 30 19:28:34 2007 @@ -167,19 +167,28 @@ method = receiverclassshadow.lookup(selector) # XXX catch MethodNotFound here and send doesNotUnderstand: if method.primitive: + # the primitive pushes the result (if any) onto the stack itself + code = method.primitive if interp.should_trace(): - print "%sActually calling primitive %d" % (interp._last_indent,method.primitive,) - func = primitives.prim_table[method.primitive] - try: - # note: argcount does not include rcvr - w_result = func(interp, argcount) - except primitives.PrimitiveFailedError: - if interp.should_trace(): - print "PRIMITIVE FAILED: %d %s" % (method.primitive, selector,) - pass # ignore this error and fall back to the Smalltalk version + print "%sActually calling primitive %d" % (interp._last_indent, code,) + if objectmodel.we_are_translated(): + for i, func in primitives.unrolling_prim_table: + if i == code: + try: + func(interp, argcount) + return + except primitives.PrimitiveFailedError: + break else: - # the primitive pushes the result (if any) onto the stack itself - return + func = primitives.prim_table[code] + try: + # note: argcount does not include rcvr + w_result = func(interp, argcount) + return + except primitives.PrimitiveFailedError: + if interp.should_trace(): + print "PRIMITIVE FAILED: %d %s" % (method.primitive, selector,) + pass # ignore this error and fall back to the Smalltalk version start = len(self.stack) - argcount assert start >= 0 # XXX check in the Blue Book what to do in this case arguments = self.stack[start:] Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Tue Oct 30 19:28:34 2007 @@ -7,7 +7,7 @@ from pypy.lang.smalltalk import constants from pypy.lang.smalltalk.error import PrimitiveFailedError, \ PrimitiveNotYetWrittenError -from pypy.rlib import rarithmetic +from pypy.rlib import rarithmetic, unroll def assert_bounds(n0, minimum, maximum): @@ -35,6 +35,7 @@ # Squeak has primitives all the way up to 575 # So all optional primitives will default to the bytecode implementation prim_table = [make_failing(i) for i in range(576)] +prim_table_implemented_only = [] # indicates that what is pushed is an index1, but it is unwrapped and # converted to an index0 @@ -104,6 +105,7 @@ interp.w_active_context.push(w_result) wrapped.func_name = "wrap_prim_" + name prim_table[code] = wrapped + prim_table_implemented_only.append((code, wrapped)) return func return decorator @@ -737,3 +739,5 @@ globals()["INST_VAR_AT_%d" % (i-264)] = i make_prim(i) + +unrolling_prim_table = unroll.unrolling_iterable(prim_table_implemented_only) From cfbolz at codespeak.net Tue Oct 30 20:47:24 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 30 Oct 2007 20:47:24 +0100 (CET) Subject: [pypy-svn] r48211 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071030194724.3559581BC@code0.codespeak.net> Author: cfbolz Date: Tue Oct 30 20:47:23 2007 New Revision: 48211 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py Log: try to reduce the number of warnings when translating. This seems to get rid of most of them. Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Tue Oct 30 20:47:23 2007 @@ -20,6 +20,18 @@ def gethash(self): raise NotImplementedError + def at0(self, index0): + raise NotImplementedError + + def atput0(self, index0, w_value): + raise NotImplementedError + + def fetch(self, n0): + raise NotImplementedError + + def store(self, n0, w_value): + raise NotImplementedError + def invariant(self): return True @@ -378,6 +390,7 @@ self.pc = 0 assert isinstance(w_home, W_MethodContext) self.w_home = w_home + assert w_sender is None or isinstance(w_sender, W_ContextPart) self.w_sender = w_sender def receiver(self): Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Tue Oct 30 20:47:23 2007 @@ -626,7 +626,7 @@ # byteodes to invoke this primitive to find them (hence +2) initialip = frame.pc + 2 w_new_context = model.W_BlockContext( - w_method_context, objtable.w_nil, argcnt, initialip) + w_method_context, None, argcnt, initialip) return w_new_context def finalize_block_ctx(interp, w_block_ctx, frame): From cfbolz at codespeak.net Tue Oct 30 21:21:42 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 30 Oct 2007 21:21:42 +0100 (CET) Subject: [pypy-svn] r48212 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071030202142.37C5681BC@code0.codespeak.net> Author: cfbolz Date: Tue Oct 30 21:21:40 2007 New Revision: 48212 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py Log: revert the callPrimitive2 stuff, since it was semantically wrong: if you add tagged_maxint to tagged_maxint you want a long int, not a float. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Tue Oct 30 21:21:40 2007 @@ -369,54 +369,44 @@ self.callPrimitive(primitive2, selector, argcount, interp) def bytecodePrimAdd(self, interp): - self.callPrimitive2(primitives.ADD, - primitives.FLOAT_ADD, - "+", 1, interp) + self.callPrimitive(primitives.ADD, + "+", 1, interp) def bytecodePrimSubtract(self, interp): - self.callPrimitive2(primitives.SUBTRACT, - primitives.FLOAT_SUBTRACT, - "-", 1, interp) + self.callPrimitive(primitives.SUBTRACT, + "-", 1, interp) def bytecodePrimLessThan(self, interp): - self.callPrimitive2(primitives.LESSTHAN, - primitives.FLOAT_LESSTHAN, - "<", 1, interp) + self.callPrimitive(primitives.LESSTHAN, + "<", 1, interp) def bytecodePrimGreaterThan(self, interp): - self.callPrimitive2(primitives.GREATERTHAN, - primitives.FLOAT_GREATERTHAN, - ">", 1, interp) + self.callPrimitive(primitives.GREATERTHAN, + ">", 1, interp) def bytecodePrimLessOrEqual(self, interp): - self.callPrimitive2(primitives.LESSOREQUAL, - primitives.FLOAT_LESSOREQUAL, - "<=", 1, interp) + self.callPrimitive(primitives.LESSOREQUAL, + "<=", 1, interp) def bytecodePrimGreaterOrEqual(self, interp): - self.callPrimitive2(primitives.GREATEROREQUAL, - primitives.FLOAT_GREATEROREQUAL, - ">=", 1, interp) + self.callPrimitive(primitives.GREATEROREQUAL, + ">=", 1, interp) def bytecodePrimEqual(self, interp): - self.callPrimitive2(primitives.EQUAL, - primitives.FLOAT_EQUAL, + self.callPrimitive(primitives.EQUAL, "=", 1, interp) def bytecodePrimNotEqual(self, interp): - self.callPrimitive2(primitives.NOTEQUAL, - primitives.FLOAT_NOTEQUAL, - "~=", 1, interp) + self.callPrimitive(primitives.NOTEQUAL, + "~=", 1, interp) def bytecodePrimMultiply(self, interp): - self.callPrimitive2(primitives.MULTIPLY, - primitives.FLOAT_MULTIPLY, - "*", 1, interp) + self.callPrimitive(primitives.MULTIPLY, + "*", 1, interp) def bytecodePrimDivide(self, interp): - self.callPrimitive2(primitives.DIVIDE, - primitives.FLOAT_DIVIDE, - "/", 1, interp) + self.callPrimitive(primitives.DIVIDE, + "/", 1, interp) def bytecodePrimMod(self, interp): self.callPrimitive(primitives.MOD, "\\\\", 1, interp) From rxe at codespeak.net Wed Oct 31 00:48:54 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 31 Oct 2007 00:48:54 +0100 (CET) Subject: [pypy-svn] r48219 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071030234854.BF87D8177@code0.codespeak.net> Author: rxe Date: Wed Oct 31 00:48:53 2007 New Revision: 48219 Modified: pypy/dist/pypy/translator/llvm/buildllvm.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/modwrapper.py pypy/dist/pypy/translator/llvm/test/test_runtest.py Log: try and make the interface richer Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Wed Oct 31 00:48:53 2007 @@ -5,7 +5,7 @@ from pypy.translator.tool import stdoutcapture from pypy.translator.llvm.log import log -from pypy.translator.llvm.modwrapper import write_ctypes_module +from pypy.translator.llvm.modwrapper import CtypesModule def llvm_is_on_path(): if py.path.local.sysfind("llvm-as") is None or \ @@ -153,7 +153,7 @@ try: self.execute_cmds() - modname = write_ctypes_module(self.genllvm, "%s.so" % base) + modname = CtypesModule(self.genllvm, "%s.so" % base).create() finally: self.lastdir.chdir() Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Wed Oct 31 00:48:53 2007 @@ -250,7 +250,7 @@ from pypy.translator.tool.cbuild import import_module_from_directory mod = import_module_from_directory(dirpath, modname) - wrap_fun = getattr(mod, 'pypy_' + self.entry_func_name + "_wrapper") + wrap_fun = getattr(mod, 'entrypoint') return mod, wrap_fun def compile_standalone(self, exe_name): Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Wed Oct 31 00:48:53 2007 @@ -1,3 +1,4 @@ +import py import ctypes from pypy.rpython.lltypesystem import lltype @@ -35,35 +36,50 @@ return code -TO_CTYPES = {lltype.Bool: "ctypes.c_int", - lltype.SingleFloat: "ctypes.c_float", - lltype.Float: "ctypes.c_double", - lltype.Char: "ctypes.c_char", - lltype.Signed: "ctypes.c_int", - lltype.Unsigned: "ctypes.c_uint", - lltype.SignedLongLong: "ctypes.c_longlong", - lltype.UnsignedLongLong: "ctypes.c_ulonglong", - lltype.Void: None - } - -def to_ctype(T): - if isinstance(T, lltype.Ptr): - return "ctypes.c_void_p" - else: - return TO_CTYPES[T] -def build_lltype_to_ctypes_to_res(T): - identity = """ -def tores(res): +class CtypesModule: + """ use ctypes to create a temporary module """ + + prolog = """ +import ctypes +from os.path import join, dirname, realpath + +_c = ctypes.CDLL(join(dirname(realpath(__file__)), "%s")) + +raised = _c.__entrypoint__raised_LLVMException +raised.argtypes = [] +raised.restype = ctypes.c_int + +GC_get_heap_size_wrapper = _c.GC_get_heap_size +GC_get_heap_size_wrapper.argtypes = [] +GC_get_heap_size_wrapper.restype = ctypes.c_int + +startup_code = _c.ctypes_RPython_StartupCode +startup_code.argtypes = [] +startup_code.restype = ctypes.c_int + +_setup = False + +class LLVMException(Exception): + pass + +def entrypoint(*args): + global _setup + if not _setup: + if not startup_code(): + raise LLVMException("Failed to startup") + _setup = True + args = to_llargs(args) + result = __entrypoint__(*args) + if raised(): + raise LLVMException("Exception raised") + return ll_to_res(result) + +def identity(res): return res - """ - to_bool = """ -def tores(res): +def to_bool(res): return bool(res) - """ - - to_str = """ class Chars(ctypes.Structure): _fields_ = [("size", ctypes.c_int), @@ -73,115 +89,117 @@ _fields_ = [("hash", ctypes.c_int), ("array", Chars)] -def tores(res): +def to_str(res): if res: s = ctypes.cast(res, ctypes.POINTER(STR)).contents return ctypes.string_at(res + (STR.array.offset + Chars.data.offset), s.array.size) else: return None - """ - to_tuple = """ -def tores(res): +def to_tuple(res, size, C_TYPE, action): if res: - t = ctypes.cast(res, ctypes.POINTER(%s * %s)).contents - return tuple(t) + t = ctypes.cast(res, ctypes.POINTER(C_TYPE * size)).contents + return tuple([action(ii) for ii in t]) else: return None - """ - - to_list = """ -def tores(res): +def to_list(res, C_TYPE, action): if res: size = ctypes.cast(res, ctypes.POINTER(ctypes.c_int)).contents.value class Array(ctypes.Structure): _fields_ = [("size", ctypes.c_int), - ("data", %s * size)] + ("data", C_TYPE * size)] array = ctypes.cast(res, ctypes.POINTER(Array)).contents - return list(array.data) + return [action(array.data[ii]) for ii in range(size)] else: return None - """ - - from pypy.rpython.lltypesystem.rstr import STR - - if T is lltype.Bool: - return to_bool - elif isinstance(T, lltype.Ptr) and T.TO is STR: - return to_str - elif isinstance(T, lltype.Ptr) and isinstance(T.TO, lltype.Struct): - fields = [getattr(T.TO, name) for name in T.TO._names_without_voids()] - if fields: - F0 = fields[0] - for F in fields[1:]: - if F0 is not F: - raise Exception("struct must be of same kind") - if F0 not in TO_CTYPES: - raise Exception("struct must be of primitve type") - - return to_tuple % (len(fields), TO_CTYPES[F0]) - - elif isinstance(T, lltype.Ptr) and isinstance(T.TO, lltype.Array): - OF = T.TO.OF - if OF not in TO_CTYPES: - raise Exception("struct must be of primitve type") - - return to_list % TO_CTYPES[OF] - else: - return identity - -def write_ctypes_module(genllvm, dllname): - """ use ctypes to create a temporary module """ - - template = """ -import ctypes -from os.path import join, dirname, realpath -_c = ctypes.CDLL(join(dirname(realpath(__file__)), "%(dllname)s")) - -_setup = False - -class LLVMException(Exception): - pass - -%(name)s = _c.__entrypoint__%(name)s -%(name)s.argtypes = %(args)s -%(name)s.restype = %(returntype)s - -%(name)s_raised = _c.__entrypoint__raised_LLVMException -%(name)s_raised.argtypes = [] -%(name)s_raised.restype = ctypes.c_int - -GC_get_heap_size_wrapper = _c.GC_get_heap_size -GC_get_heap_size_wrapper.argtypes = [] -GC_get_heap_size_wrapper.restype = ctypes.c_int - -startup_code = _c.ctypes_RPython_StartupCode -startup_code.argtypes = [] -startup_code.restype = ctypes.c_int - -def %(name)s_wrapper(*args): - global _setup - if not _setup: - if not startup_code(): - raise LLVMException("Failed to startup") - _setup = True - result = %(name)s(*args) - if %(name)s_raised(): - raise LLVMException("Exception raised") - return tores(result) """ - basename = genllvm.filename.purebasename + '_wrapper.py' - modfilename = genllvm.filename.new(basename = basename) - name = genllvm.entrynode.ref.strip("%") - - g = genllvm.entrynode.graph - rt = g.returnblock.inputargs[0].concretetype - returntype = to_ctype(rt) - inputargtypes = [TO_CTYPES[a.concretetype] for a in g.startblock.inputargs] - args = '[%s]' % ", ".join(inputargtypes) + epilog = """ +to_llargs = %(to_llargs)s +ll_to_res = %(ll_to_res)s +__entrypoint__ = _c.__entrypoint__%(name)s +__entrypoint__.argtypes = %(args)s +__entrypoint__.restype = %(returntype)s + """ + def __init__(self, genllvm, dllname): + self.genllvm = genllvm + self.dllname = dllname + basename = self.genllvm.filename.purebasename + '_wrapper.py' + self.modfilename = genllvm.filename.new(basename=basename) + self.count = 0 + + def create(self): + self.file = open(str(self.modfilename), 'w') + self.file.write(self.prolog % self.dllname) + + g = self.genllvm.entrynode.graph + name = "pypy_" + g.name - modfilename.write(template % locals() + build_lltype_to_ctypes_to_res(rt)) + inputargtypes = [self.to_ctype(a.concretetype) for a in g.startblock.inputargs] + to_llargs = 'identity' + args = '[%s]' % ', '.join(inputargtypes) - return modfilename.purebasename + RT = g.returnblock.inputargs[0].concretetype + returntype, ll_to_res = self.build_lltype_to_ctypes_to_res(RT) + + self.file.write(self.epilog % locals()) + self.file.close() + return self.modfilename.purebasename + + def create_simple_closure(self, args, code): + name = 'tmpfunction_%s' % self.count + self.count += 1 + self.file.write('\n') + self.file.write('def %s(%s): return %s' % (name, args, code)) + self.file.write('\n') + return name + + def build_lltype_to_ctypes_to_res(self, T): + from pypy.rpython.lltypesystem.rstr import STR + + if T is lltype.Bool: + action = 'to_bool' + + elif isinstance(T, lltype.Ptr) and T.TO is STR: + action = 'to_str' + + elif isinstance(T, lltype.Ptr) and isinstance(T.TO, lltype.Struct): + fields = [getattr(T.TO, name) for name in T.TO._names_without_voids()] + if fields: + F0 = fields[0] + _c_type, _action = self.build_lltype_to_ctypes_to_res(F0) + action = self.create_simple_closure('res', 'to_tuple(res, %s, %s, %s)' % (len(fields), + _c_type, + _action)) + + elif isinstance(T, lltype.Ptr) and isinstance(T.TO, lltype.Array): + OF = T.TO.OF + _c_type, _action = self.build_lltype_to_ctypes_to_res(OF) + action = self.create_simple_closure('res', 'to_list(res, %s, %s)' % (_c_type, _action)) + + else: + assert not isinstance(T, lltype.Ptr) + action = 'identity' + + c_type = self.to_ctype(T) + return c_type, action + + def to_ctype(self, T): + TO_CTYPES = {lltype.Bool: "ctypes.c_byte", + lltype.SingleFloat: "ctypes.c_float", + lltype.Float: "ctypes.c_double", + lltype.Char: "ctypes.c_char", + lltype.Signed: "ctypes.c_int", + lltype.Unsigned: "ctypes.c_uint", + lltype.SignedLongLong: "ctypes.c_longlong", + lltype.UnsignedLongLong: "ctypes.c_ulonglong", + lltype.Void: None, + lltype.UniChar: "ctypes.c_uint", + } + + if isinstance(T, lltype.Ptr): + return 'ctypes.c_void_p' + else: + return TO_CTYPES[T] + Modified: pypy/dist/pypy/translator/llvm/test/test_runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_runtest.py Wed Oct 31 00:48:53 2007 @@ -8,3 +8,18 @@ y = 1 + x return None assert self.interpret(fn,[1]) == None + + def test_list_of_strings(self): + def fn(): + return ['abc', 'def'] + assert self.interpret(fn, []) == ['abc', 'def'] + + def test_list_of_bools(self): + def fn(): + return [True, True, False] + assert self.interpret(fn, []) == [True, True, False] + + def test_tuple_of_list_of_stringss(self): + def fn(): + return ['abc', 'def'], ['abc', 'def'] + assert self.interpret(fn, []) == (['abc', 'def'], ['abc', 'def']) From arigo at codespeak.net Wed Oct 31 14:51:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 31 Oct 2007 14:51:06 +0100 (CET) Subject: [pypy-svn] r48225 - pypy/dist/pypy/jit/timeshifter/test Message-ID: <20071031135106.26F1D81B7@code0.codespeak.net> Author: arigo Date: Wed Oct 31 14:51:05 2007 New Revision: 48225 Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Log: Fix the test, add a new one. Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Wed Oct 31 14:51:05 2007 @@ -1185,7 +1185,24 @@ res = self.timeshift(f, [0, 2, 7, 5], [0], policy=P_NOVIRTUAL) assert res == 44 - self.check_insns({'int_mul': 1, 'int_add': 1, 'int_sub': 1}) + self.check_insns({'int_mul': 1, 'int_add': 1, 'int_sub': 1}) + + def test_deepfrozen_interior(self): + T = lltype.Struct('T', ('x', lltype.Signed)) + A = lltype.Array(T) + S = lltype.GcStruct('S', ('a', A)) + s = lltype.malloc(S, 3) + s.a[2].x = 42 + def f(n): + s1 = hint(s, variable=True) + s1 = hint(s1, deepfreeze=True) + return s1.a[n].x + + # malloc-remove the interior ptr + res = self.timeshift(f, [2], [0], policy=P_NOVIRTUAL, + backendoptimize=True) + assert res == 42 + self.check_insns({}) def test_compile_time_const_tuple(self): d = {(4, 5): 42, (6, 7): 12} @@ -1193,7 +1210,9 @@ d1 = hint(d, deepfreeze=True) return d1[a, b] - res = self.timeshift(f, [4, 5], [0, 1], policy=P_NOVIRTUAL) + # malloc-remove the interior ptr + res = self.timeshift(f, [4, 5], [0, 1], policy=P_NOVIRTUAL, + backendoptimize=True) assert res == 42 self.check_insns({}) From rxe at codespeak.net Wed Oct 31 22:13:47 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 31 Oct 2007 22:13:47 +0100 (CET) Subject: [pypy-svn] r48229 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071031211347.D50DD8151@code0.codespeak.net> Author: rxe Date: Wed Oct 31 22:13:44 2007 New Revision: 48229 Modified: pypy/dist/pypy/translator/llvm/test/test_symbolic.py Log: fix translator.llvm.test.test_symbolic: test_sizeof_constsize_struct by arigo r48201 Modified: pypy/dist/pypy/translator/llvm/test/test_symbolic.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_symbolic.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_symbolic.py Wed Oct 31 22:13:44 2007 @@ -74,7 +74,6 @@ assert res == 1234501234 def test_sizeof_constsize_struct(): - py.test.skip("failing in genc too") # _not_ a GcStruct, since we want to raw_malloc it STRUCT = lltype.Struct("s", ("x", lltype.Signed), ("y", lltype.Signed)) STRUCTPTR = lltype.Ptr(STRUCT) From rxe at codespeak.net Wed Oct 31 22:29:55 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 31 Oct 2007 22:29:55 +0100 (CET) Subject: [pypy-svn] r48230 - pypy/dist/pypy/translator/llvm Message-ID: <20071031212955.6885F8151@code0.codespeak.net> Author: rxe Date: Wed Oct 31 22:29:53 2007 New Revision: 48230 Modified: pypy/dist/pypy/translator/llvm/modwrapper.py Log: support richer inputs (like strings, unichars) for testing Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Wed Oct 31 22:29:53 2007 @@ -1,6 +1,7 @@ import py import ctypes from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem.rstr import STR def _noresult(returntype): r = returntype.strip() @@ -26,14 +27,6 @@ else: code += entrycode % locals() - print 'XXXXXXXXXXXXXXX' - print returntype - print entrypointname - print noresult - print entrynode.graph.returnblock.inputargs[0].concretetype - print code - print 'XXXXXXXXXXXXXXX' - return code @@ -69,7 +62,7 @@ if not startup_code(): raise LLVMException("Failed to startup") _setup = True - args = to_llargs(args) + args = [f(a) for a, f in zip(args, to_llargs)] result = __entrypoint__(*args) if raised(): raise LLVMException("Exception raised") @@ -78,18 +71,41 @@ def identity(res): return res +def from_unichar(arg): + return ord(arg) + def to_bool(res): return bool(res) -class Chars(ctypes.Structure): - _fields_ = [("size", ctypes.c_int), - ("data", ctypes.c_char * 1)] - -class STR(ctypes.Structure): - _fields_ = [("hash", ctypes.c_int), - ("array", Chars)] +def to_unichar(res): + return unichr(res) + +def from_str(arg): + # XXX wont work over isolate : arg should be converted into a string first + n = len(arg.chars) + class Chars(ctypes.Structure): + _fields_ = [("size", ctypes.c_int), + ("data", ctypes.c_byte * n)] + + class STR(ctypes.Structure): + _fields_ = [("hash", ctypes.c_int), + ("chars", Chars)] + s = STR() + s.hash = 0 + s.chars.size = len(arg.chars) + for ii in range(s.chars.size): + s.chars.data[ii] = ord(arg.chars[ii]) + return ctypes.addressof(s) def to_str(res): + class Chars(ctypes.Structure): + _fields_ = [("size", ctypes.c_int), + ("data", ctypes.c_char * 1)] + + class STR(ctypes.Structure): + _fields_ = [("hash", ctypes.c_int), + ("array", Chars)] + if res: s = ctypes.cast(res, ctypes.POINTER(STR)).contents return ctypes.string_at(res + (STR.array.offset + Chars.data.offset), s.array.size) @@ -116,12 +132,28 @@ """ epilog = """ -to_llargs = %(to_llargs)s -ll_to_res = %(ll_to_res)s __entrypoint__ = _c.__entrypoint__%(name)s + +# %(RT)r +to_llargs = %(to_llargs)s __entrypoint__.argtypes = %(args)s + +# %(ARGS)r +ll_to_res = %(ll_to_res)s __entrypoint__.restype = %(returntype)s """ + TO_CTYPES = {lltype.Bool: "ctypes.c_byte", + lltype.SingleFloat: "ctypes.c_float", + lltype.Float: "ctypes.c_double", + lltype.Char: "ctypes.c_char", + lltype.Signed: "ctypes.c_int", + lltype.Unsigned: "ctypes.c_uint", + lltype.SignedLongLong: "ctypes.c_longlong", + lltype.UnsignedLongLong: "ctypes.c_ulonglong", + lltype.Void: None, + lltype.UniChar: "ctypes.c_uint", + } + def __init__(self, genllvm, dllname): self.genllvm = genllvm self.dllname = dllname @@ -136,9 +168,11 @@ g = self.genllvm.entrynode.graph name = "pypy_" + g.name - inputargtypes = [self.to_ctype(a.concretetype) for a in g.startblock.inputargs] - to_llargs = 'identity' + ARGS = [a.concretetype for a in g.startblock.inputargs] + inputargtypes, to_llargs = self.build_args_to_ctypes_to_lltype(ARGS) + args = '[%s]' % ', '.join(inputargtypes) + to_llargs = '[%s]' % ', '.join(to_llargs) RT = g.returnblock.inputargs[0].concretetype returntype, ll_to_res = self.build_lltype_to_ctypes_to_res(RT) @@ -155,12 +189,33 @@ self.file.write('\n') return name - def build_lltype_to_ctypes_to_res(self, T): - from pypy.rpython.lltypesystem.rstr import STR + def build_args_to_ctypes_to_lltype(self, ARGS): + ctype_s = [] + actions = [] + + for A in ARGS: + ctype_s.append(self.to_ctype(A)) + + if A is lltype.UniChar: + action = 'from_unichar' + + elif isinstance(A, lltype.Ptr) and A.TO is STR: + action = 'from_str' + else: + assert A in self.TO_CTYPES + action = 'identity' + actions.append(action) + + return ctype_s, actions + + def build_lltype_to_ctypes_to_res(self, T): if T is lltype.Bool: action = 'to_bool' + elif T is lltype.UniChar: + action = 'to_unichar' + elif isinstance(T, lltype.Ptr) and T.TO is STR: action = 'to_str' @@ -186,20 +241,8 @@ return c_type, action def to_ctype(self, T): - TO_CTYPES = {lltype.Bool: "ctypes.c_byte", - lltype.SingleFloat: "ctypes.c_float", - lltype.Float: "ctypes.c_double", - lltype.Char: "ctypes.c_char", - lltype.Signed: "ctypes.c_int", - lltype.Unsigned: "ctypes.c_uint", - lltype.SignedLongLong: "ctypes.c_longlong", - lltype.UnsignedLongLong: "ctypes.c_ulonglong", - lltype.Void: None, - lltype.UniChar: "ctypes.c_uint", - } - if isinstance(T, lltype.Ptr): return 'ctypes.c_void_p' else: - return TO_CTYPES[T] + return self.TO_CTYPES[T]