From benjamin at codespeak.net Thu Jul 1 00:14:49 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 00:14:49 +0200 (CEST) Subject: [pypy-svn] r75707 - in pypy/branch/fast-forward/pypy: objspace/std objspace/std/test rlib Message-ID: <20100630221449.5172A282B9C@codespeak.net> Author: benjamin Date: Thu Jul 1 00:14:47 2010 New Revision: 75707 Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py pypy/branch/fast-forward/pypy/objspace/std/newformat.py pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Log: cobble some code together for float formatting Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Thu Jul 1 00:14:47 2010 @@ -1,7 +1,7 @@ import operator, new from pypy.interpreter import gateway from pypy.interpreter.error import OperationError -from pypy.objspace.std import model +from pypy.objspace.std import model, newformat from pypy.objspace.std.multimethod import FailedToImplementArgs from pypy.objspace.std.model import registerimplementation, W_Object from pypy.objspace.std.register_all import register_all @@ -106,6 +106,9 @@ def str__Float(space, w_float): return float2string(space, w_float, "%.12g") +def format__Float_ANY(space, w_float, w_spec): + return newformat.get_formatter(space, w_spec).format_float(w_float) + # ____________________________________________________________ # A mess to handle all cases of float comparison without relying # on delegation, which can unfortunately loose precision when Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Thu Jul 1 00:14:47 2010 @@ -419,15 +419,15 @@ self._calc_padding(string, length) return space.wrap(self._pad(string)) - def _get_locale(self, locale_kind): + def _get_locale(self, tp): space = self.space - if locale_kind == CURRENT_LOCALE: + if tp == "n": dec, thousands, grouping = rlocale.numeric_formatting() - elif locale_kind == DEFAULT_LOCALE: + elif self._thousands_sep: dec = "." thousands = "," grouping = "\3\0" - elif locale_kind == NO_LOCALE: + else: dec = "." thousands = "" grouping = "\256" @@ -445,7 +445,7 @@ spec.n_digits = n_number - n_remainder - has_dec spec.n_prefix = n_prefix spec.n_lpadding = 0 - spec.n_decimal = 1 if has_dec else 0 + spec.n_decimal = int(has_dec) spec.n_remainder = n_remainder spec.n_spadding = 0 spec.n_rpadding = 0 @@ -548,7 +548,7 @@ def _fill_number(self, spec, num, to_digits, n_digits, to_prefix, fill_char, - upper): + to_remainder, upper): out = self._builder() if spec.n_lpadding: out.append_multiple_char(fill_char, spec.n_lpadding) @@ -563,16 +563,16 @@ out.append_multiple_char(fill_char, spec.n_spadding) if spec.n_digits != 0: if self._loc_thousands: - num = self._grouped_digits + digits = self._grouped_digits else: - num = num[to_digits:] + digits = num[to_digits:to_digits + spec.n_digits] if upper: - num = self._upcase_string(num) - out.append(num) + digits = self._upcase_string(digits) + out.append(digits) if spec.n_decimal: out.append(".") if spec.n_remainder: - out.append(num[0]) + out.append(num[to_remainder:]) if spec.n_rpadding: out.append_multiple_char(fill_char, spec.n_rpadding) return self.space.wrap(out.build()) @@ -598,6 +598,7 @@ result = chr(value) n_digits = 1 n_remainder = 1 + to_remainder = 0 n_prefix = 0 to_prefix = 0 to_numeric = 0 @@ -628,20 +629,15 @@ to_prefix += 1 n_digits = len(result) - skip_leading n_remainder = 0 + to_remainder = 0 to_numeric = skip_leading - if tp == "n": - locale_kind = CURRENT_LOCALE - elif self._thousands_sep: - locale_kind = DEFAULT_LOCALE - else: - locale_kind = NO_LOCALE - self._get_locale(locale_kind) + self._get_locale(tp) spec = self._calc_num_width(n_prefix, sign_char, to_numeric, n_digits, n_remainder, False, result) fill = " " if self._fill_char == "\0" else self._fill_char upper = self._type == "X" return self._fill_number(spec, result, to_numeric, n_digits, to_prefix, - fill, upper) + fill, to_remainder, upper) def _long_to_base(self, base, value): prefix = "" @@ -718,24 +714,63 @@ tp == "G" or tp == "%"): w_float = space.float(w_num) - self._format_float(w_float) + return self._format_float(w_float) else: self._unknown_presentation("int" if kind == INT_KIND else "long") + def _parse_number(self, s, i): + length = len(s) + while i < length and "0" <= s[i] <= "9": + i += 1 + rest = i + dec_point = i < length and s[i] == "." + if dec_point: + rest += 1 + return dec_point, rest + def _format_float(self, w_float): space = self.space + flags = 0 + default_precision = 6 if self._alternate: msg = "alternate form not allowed in float formats" raise OperationError(space.w_ValueError, space.wrap(msg)) tp = self._type if tp == "\0": tp = "g" + default_prec = 12 + flags |= rarithmetic.DTSF_ADD_DOT_0 + elif tp == "n": + tp = "g" value = space.float_w(w_float) if tp == "%": tp = "f" value *= 100 + add_pct = True + else: + add_pct = False if self._precision == -1: - self._precision = 6 + self._precision = default_precision + result, special = rarithmetic.double_to_string(value, tp, + self._precision, flags) + if add_pct: + result += "%" + n_digits = len(result) + if result[0] == "-": + sign = "-" + to_number = 1 + n_digits -= 1 + else: + sign = "\0" + to_number = 0 + have_dec_point, to_remainder = self._parse_number(result, to_number) + n_remainder = len(result) - to_remainder + self._get_locale(tp) + spec = self._calc_num_width(0, sign, to_number, n_digits, + n_remainder, have_dec_point, result) + fill = " " if self._fill_char == "\0" else self._fill_char + return self._fill_number(spec, result, to_number, None, 0, fill, + to_remainder, False) def format_float(self, w_float): space = self.space Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py Thu Jul 1 00:14:47 2010 @@ -215,3 +215,15 @@ def setup_class(cls): cls.w_i = cls.space.w_long + + +class AppTestFloatFormatting: + + def test_alternate(self): + raises(ValueError, format, 1.0, "#") + + def test_simple(self): + assert format(0.0, "f") == "0.000000" + + def test_sign(self): + assert format(-1.23, "1") == "-1.23" Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Thu Jul 1 00:14:47 2010 @@ -465,6 +465,22 @@ return formatd(fmt, x) +DTSF_ADD_DOT_0 = 1 + +DIST_FINITE = 1 +DIST_NAN = 2 +DIST_INFINITY = 3 + +def double_to_string(value, tp, precision, flags): + if isnan(value): + special = DIST_NAN + elif isinf(value): + special = DIST_INFINITY + else: + special = DIST_FINITE + result = formatd_overflow(False, precision, tp, value) + return result, special + # the 'float' C type class r_singlefloat(object): From benjamin at codespeak.net Thu Jul 1 00:51:31 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 00:51:31 +0200 (CEST) Subject: [pypy-svn] r75708 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100630225131.23AAF282B9C@codespeak.net> Author: benjamin Date: Thu Jul 1 00:51:28 2010 New Revision: 75708 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py Log: fix more float formatting bugs Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Thu Jul 1 00:51:28 2010 @@ -467,7 +467,7 @@ if self._fill_char == "0" and self._align == "=": spec.n_min_width = self._width - extra_length if self._loc_thousands: - self._group_digits(spec, digits) + self._group_digits(spec, digits[to_number:]) n_grouped_digits = len(self._grouped_digits) else: n_grouped_digits = spec.n_digits @@ -500,7 +500,7 @@ buf = [] grouping = self._loc_grouping min_width = spec.n_min_width - i = 0 + grouping_state = 0 count = 0 left = spec.n_digits n_ts = len(self._loc_thousands) @@ -508,11 +508,11 @@ done = False groupings = len(grouping) while True: - group = ord(grouping[i]) + group = ord(grouping[grouping_state]) if group > 0: if group == 256: break - i += 1 + grouping_state += 1 previous = group else: group = previous Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py Thu Jul 1 00:51:28 2010 @@ -227,3 +227,10 @@ def test_sign(self): assert format(-1.23, "1") == "-1.23" + + def test_digit_separator(self): + assert format(-1234., "012,f") == "-1,234.000000" + + def test_dont_switch_to_g(self): + skip("must fix when float formatting is figured out") + assert len(format(1.1234e90, "f")) == 98 From benjamin at codespeak.net Thu Jul 1 01:14:32 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 01:14:32 +0200 (CEST) Subject: [pypy-svn] r75709 - in pypy/branch/fast-forward/pypy/interpreter: . test Message-ID: <20100630231432.CCFFC282B9C@codespeak.net> Author: benjamin Date: Thu Jul 1 01:14:31 2010 New Revision: 75709 Modified: pypy/branch/fast-forward/pypy/interpreter/function.py pypy/branch/fast-forward/pypy/interpreter/test/test_function.py Log: complain when both self and class are None Modified: pypy/branch/fast-forward/pypy/interpreter/function.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/function.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/function.py Thu Jul 1 01:14:31 2010 @@ -430,8 +430,11 @@ self.w_class = w_class # possibly space.w_None def descr_method__new__(space, w_subtype, w_function, w_instance, w_class=None): - if space.is_w( w_instance, space.w_None ): + if space.is_w(w_instance, space.w_None): w_instance = None + if w_instance is None and space.is_w(w_class, space.w_None): + raise OperationError(space.w_TypeError, + space.wrap("unbound methods must have class")) method = space.allocate_instance(Method, w_subtype) Method.__init__(method, space, w_function, w_instance, w_class) return space.wrap(method) Modified: pypy/branch/fast-forward/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/test/test_function.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/test/test_function.py Thu Jul 1 01:14:31 2010 @@ -482,6 +482,11 @@ raises(TypeError, m, MyInst(None)) raises(TypeError, m, MyInst(42)) + def test_invalid_creation(self): + import new + def f(): pass + raises(TypeError, new.instancemethod, f, None) + class TestMethod: def setup_method(self, method): From afa at codespeak.net Thu Jul 1 10:48:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 10:48:22 +0200 (CEST) Subject: [pypy-svn] r75710 - pypy/branch/interplevel-codecs/pypy/module/_codecs/test Message-ID: <20100701084822.5F34C282B9E@codespeak.net> Author: afa Date: Thu Jul 1 10:48:19 2010 New Revision: 75710 Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py Log: These tests pass, actually Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py Thu Jul 1 10:48:19 2010 @@ -12,9 +12,6 @@ raises(TypeError, _codecs.register, 1) def test_bigU_codecs(self): - import sys - if sys.maxunicode <= 0xffff: - return # this test cannot run on UCS2 builds u = u'\U00010001\U00020002\U00030003\U00040004\U00050005' for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', 'raw_unicode_escape', @@ -22,9 +19,6 @@ assert unicode(u.encode(encoding),encoding) == u def test_ucs4(self): - import sys - if sys.maxunicode <= 0xffff: - return # this test cannot run on UCS2 builds x = u'\U00100000' y = x.encode("raw-unicode-escape").decode("raw-unicode-escape") assert x == y From afa at codespeak.net Thu Jul 1 10:52:11 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 10:52:11 +0200 (CEST) Subject: [pypy-svn] r75711 - pypy/branch/interplevel-codecs/pypy/rlib Message-ID: <20100701085211.2EE30282B9E@codespeak.net> Author: afa Date: Thu Jul 1 10:52:09 2010 New Revision: 75711 Modified: pypy/branch/interplevel-codecs/pypy/rlib/runicode.py Log: - Uniformly handle the decoding of \U and \u - Decoded \U sequences must be unsigned int, otherwise \UFFFFFEEEE wraps and becomes \ueeee... Modified: pypy/branch/interplevel-codecs/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/rlib/runicode.py (original) +++ pypy/branch/interplevel-codecs/pypy/rlib/runicode.py Thu Jul 1 10:52:09 2010 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rlib.rstring import StringBuilder, UnicodeBuilder +from pypy.rlib.rarithmetic import r_uint if rffi.sizeof(lltype.UniChar) == 4: MAXUNICODE = 0x10ffff @@ -780,7 +781,8 @@ hexdigits = "0123456789ABCDEFabcdef" -def hexescape(builder, s, pos, digits, errorhandler, message, errors): +def hexescape(builder, s, pos, digits, + encoding, errorhandler, message, errors): import sys chr = 0 if (pos+digits>len(s)): @@ -790,12 +792,12 @@ builder.append(res) else: try: - chr = int(s[pos:pos+digits], 16) + chr = r_uint(int(s[pos:pos+digits], 16)) except ValueError: endinpos = pos while s[endinpos] in hexdigits: endinpos += 1 - res, pos = errorhandler(errors, "unicodeescape", + res, pos = errorhandler(errors, encoding, message, s, pos-2, endinpos+1) builder.append(res) else: @@ -811,8 +813,8 @@ pos += digits else: message = "illegal Unicode character" - res, pos = errorhandler(errors, "unicodeescape", - message, s, pos-2, pos+1) + res, pos = errorhandler(errors, encoding, + message, s, pos-2, pos+digits) builder.append(res) return pos @@ -878,21 +880,21 @@ digits = 2 message = "truncated \\xXX escape" pos = hexescape(builder, s, pos, digits, - errorhandler, message, errors) + "unicodeescape", errorhandler, message, errors) # \uXXXX elif ch == 'u': digits = 4 message = "truncated \\uXXXX escape" pos = hexescape(builder, s, pos, digits, - errorhandler, message, errors) + "unicodeescape", errorhandler, message, errors) # \UXXXXXXXX elif ch == 'U': digits = 8 message = "truncated \\UXXXXXXXX escape" pos = hexescape(builder, s, pos, digits, - errorhandler, message, errors) + "unicodeescape", errorhandler, message, errors) # \N{name} elif ch == 'N': @@ -1052,31 +1054,14 @@ continue if s[pos] == 'u': - count = 4 + digits = 4 + message = "truncated \\uXXXX escape" else: - count = 8 + digits = 8 + message = "truncated \\UXXXXXXXX escape" pos += 1 - - # \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 - x = 0 - try: - x = int(s[pos:pos+count], 16) - except ValueError: - res, pos = errorhandler(errors, "rawunicodeescape", - "truncated \\uXXXX", - s, pos, size) - result.append(res) - continue - - if (x > MAXUNICODE): - res, pos = errorhandler(errors, "rawunicodeescape", - "\\Uxxxxxxxx out of range", - s, pos, size) - result.append(res) - continue - - result.append(unichr(x)) - pos += count + pos = hexescape(result, s, pos, digits, + "rawunicodeescape", errorhandler, message, errors) return result.build(), pos @@ -1147,10 +1132,10 @@ if pos > size - unicode_bytes: break continue - t = 0 + t = r_uint(0) h = 0 for j in range(start, stop, step): - t += ord(s[pos + j]) << (h*8) + t += r_uint(ord(s[pos + j])) << (h*8) h += 1 if t > MAXUNICODE: res, pos = errorhandler(errors, "unicode_internal", From fijal at codespeak.net Thu Jul 1 12:05:43 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Jul 2010 12:05:43 +0200 (CEST) Subject: [pypy-svn] r75712 - pypy/trunk/pypy/module/operator/test Message-ID: <20100701100543.357A4282B9E@codespeak.net> Author: fijal Date: Thu Jul 1 12:05:41 2010 New Revision: 75712 Modified: pypy/trunk/pypy/module/operator/test/test_operator.py Log: Strike unnecessary import Modified: pypy/trunk/pypy/module/operator/test/test_operator.py ============================================================================== --- pypy/trunk/pypy/module/operator/test/test_operator.py (original) +++ pypy/trunk/pypy/module/operator/test/test_operator.py Thu Jul 1 12:05:41 2010 @@ -154,7 +154,6 @@ def test_irepeat(self): import operator - import py class X(object): def __index__(self): From afa at codespeak.net Thu Jul 1 12:10:33 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 12:10:33 +0200 (CEST) Subject: [pypy-svn] r75713 - pypy/branch/interplevel-codecs/pypy/rlib Message-ID: <20100701101033.BD3AA282B9E@codespeak.net> Author: afa Date: Thu Jul 1 12:10:32 2010 New Revision: 75713 Modified: pypy/branch/interplevel-codecs/pypy/rlib/runicode.py Log: remove C-style parentheses Modified: pypy/branch/interplevel-codecs/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/rlib/runicode.py (original) +++ pypy/branch/interplevel-codecs/pypy/rlib/runicode.py Thu Jul 1 12:10:32 2010 @@ -81,7 +81,7 @@ errorhandler=None): if errorhandler is None: errorhandler = raise_unicode_exception_decode - if (size == 0): + if size == 0: return u'', 0 result = UnicodeBuilder(size) @@ -95,14 +95,14 @@ continue n = utf8_code_length[ordch1] - if (pos + n > size): + if pos + n > size: if not final: break else: r, pos = errorhandler(errors, "utf-8", "unexpected end of data", s, pos, size) result.append(r) - if (pos + n > size): + if pos + n > size: break if n == 0: r, pos = errorhandler(errors, "utf-8", "unexpected code byte", @@ -117,7 +117,7 @@ z, two = splitter[6, 2](ordch2) y, six = splitter[5, 3](ordch1) assert six == 6 - if (two != 2): + if two != 2: r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 2) result.append(r) @@ -138,7 +138,7 @@ y, two2 = splitter[6, 2](ordch2) x, fourteen = splitter[4, 4](ordch1) assert fourteen == 14 - if (two1 != 2 or two2 != 2): + if two1 != 2 or two2 != 2: r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 3) result.append(r) @@ -167,7 +167,7 @@ x, two3 = splitter[6, 2](ordch2) w, thirty = splitter[3, 5](ordch1) assert thirty == 30 - if (two1 != 2 or two2 != 2 or two3 != 2): + if two1 != 2 or two2 != 2 or two3 != 2: r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 4) result.append(r) @@ -175,7 +175,7 @@ c = (w << 18) + (x << 12) + (y << 6) + z # minimum value allowed for 4 byte encoding # maximum value allowed for UTF-16 - if ((c < 0x10000) or (c > 0x10ffff)): + if c < 0x10000 or c > 0x10ffff: r, pos = errorhandler(errors, "utf-8", "illegal encoding", s, pos, pos + 4) result.append(r) @@ -214,22 +214,22 @@ while i < size: ch = ord(s[i]) i += 1 - if (ch < 0x80): - # Encode ASCII + if ch < 0x80: + # Encode ASCII result.append(chr(ch)) - elif (ch < 0x0800) : - # Encode Latin-1 + elif ch < 0x0800: + # Encode Latin-1 result.append(chr((0xc0 | (ch >> 6)))) result.append(chr((0x80 | (ch & 0x3f)))) else: # Encode UCS2 Unicode ordinals - if (ch < 0x10000): + if ch < 0x10000: # Special case: check for high surrogate - if (0xD800 <= ch and ch <= 0xDBFF and i != size) : + if 0xD800 <= ch <= 0xDBFF and i != size: ch2 = ord(s[i]) # Check for low surrogate and combine the two to # form a UCS4 value - if (0xDC00 <= ch2 and ch2 <= 0xDFFF) : + if 0xDC00 <= ch2 <= 0xDFFF: ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000 i += 1 _encodeUCS4(result, ch3) @@ -285,10 +285,10 @@ # stream as-is (giving a ZWNBSP character). pos = 0 if byteorder == 'native': - if (size >= 2): + if size >= 2: bom = (ord(s[ihi]) << 8) | ord(s[ilo]) if BYTEORDER == 'little': - if (bom == 0xFEFF): + if bom == 0xFEFF: pos += 2 bo = -1 elif bom == 0xFFFE: @@ -305,14 +305,14 @@ bo = -1 else: bo = 1 - if (size == 0): + if size == 0: return u'', 0, bo - if (bo == -1): + if bo == -1: # force little endian ihi = 1 ilo = 0 - elif (bo == 1): + elif bo == 1: # force big endian ihi = 0 ilo = 1 @@ -332,7 +332,7 @@ break ch = (ord(s[pos + ihi]) << 8) | ord(s[pos + ilo]) pos += 2 - if (ch < 0xD800 or ch > 0xDFFF): + if ch < 0xD800 or ch > 0xDFFF: result.append(unichr(ch)) continue # UTF-16 code pair: @@ -344,10 +344,10 @@ result.append(r) if len(s) - pos < 2: break - elif (0xD800 <= ch and ch <= 0xDBFF): + elif 0xD800 <= ch <= 0xDBFF: ch2 = (ord(s[pos+ihi]) << 8) | ord(s[pos+ilo]) pos += 2 - if (0xDC00 <= ch2 and ch2 <= 0xDFFF): + if 0xDC00 <= ch2 <= 0xDFFF: if MAXUNICODE < 65536: result.append(unichr(ch)) result.append(unichr(ch2)) @@ -384,7 +384,7 @@ return "" result = StringBuilder(size * 2 + 2) - if (byteorder == 'native'): + if byteorder == 'native': _STORECHAR(result, 0xFEFF, BYTEORDER) byteorder = BYTEORDER @@ -393,7 +393,7 @@ ch = ord(s[i]) i += 1 ch2 = 0 - if (ch >= 0x10000) : + if ch >= 0x10000: ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) ch = 0xD800 | ((ch-0x10000) >> 10) @@ -465,8 +465,8 @@ else: return ord(c) + 4 -def _utf7_ENCODE(result, ch, bits) : - while (bits >= 6): +def _utf7_ENCODE(result, ch, bits): + while bits >= 6: result.append(_utf7_TO_BASE64(ch >> (bits - 6))) bits -= 6 return bits @@ -475,7 +475,7 @@ errorhandler=None): if errorhandler is None: errorhandler = raise_unicode_exception_decode - if (size == 0): + if size == 0: return u'', 0 inShift = False @@ -533,7 +533,7 @@ result.append(u'-') inShift = True - elif _utf7_SPECIAL(oc) : + elif _utf7_SPECIAL(oc): msg = "unexpected special character" res, pos = errorhandler(errors, 'utf-7', msg, s, pos-1, pos) @@ -572,7 +572,7 @@ return result.build(), pos def unicode_encode_utf_7(s, size, errors, errorhandler=None): - if (size == 0): + if size == 0: return '' result = StringBuilder(size) @@ -622,9 +622,9 @@ ch2 = s[pos + 1] oc2 = ord(ch2) - if (_utf7_SPECIAL(oc2, encodeSetO, encodeWhiteSpace)): + if _utf7_SPECIAL(oc2, encodeSetO, encodeWhiteSpace): pass - elif (_utf7_B64CHAR(oc2) or ch2 == u'-'): + elif _utf7_B64CHAR(oc2) or ch2 == u'-': result.append('-') inShift = False else: @@ -683,8 +683,8 @@ else: reason = "ordinal not in range(128)" encoding = "ascii" - - if (size == 0): + + if size == 0: return '' result = StringBuilder(size) pos = 0 @@ -785,7 +785,7 @@ encoding, errorhandler, message, errors): import sys chr = 0 - if (pos+digits>len(s)): + if pos + digits > len(s): message = "end of string in escape sequence" res, pos = errorhandler(errors, "unicodeescape", message, s, pos-2, len(s)) @@ -806,7 +806,7 @@ builder.append(unichr(chr)) pos += digits - elif (chr <= 0x10ffff): + elif chr <= 0x10ffff: chr -= 0x10000L builder.append(unichr(0xD800 + (chr >> 10))) builder.append(unichr(0xDC00 + (chr & 0x03FF))) @@ -824,7 +824,7 @@ if errorhandler is None: errorhandler = raise_unicode_exception_decode - if (size == 0): + if size == 0: return u'', 0 builder = UnicodeBuilder(size) @@ -833,7 +833,7 @@ ch = s[pos] # Non-escape characters are interpreted as Unicode ordinals - if (ch != '\\') : + if ch != '\\': builder.append(unichr(ord(ch))) pos += 1 continue @@ -899,7 +899,6 @@ # \N{name} elif ch == 'N': message = "malformed \\N character escape" - #pos += 1 look = pos if unicodedata_handler is None: message = ("\\N escapes not supported " @@ -911,9 +910,9 @@ if look < size and s[look] == '{': # look for the closing brace - while (look < size and s[look] != '}'): + while look < size and s[look] != '}': look += 1 - if (look > pos+1 and look < size and s[look] == '}'): + if look < size and s[look] == '}': # found a name. look it up in the unicode database message = "unknown Unicode character name" name = s[pos+1:look] @@ -1016,7 +1015,7 @@ errorhandler=None): if errorhandler is None: errorhandler = raise_unicode_exception_decode - if (size == 0): + if size == 0: return u'', 0 result = UnicodeBuilder(size) @@ -1025,7 +1024,7 @@ ch = s[pos] # Non-escape characters are interpreted as Unicode ordinals - if (ch != '\\'): + if ch != '\\': result.append(unichr(ord(ch))) pos += 1 continue @@ -1045,9 +1044,9 @@ result.append(u'\\') break - if (((pos - bs) & 1) == 0 or + if ((pos - bs) & 1 == 0 or pos >= size or - (s[pos] != 'u' and s[pos] != 'U')) : + (s[pos] != 'u' and s[pos] != 'U')): result.append(u'\\') result.append(unichr(ord(s[pos]))) pos += 1 @@ -1084,7 +1083,7 @@ def unicode_encode_raw_unicode_escape(s, size, errors, errorhandler=None): # errorhandler is not used: this function cannot cause Unicode errors - if (size == 0): + if size == 0: return '' result = StringBuilder(size) pos = 0 @@ -1105,7 +1104,7 @@ errorhandler=None): if errorhandler is None: errorhandler = raise_unicode_exception_decode - if (size == 0): + if size == 0: return u'', 0 if MAXUNICODE < 65536: @@ -1148,7 +1147,7 @@ return result.build(), pos def unicode_encode_unicode_internal(s, size, errors, errorhandler=None): - if (size == 0): + if size == 0: return '' if MAXUNICODE < 65536: From jcreigh at codespeak.net Thu Jul 1 15:07:18 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 1 Jul 2010 15:07:18 +0200 (CEST) Subject: [pypy-svn] r75714 - pypy/branch/x86-64-jit-backend/pypy/module/signal Message-ID: <20100701130718.2F259282B9E@codespeak.net> Author: jcreigh Date: Thu Jul 1 15:07:16 2010 New Revision: 75714 Modified: pypy/branch/x86-64-jit-backend/pypy/module/signal/interp_signal.py Log: add jit.dont_look_inside to some methods in module/signal/interp_signal.py that call external functions (fix for 64-bit) Modified: pypy/branch/x86-64-jit-backend/pypy/module/signal/interp_signal.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/signal/interp_signal.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/signal/interp_signal.py Thu Jul 1 15:07:16 2010 @@ -7,6 +7,7 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo import py from pypy.tool import autopath +from pypy.rlib import jit def setup(): for key, value in cpy_signal.__dict__.items(): @@ -159,10 +160,12 @@ return space.wrap(SIG_DFL) getsignal.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def alarm(space, timeout): return space.wrap(c_alarm(timeout)) alarm.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def pause(space): c_pause() return space.w_None @@ -173,6 +176,7 @@ raise OperationError(space.w_ValueError, space.wrap("signal number out of range")) + at jit.dont_look_inside def signal(space, signum, w_handler): """ signal(sig, action) -> action From benjamin at codespeak.net Thu Jul 1 15:18:46 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 15:18:46 +0200 (CEST) Subject: [pypy-svn] r75715 - pypy/branch/fast-forward/pypy/module/marshal Message-ID: <20100701131846.683FE282B9E@codespeak.net> Author: benjamin Date: Thu Jul 1 15:18:44 2010 New Revision: 75715 Modified: pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py Log: replace artificial marshall depth with checking for stack overflow Modified: pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py (original) +++ pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py Thu Jul 1 15:18:44 2010 @@ -1,6 +1,7 @@ from pypy.interpreter.baseobjspace import ObjSpace from pypy.interpreter.error import OperationError from pypy.rlib.rarithmetic import intmask +from pypy.rlib import rstackovf from pypy.module._file.interp_file import W_File from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror import sys @@ -20,7 +21,7 @@ # so we have to pass the instance in, instead. ##m = Marshaller(space, writer.write, space.int_w(w_version)) m = Marshaller(space, writer, space.int_w(w_version)) - m.put_w_obj(w_data) + m.dump_w_obj(w_data) finally: writer.finished() @@ -28,7 +29,7 @@ """Return the string that would have been written to a file by dump(data, file).""" m = StringMarshaller(space, space.int_w(w_version)) - m.put_w_obj(w_data) + m.dump_w_obj(w_data) return space.wrap(m.get_value()) def load(space, w_f): @@ -41,7 +42,7 @@ reader = FileReader(space, w_f) try: u = Unmarshaller(space, reader) - return u.get_w_obj(False) + return u.load_w_obj(False) finally: reader.finished() @@ -50,7 +51,7 @@ ignored.""" space.timer.start("marshal loads") u = StringUnmarshaller(space, w_str) - obj = u.get_w_obj(False) + obj = u.load_w_obj(False) space.timer.stop("marshal loads") return obj @@ -132,13 +133,6 @@ return data -MAX_MARSHAL_DEPTH = 5000 - -# the above is unfortunately necessary because CPython -# relies on it without run-time checking. -# PyPy is currently in much bigger trouble, because the -# multimethod dispatches cause deeper stack nesting. - class _Base(object): def raise_exc(self, msg): space = self.space @@ -153,7 +147,6 @@ ## self.put = putfunc self.writer = writer self.version = version - self.nesting = 0 # contribution to compatibility self.stringtable = {} ## currently we cannot use a put that is a bound method @@ -224,28 +217,25 @@ self.put(x) def put_w_obj(self, w_obj): - self.nesting += 1 - if self.nesting < MAX_MARSHAL_DEPTH: - self.space.marshal_w(w_obj, self) - else: + self.space.marshal_w(w_obj, self) + + def dump_w_obj(self, w_obj): + try: + self.put_w_obj(w_obj) + except rstackovf.StackOverflow: + rstackovf.check_stack_overflow() self._overflow() - self.nesting -= 1 def put_tuple_w(self, typecode, lst_w): - self.nesting += 1 self.start(typecode) lng = len(lst_w) self.put_int(lng) idx = 0 space = self.space - if self.nesting < MAX_MARSHAL_DEPTH: - while idx < lng: - w_obj = lst_w[idx] - self.space.marshal_w(w_obj, self) - idx += 1 - else: - self._overflow() - self.nesting -= 1 + while idx < lng: + w_obj = lst_w[idx] + self.space.marshal_w(w_obj, self) + idx += 1 def _overflow(self): self.raise_exc('object too deeply nested to marshal') @@ -357,7 +347,6 @@ def __init__(self, space, reader): self.space = space self.reader = reader - self.nesting = 0 self.stringtable_w = [] def get(self, n): @@ -433,42 +422,39 @@ return self.get(lng) def get_w_obj(self, allow_null): - self.nesting += 1 space = self.space w_ret = space.w_None # something not None - if self.nesting < MAX_MARSHAL_DEPTH: - tc = self.get1() - w_ret = self._dispatch[ord(tc)](space, self, tc) - if w_ret is None and not allow_null: - raise OperationError(space.w_TypeError, space.wrap( - 'NULL object in marshal data')) - else: - self._overflow() - self.nesting -= 1 + tc = self.get1() + w_ret = self._dispatch[ord(tc)](space, self, tc) + if w_ret is None and not allow_null: + raise OperationError(space.w_TypeError, space.wrap( + 'NULL object in marshal data')) return w_ret - # inlined version to save a nesting level + def load_w_obj(self, allow_null): + try: + return self.get_w_obj(allow_null) + except rstackovf.StackOverflow: + rstackovf.check_stack_overflow() + self._overflow() + + # inlined version to save a recursion level def get_tuple_w(self): - self.nesting += 1 lng = self.get_lng() res_w = [None] * lng idx = 0 space = self.space - w_ret = space.w_None # something not None - if self.nesting < MAX_MARSHAL_DEPTH: - while idx < lng: - tc = self.get1() - w_ret = self._dispatch[ord(tc)](space, self, tc) - if w_ret is None: - break - res_w[idx] = w_ret - idx += 1 - else: - self._overflow() + w_ret = space.w_None # something not + while idx < lng: + tc = self.get1() + w_ret = self._dispatch[ord(tc)](space, self, tc) + if w_ret is None: + break + res_w[idx] = w_ret + idx += 1 if w_ret is None: raise OperationError(space.w_TypeError, space.wrap( 'NULL object in marshal data')) - self.nesting -= 1 return res_w def get_list_w(self): From benjamin at codespeak.net Thu Jul 1 15:30:57 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 15:30:57 +0200 (CEST) Subject: [pypy-svn] r75716 - in pypy/branch/fast-forward/pypy/module/marshal: . test Message-ID: <20100701133057.637E8282B9E@codespeak.net> Author: benjamin Date: Thu Jul 1 15:30:55 2010 New Revision: 75716 Modified: pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py pypy/branch/fast-forward/pypy/module/marshal/test/test_marshal.py Log: reject subclasses of builtins being marshalled Modified: pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py (original) +++ pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py Thu Jul 1 15:30:55 2010 @@ -220,6 +220,12 @@ self.space.marshal_w(w_obj, self) def dump_w_obj(self, w_obj): + space = self.space + if (space.type(w_obj).is_heaptype() and + not space.eq_w(space.getattr(w_obj, space.wrap("__module__")), + space.wrap("array"))): + w_err = space.wrap("only builtins can be marshaled") + raise OperationError(space.w_ValueError, w_err) try: self.put_w_obj(w_obj) except rstackovf.StackOverflow: Modified: pypy/branch/fast-forward/pypy/module/marshal/test/test_marshal.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/marshal/test/test_marshal.py (original) +++ pypy/branch/fast-forward/pypy/module/marshal/test/test_marshal.py Thu Jul 1 15:30:55 2010 @@ -171,6 +171,14 @@ self.marshal_check(unichr(sys.maxunicode)) + def test_reject_subtypes(self): + import marshal + types = (float, complex, int, long, tuple, list, dict, set, frozenset) + for cls in types: + class subtype(cls): + pass + raises(ValueError, marshal.dumps, subtype) + class AppTestRope(AppTestMarshal): def setup_class(cls): From benjamin at codespeak.net Thu Jul 1 15:32:08 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 15:32:08 +0200 (CEST) Subject: [pypy-svn] r75717 - pypy/branch/fast-forward/pypy/module/marshal Message-ID: <20100701133208.C814F282B9E@codespeak.net> Author: benjamin Date: Thu Jul 1 15:32:07 2010 New Revision: 75717 Modified: pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py Log: less hacky hack Modified: pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py (original) +++ pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py Thu Jul 1 15:32:07 2010 @@ -222,8 +222,7 @@ def dump_w_obj(self, w_obj): space = self.space if (space.type(w_obj).is_heaptype() and - not space.eq_w(space.getattr(w_obj, space.wrap("__module__")), - space.wrap("array"))): + space.lookup(w_obj, "__buffer__") is None): w_err = space.wrap("only builtins can be marshaled") raise OperationError(space.w_ValueError, w_err) try: From antocuni at codespeak.net Thu Jul 1 16:01:39 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 1 Jul 2010 16:01:39 +0200 (CEST) Subject: [pypy-svn] r75718 - pypy/trunk/dotviewer Message-ID: <20100701140139.19772282B9E@codespeak.net> Author: antocuni Date: Thu Jul 1 16:01:38 2010 New Revision: 75718 Modified: pypy/trunk/dotviewer/sshgraphserver.py Log: explicitly disable the "control master" option. Else, if ControlMaster is automatically enabled (e.g. by an option in .ssh/config), and there is already an active connection, the port forwarding is not setup Modified: pypy/trunk/dotviewer/sshgraphserver.py ============================================================================== --- pypy/trunk/dotviewer/sshgraphserver.py (original) +++ pypy/trunk/dotviewer/sshgraphserver.py Thu Jul 1 16:01:38 2010 @@ -21,7 +21,7 @@ remoteport = random.randrange(10000, 20000) # ^^^ and just hope there is no conflict - args = ['ssh', '-C', '-R%d:127.0.0.1:%d' % (remoteport, localport)] + args = ['ssh', '-S', 'none', '-C', '-R%d:127.0.0.1:%d' % (remoteport, localport)] args = args + sshargs + ['python -u -c "exec input()"'] print ' '.join(args[:-1]) p = subprocess.Popen(args, bufsize=0, From afa at codespeak.net Thu Jul 1 16:27:23 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 16:27:23 +0200 (CEST) Subject: [pypy-svn] r75719 - in pypy/branch/interplevel-codecs/pypy/module/_codecs: . test Message-ID: <20100701142723.6030F282B9E@codespeak.net> Author: afa Date: Thu Jul 1 16:27:21 2010 New Revision: 75719 Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py Log: Avoid buffer overrun in append_slice(); Test and fix. Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py Thu Jul 1 16:27:21 2010 @@ -232,10 +232,11 @@ else: builder.append(u"\\x") zeros = 2 - nb = zeros + 2 - len(num) # num starts with '0x' + lnum = len(num) + nb = zeros + 2 - lnum # num starts with '0x' if nb > 0: builder.append_multiple_char(u'0', nb) - builder.append_slice(unicode(num), 2, 8) + builder.append_slice(unicode(num), 2, lnum) pos += 1 return space.newtuple([space.wrap(builder.build()), w_end]) else: Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py Thu Jul 1 16:27:21 2010 @@ -460,6 +460,9 @@ assert '\xff'.decode('utf-7', 'ignore') == '' assert '\x00'.decode('unicode-internal', 'ignore') == '' + def test_backslahreplace(self): + assert u'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == 'a\\xac\u1234\u20ac\u8000' + def test_badhandler(self): import codecs results = ( 42, u"foo", (1,2,3), (u"foo", 1, 3), (u"foo", None), (u"foo",), ("foo", 1, 3), ("foo", None), ("foo",) ) From afa at codespeak.net Thu Jul 1 16:29:21 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 16:29:21 +0200 (CEST) Subject: [pypy-svn] r75720 - in pypy/branch/interplevel-codecs/pypy: module/_codecs/test rlib Message-ID: <20100701142921.266C1282B9E@codespeak.net> Author: afa Date: Thu Jul 1 16:29:19 2010 New Revision: 75720 Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py pypy/branch/interplevel-codecs/pypy/rlib/runicode.py Log: Decode utf-7 characters whenever possible, to avoid integer overflow when the "inShift" excursion is too long. Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py Thu Jul 1 16:29:19 2010 @@ -541,6 +541,9 @@ assert exc.start == 0 assert exc.end == 3 + def test_utf7_surrogate(self): + raises(UnicodeDecodeError, '+3ADYAA-'.decode, 'utf-7') + def test_utf_16_encode_decode(self): import codecs x = u'123abc' Modified: pypy/branch/interplevel-codecs/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/rlib/runicode.py (original) +++ pypy/branch/interplevel-codecs/pypy/rlib/runicode.py Thu Jul 1 16:29:19 2010 @@ -471,6 +471,32 @@ bits -= 6 return bits +def _utf7_DECODE(s, result, errorhandler, errors, + pos, charsleft, bitsleft, surrogate): + while bitsleft >= 16: + outCh = (charsleft >> (bitsleft-16)) & 0xffff + bitsleft -= 16 + + if surrogate: + ## We have already generated an error for the high + ## surrogate so let's not bother seeing if the low + ## surrogate is correct or not + surrogate = False + elif 0xDC00 <= outCh <= 0xDFFF: + ## This is a surrogate pair. Unfortunately we can't + ## represent it in a 16-bit character + surrogate = True + msg = "code pairs are not supported" + res, pos = errorhandler(errors, 'utf-7', + msg, s, pos-1, pos) + result.append(res) + bitsleft = 0 + break + else: + result.append(unichr(outCh)) + return pos, charsleft, bitsleft, surrogate + + def str_decode_utf_7(s, size, errors, final=False, errorhandler=None): if errorhandler is None: @@ -495,27 +521,9 @@ inShift = 0 pos += 1 - while bitsleft >= 16: - outCh = (charsleft >> (bitsleft-16)) & 0xffff - bitsleft -= 16 - - if surrogate: - ## We have already generated an error for the high - ## surrogate so let's not bother seeing if the low - ## surrogate is correct or not - surrogate = False - elif 0xDC00 <= outCh <= 0xDFFF: - ## This is a surrogate pair. Unfortunately we can't - ## represent it in a 16-bit character - surrogate = True - msg = "code pairs are not supported" - res, pos = errorhandler(errors, 'utf-7', - msg, s, pos-1, pos) - result.append(res) - bitsleft = 0 - break - else: - result.append(unichr(outCh)) + pos, charsleft, bitsleft, surrogate = _utf7_DECODE( + s, result, errorhandler, errors, + pos, charsleft, bitsleft, surrogate) if bitsleft >= 6: ## The shift sequence has a partial character in it. If ## bitsleft < 6 then we could just classify it as padding @@ -544,6 +552,10 @@ charsleft = (charsleft << 6) | _utf7_FROM_BASE64(ch) bitsleft += 6 pos += 1 + + pos, charsleft, bitsleft, surrogate = _utf7_DECODE( + s, result, errorhandler, errors, + pos, charsleft, bitsleft, surrogate) elif ch == '+': startinpos = pos pos += 1 From benjamin at codespeak.net Thu Jul 1 18:23:34 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 18:23:34 +0200 (CEST) Subject: [pypy-svn] r75721 - in pypy/branch/fast-forward/pypy/module: marshal/test math Message-ID: <20100701162334.92ED2282B9E@codespeak.net> Author: benjamin Date: Thu Jul 1 18:23:31 2010 New Revision: 75721 Modified: pypy/branch/fast-forward/pypy/module/marshal/test/test_marshal.py pypy/branch/fast-forward/pypy/module/math/interp_math.py Log: remove sys import Modified: pypy/branch/fast-forward/pypy/module/marshal/test/test_marshal.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/marshal/test/test_marshal.py (original) +++ pypy/branch/fast-forward/pypy/module/marshal/test/test_marshal.py Thu Jul 1 18:23:31 2010 @@ -2,7 +2,6 @@ def make_check(space): return space.appexec([], """(): - import sys import marshal, StringIO def marshal_check(case): s = marshal.dumps(case) Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Thu Jul 1 18:23:31 2010 @@ -46,6 +46,13 @@ return space.wrap(r) math2._annspecialcase_ = 'specialize:arg(1)' +def copysign(space, x, y): + """copysign(x, y) + + Return x with the sign of y.""" + return math2(space, math.copysign, x, y) +copysign.unwrap_spec = [ObjSpace, float, float] + def pow(space, x, y): """pow(x,y) From benjamin at codespeak.net Thu Jul 1 18:28:57 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 18:28:57 +0200 (CEST) Subject: [pypy-svn] r75722 - in pypy/branch/fast-forward/pypy: module/math objspace/std rlib/rstruct rpython/lltypesystem/module Message-ID: <20100701162857.F10E8282B9E@codespeak.net> Author: benjamin Date: Thu Jul 1 18:28:56 2010 New Revision: 75722 Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Log: revert unintended math changes Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Thu Jul 1 18:28:56 2010 @@ -46,13 +46,6 @@ return space.wrap(r) math2._annspecialcase_ = 'specialize:arg(1)' -def copysign(space, x, y): - """copysign(x, y) - - Return x with the sign of y.""" - return math2(space, math.copysign, x, y) -copysign.unwrap_spec = [ObjSpace, float, float] - def pow(space, x, y): """pow(x,y) Modified: pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py Thu Jul 1 18:28:56 2010 @@ -169,11 +169,11 @@ def pack_float(f): result = [] - ieee.pack_float(result, f, 8, False) + ieee.pack_float8(result, f) return ''.join(result) def unpack_float(s): - return ieee.unpack_float(s, False) + return ieee.unpack_float8(s) def marshal_w__Float(space, w_float, m): if m.version > 1: Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Thu Jul 1 18:28:56 2010 @@ -3,7 +3,124 @@ """ import math -from pypy.rlib.rarithmetic import r_longlong, isinf, isnan, INFINITY, NAN +from pypy.rlib.rarithmetic import r_longlong, r_ulonglong, isinf, isnan, INFINITY, NAN + + +def py3round(x): + """Python 3 round function semantics, in Python 2 code. + + We want to round x to the nearest int, but: + - return an int, not a float + - do round-half-to-even, not round-half-away-from-zero. + + We assume that x is finite and nonnegative. + + """ + int_part = r_ulonglong(x) + frac_part = x - int_part + if frac_part > 0.5 or frac_part == 0.5 and (int_part & 1): + int_part += 1 + return int_part + + +def float_pack(x, size): + """Convert a Python float x into a 64-bit unsigned integer + with the same byte representation.""" + + if size == 8: + MIN_EXP = -1021 # = sys.float_info.min_exp + MAX_EXP = 1024 # = sys.float_info.max_exp + MANT_DIG = 53 # = sys.float_info.mant_dig + elif size == 4: + MIN_EXP = -125 # C's FLT_MIN_EXP + MAX_EXP = 128 # FLT_MAX_EXP + MANT_DIG = 24 # FLT_MANT_DIG + else: + raise ValueError("invalid size value") + + sign = math.copysign(1.0, x) < 0.0 + if isinf(x): + mant = 0 + exp = MAX_EXP - MIN_EXP + 2 + elif isnan(x): + mant = 1 << (MANT_DIG-2) + exp = MAX_EXP - MIN_EXP + 2 + elif x == 0.0: + mant = 0 + exp = 0 + else: + m, e = math.frexp(abs(x)) + if e < MIN_EXP: + # Subnormal result (or possibly smallest normal, after rounding). + if e < MIN_EXP - MANT_DIG: + # Underflow to zero; not possible when size == 8. + mant = r_ulonglong(0) + else: + # For size == 8, can substitute 'int' for 'py3round', both + # here and below: the argument will always be integral. + mant = py3round(m * (1 << e - (MIN_EXP - MANT_DIG))) + exp = 0 + elif e > MAX_EXP: + # Overflow to infinity: not possible when size == 8. + raise OverflowError("float too large to pack with f format") + else: + mant = py3round(m * (1 << MANT_DIG)) + exp = e - MIN_EXP + # N.B. It's important that the + mant really is an addition, not just a + # bitwise 'or'. In extreme cases, mant may have MANT_DIG+1 significant + # bits, as a result of rounding. + return ((sign << 8*size - 1) | (exp << MANT_DIG - 1)) + mant + + +def float_unpack(Q, size): + """Convert a 32-bit or 64-bit integer created + by float_pack into a Python float.""" + + if size == 8: + MIN_EXP = -1021 # = sys.float_info.min_exp + MAX_EXP = 1024 # = sys.float_info.max_exp + MANT_DIG = 53 # = sys.float_info.mant_dig + elif size == 4: + MIN_EXP = -125 # C's FLT_MIN_EXP + MAX_EXP = 128 # FLT_MAX_EXP + MANT_DIG = 24 # FLT_MANT_DIG + else: + raise ValueError("invalid size value") + + # extract pieces + sign = Q >> 8*size - 1 + Q -= sign << 8*size - 1 + exp = Q >> MANT_DIG - 1 + Q -= exp << MANT_DIG - 1 + mant = Q + + if exp == MAX_EXP - MIN_EXP + 2: + # nan or infinity + result = float('nan') if mant else float('inf') + elif exp == 0: + # subnormal or zero + result = math.ldexp(float(mant), MIN_EXP - MANT_DIG) + else: + # normal + exp -= 1 + mant += 1 << MANT_DIG - 1 + result = math.ldexp(float(mant), exp + MIN_EXP - MANT_DIG) + return -result if sign else result + + +def pack_float8(result, x): + unsigned = float_pack(x, 8) + for i in range(8): + result.append(chr((unsigned >> (i * 8)) & 0xFF)) + + +def unpack_float8(s): + unsigned = r_ulonglong(0) + for i in range(8): + unsigned |= ord(s[7 - i]) << (i * 8) + print unsigned + return float_unpack(unsigned, 8) + def pack_float(result, number, size, bigendian): """Append to 'result' the 'size' characters of the 32-bit or 64-bit Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Thu Jul 1 18:28:56 2010 @@ -36,6 +36,8 @@ math_fmod = llexternal('fmod', [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE) math_hypot = llexternal(underscore + 'hypot', [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE) +math_isnan = llexternal('isnan', [rffi.DOUBLE], rffi.INT) +math_isinf = llexternal('isinf', [rffi.DOUBLE], rffi.INT) # ____________________________________________________________ # From afa at codespeak.net Thu Jul 1 18:35:55 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 18:35:55 +0200 (CEST) Subject: [pypy-svn] r75723 - in pypy/branch/interplevel-codecs/pypy: module/_codecs/test rlib Message-ID: <20100701163555.85DD2282B9E@codespeak.net> Author: afa Date: Thu Jul 1 18:35:53 2010 New Revision: 75723 Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py pypy/branch/interplevel-codecs/pypy/rlib/runicode.py Log: fix encode_charmap with 'replace': the replacement character must be translated again. Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py Thu Jul 1 18:35:53 2010 @@ -526,6 +526,14 @@ def test_charmap_encode(self): assert 'xxx'.encode('charmap') == 'xxx' + def test_charmap_encode_replace(self): + charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"]) + charmap[ord("?")] = "XYZ" + import codecs + sin = u"abcDEF" + sout = codecs.charmap_encode(sin, "replace", charmap)[0] + assert sout == "AABBCCXYZXYZXYZ" + def test_charmap_decode_2(self): assert 'foo'.decode('charmap') == 'foo' Modified: pypy/branch/interplevel-codecs/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/rlib/runicode.py (original) +++ pypy/branch/interplevel-codecs/pypy/rlib/runicode.py Thu Jul 1 18:35:53 2010 @@ -779,10 +779,17 @@ c = mapping.get(ch, '') if len(c) == 0: - r, pos = errorhandler(errors, "charmap", - "character maps to ", - s, pos, pos + 1) - result.append(r) + res, pos = errorhandler(errors, "charmap", + "character maps to ", + s, pos, pos + 1) + for ch2 in res: + c2 = mapping.get(unichr(ord(ch2)), '') + if len(c2) == 0: + raise_unicode_exception_encode( + errors, "charmap", + "character maps to ", + s, pos, pos + 1) + result.append(c2) continue result.append(c) pos += 1 From benjamin at codespeak.net Thu Jul 1 18:42:03 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 18:42:03 +0200 (CEST) Subject: [pypy-svn] r75724 - in pypy/branch/fast-forward/pypy: objspace/std rlib/rstruct rpython/lltypesystem/module Message-ID: <20100701164203.F2646282B9E@codespeak.net> Author: benjamin Date: Thu Jul 1 18:42:02 2010 New Revision: 75724 Modified: pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Log: revert revert Modified: pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py Thu Jul 1 18:42:02 2010 @@ -169,11 +169,11 @@ def pack_float(f): result = [] - ieee.pack_float8(result, f) + ieee.pack_float(result, f, 8, False) return ''.join(result) def unpack_float(s): - return ieee.unpack_float8(s) + return ieee.unpack_float(s, False) def marshal_w__Float(space, w_float, m): if m.version > 1: Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Thu Jul 1 18:42:02 2010 @@ -3,124 +3,7 @@ """ import math -from pypy.rlib.rarithmetic import r_longlong, r_ulonglong, isinf, isnan, INFINITY, NAN - - -def py3round(x): - """Python 3 round function semantics, in Python 2 code. - - We want to round x to the nearest int, but: - - return an int, not a float - - do round-half-to-even, not round-half-away-from-zero. - - We assume that x is finite and nonnegative. - - """ - int_part = r_ulonglong(x) - frac_part = x - int_part - if frac_part > 0.5 or frac_part == 0.5 and (int_part & 1): - int_part += 1 - return int_part - - -def float_pack(x, size): - """Convert a Python float x into a 64-bit unsigned integer - with the same byte representation.""" - - if size == 8: - MIN_EXP = -1021 # = sys.float_info.min_exp - MAX_EXP = 1024 # = sys.float_info.max_exp - MANT_DIG = 53 # = sys.float_info.mant_dig - elif size == 4: - MIN_EXP = -125 # C's FLT_MIN_EXP - MAX_EXP = 128 # FLT_MAX_EXP - MANT_DIG = 24 # FLT_MANT_DIG - else: - raise ValueError("invalid size value") - - sign = math.copysign(1.0, x) < 0.0 - if isinf(x): - mant = 0 - exp = MAX_EXP - MIN_EXP + 2 - elif isnan(x): - mant = 1 << (MANT_DIG-2) - exp = MAX_EXP - MIN_EXP + 2 - elif x == 0.0: - mant = 0 - exp = 0 - else: - m, e = math.frexp(abs(x)) - if e < MIN_EXP: - # Subnormal result (or possibly smallest normal, after rounding). - if e < MIN_EXP - MANT_DIG: - # Underflow to zero; not possible when size == 8. - mant = r_ulonglong(0) - else: - # For size == 8, can substitute 'int' for 'py3round', both - # here and below: the argument will always be integral. - mant = py3round(m * (1 << e - (MIN_EXP - MANT_DIG))) - exp = 0 - elif e > MAX_EXP: - # Overflow to infinity: not possible when size == 8. - raise OverflowError("float too large to pack with f format") - else: - mant = py3round(m * (1 << MANT_DIG)) - exp = e - MIN_EXP - # N.B. It's important that the + mant really is an addition, not just a - # bitwise 'or'. In extreme cases, mant may have MANT_DIG+1 significant - # bits, as a result of rounding. - return ((sign << 8*size - 1) | (exp << MANT_DIG - 1)) + mant - - -def float_unpack(Q, size): - """Convert a 32-bit or 64-bit integer created - by float_pack into a Python float.""" - - if size == 8: - MIN_EXP = -1021 # = sys.float_info.min_exp - MAX_EXP = 1024 # = sys.float_info.max_exp - MANT_DIG = 53 # = sys.float_info.mant_dig - elif size == 4: - MIN_EXP = -125 # C's FLT_MIN_EXP - MAX_EXP = 128 # FLT_MAX_EXP - MANT_DIG = 24 # FLT_MANT_DIG - else: - raise ValueError("invalid size value") - - # extract pieces - sign = Q >> 8*size - 1 - Q -= sign << 8*size - 1 - exp = Q >> MANT_DIG - 1 - Q -= exp << MANT_DIG - 1 - mant = Q - - if exp == MAX_EXP - MIN_EXP + 2: - # nan or infinity - result = float('nan') if mant else float('inf') - elif exp == 0: - # subnormal or zero - result = math.ldexp(float(mant), MIN_EXP - MANT_DIG) - else: - # normal - exp -= 1 - mant += 1 << MANT_DIG - 1 - result = math.ldexp(float(mant), exp + MIN_EXP - MANT_DIG) - return -result if sign else result - - -def pack_float8(result, x): - unsigned = float_pack(x, 8) - for i in range(8): - result.append(chr((unsigned >> (i * 8)) & 0xFF)) - - -def unpack_float8(s): - unsigned = r_ulonglong(0) - for i in range(8): - unsigned |= ord(s[7 - i]) << (i * 8) - print unsigned - return float_unpack(unsigned, 8) - +from pypy.rlib.rarithmetic import r_longlong, isinf, isnan, INFINITY, NAN def pack_float(result, number, size, bigendian): """Append to 'result' the 'size' characters of the 32-bit or 64-bit Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Thu Jul 1 18:42:02 2010 @@ -36,8 +36,6 @@ math_fmod = llexternal('fmod', [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE) math_hypot = llexternal(underscore + 'hypot', [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE) -math_isnan = llexternal('isnan', [rffi.DOUBLE], rffi.INT) -math_isinf = llexternal('isinf', [rffi.DOUBLE], rffi.INT) # ____________________________________________________________ # From afa at codespeak.net Thu Jul 1 18:49:56 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 18:49:56 +0200 (CEST) Subject: [pypy-svn] r75725 - in pypy/branch/interplevel-codecs/pypy/module/_codecs: . test Message-ID: <20100701164956.21317282B9E@codespeak.net> Author: afa Date: Thu Jul 1 18:49:54 2010 New Revision: 75725 Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py Log: charmap_encode: mapping should yield bytes <256. Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py Thu Jul 1 18:49:54 2010 @@ -534,7 +534,11 @@ if not e.match(space, space.w_TypeError): raise else: - return chr(x) + if 0 <= x < 256: + return chr(x) + else: + raise OperationError(space.w_TypeError, space.wrap( + "character mapping must be in range(256)")) # Charmap may return None if space.is_w(w_ch, space.w_None): Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py Thu Jul 1 18:49:54 2010 @@ -526,6 +526,9 @@ def test_charmap_encode(self): assert 'xxx'.encode('charmap') == 'xxx' + import codecs + raises(TypeError, codecs.charmap_encode, u'\xff', "replace", {0xff: 300}) + def test_charmap_encode_replace(self): charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"]) charmap[ord("?")] = "XYZ" From afa at codespeak.net Thu Jul 1 18:56:19 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 18:56:19 +0200 (CEST) Subject: [pypy-svn] r75726 - in pypy/branch/interplevel-codecs/pypy/module/_codecs: . test Message-ID: <20100701165619.973E9282B9E@codespeak.net> Author: afa Date: Thu Jul 1 18:56:17 2010 New Revision: 75726 Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py Log: charmap_decode: mapping should yield chars <= 65535 (even on wide unicode builds) Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py Thu Jul 1 18:56:17 2010 @@ -494,7 +494,11 @@ if not e.match(space, space.w_TypeError): raise else: - return unichr(x) + if 0 <= x < 65536: # Even on wide unicode builds... + return unichr(x) + else: + raise OperationError(space.w_TypeError, space.wrap( + "character mapping must be in range(65536)")) # Charmap may return None if space.is_w(w_ch, space.w_None): Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py Thu Jul 1 18:56:17 2010 @@ -106,12 +106,15 @@ def test_charmap_decode(self): from _codecs import charmap_decode + import sys assert charmap_decode('', 'strict', 'blablabla') == ('', 0) assert charmap_decode('xxx') == ('xxx', 3) assert charmap_decode('xxx', 'strict', {ord('x'): u'XX'}) == ('XXXXXX', 3) map = tuple([unichr(i) for i in range(256)]) assert charmap_decode('xxx\xff', 'strict', map) == (u'xxx\xff', 4) + raises(TypeError, charmap_decode, '\xff', "replace", {0xff: 0x10001}) + def test_unicode_escape(self): from _codecs import unicode_escape_encode, unicode_escape_decode assert unicode_escape_encode(u'abc') == (u'abc'.encode('unicode_escape'), 3) From benjamin at codespeak.net Thu Jul 1 19:29:23 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 19:29:23 +0200 (CEST) Subject: [pypy-svn] r75729 - in pypy/branch/fast-forward/pypy/interpreter: . test Message-ID: <20100701172923.9B890282B9E@codespeak.net> Author: benjamin Date: Thu Jul 1 19:29:22 2010 New Revision: 75729 Modified: pypy/branch/fast-forward/pypy/interpreter/nestedscope.py pypy/branch/fast-forward/pypy/interpreter/test/test_nestedscope.py Log: fix one of my stupider mistakes; don't mutate co_cellvars, ever Modified: pypy/branch/fast-forward/pypy/interpreter/nestedscope.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/nestedscope.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/nestedscope.py Thu Jul 1 19:29:22 2010 @@ -123,7 +123,7 @@ super_fast2locals(self) # cellvars are values exported to inner scopes # freevars are values coming from outer scopes - freevarnames = self.pycode.co_cellvars + freevarnames = list(self.pycode.co_cellvars) if self.pycode.co_flags & consts.CO_OPTIMIZED: freevarnames.extend(self.pycode.co_freevars) for i in range(len(freevarnames)): Modified: pypy/branch/fast-forward/pypy/interpreter/test/test_nestedscope.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/test/test_nestedscope.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/test/test_nestedscope.py Thu Jul 1 19:29:22 2010 @@ -89,3 +89,14 @@ locals() return X assert f(1).x == 12 + + def test_nested_scope_locals_mutating_cellvars(self): + def f(): + x = 12 + def m(): + locals() + x + locals() + return x + return m + assert f()() == 12 From getxsick at codespeak.net Thu Jul 1 19:54:33 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 1 Jul 2010 19:54:33 +0200 (CEST) Subject: [pypy-svn] r75730 - in pypy/build/ubuntu: 1.3.0/debian trunk/debian Message-ID: <20100701175433.0F55E282B9E@codespeak.net> Author: getxsick Date: Thu Jul 1 19:54:32 2010 New Revision: 75730 Modified: pypy/build/ubuntu/1.3.0/debian/configure.py pypy/build/ubuntu/trunk/debian/configure.py Log: build package only if there is enough amount of RAM on the building machine (the current threshold is set to 1.4G) Modified: pypy/build/ubuntu/1.3.0/debian/configure.py ============================================================================== --- pypy/build/ubuntu/1.3.0/debian/configure.py (original) +++ pypy/build/ubuntu/1.3.0/debian/configure.py Thu Jul 1 19:54:32 2010 @@ -35,7 +35,7 @@ return make_in % substitutions def run(): - check_memory() + check_memory(exit=True) build_arch = None host_arch = None prefix= None Modified: pypy/build/ubuntu/trunk/debian/configure.py ============================================================================== --- pypy/build/ubuntu/trunk/debian/configure.py (original) +++ pypy/build/ubuntu/trunk/debian/configure.py Thu Jul 1 19:54:32 2010 @@ -35,7 +35,7 @@ return make_in % substitutions def run(): - check_memory() + check_memory(exit=True) build_arch = None host_arch = None prefix= None From getxsick at codespeak.net Thu Jul 1 20:11:19 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 1 Jul 2010 20:11:19 +0200 (CEST) Subject: [pypy-svn] r75731 - in pypy/build/ubuntu: 1.3.0/debian trunk/debian Message-ID: <20100701181119.03BAF282B9E@codespeak.net> Author: getxsick Date: Thu Jul 1 20:11:17 2010 New Revision: 75731 Modified: pypy/build/ubuntu/1.3.0/debian/changelog pypy/build/ubuntu/trunk/debian/changelog Log: update changelog Modified: pypy/build/ubuntu/1.3.0/debian/changelog ============================================================================== --- pypy/build/ubuntu/1.3.0/debian/changelog (original) +++ pypy/build/ubuntu/1.3.0/debian/changelog Thu Jul 1 20:11:17 2010 @@ -1,3 +1,9 @@ +pypy (1.3.0-ppa1+lucid3) lucid; urgency=low + + * Check if there is enough memory on build machine before building. + + -- Bartosz Skowron Thu, 01 Jul 2010 20:06:13 +0200 + pypy (1.3.0-ppa1+lucid2) lucid; urgency=low * New upstream release. Modified: pypy/build/ubuntu/trunk/debian/changelog ============================================================================== --- pypy/build/ubuntu/trunk/debian/changelog (original) +++ pypy/build/ubuntu/trunk/debian/changelog Thu Jul 1 20:11:17 2010 @@ -1,3 +1,21 @@ +pypy (1.3.0-ppa1+lucid3) lucid; urgency=low + + * Check if there is enough memory on build machine before building. + + -- Bartosz Skowron Thu, 01 Jul 2010 20:06:13 +0200 + +pypy (1.3.0-ppa1+lucid2) lucid; urgency=low + + * New upstream release. + + -- Bartosz Skowron Mon, 28 Jun 2010 20:02:10 +0200 + +pypy (1.2.0-ppa1+karmic9) karmic; urgency=low + + * Fix failing tests. + + -- Bartosz Skowron Tue, 30 Mar 2010 18:07:06 +0200 + pypy (1.2.0-4) karmic; urgency=low * Add missing build-depends. From afa at codespeak.net Thu Jul 1 20:13:29 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 20:13:29 +0200 (CEST) Subject: [pypy-svn] r75732 - in pypy/branch/interplevel-codecs/pypy: module/_codecs/test rlib Message-ID: <20100701181329.A45BA282B9E@codespeak.net> Author: afa Date: Thu Jul 1 20:13:26 2010 New Revision: 75732 Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py pypy/branch/interplevel-codecs/pypy/rlib/runicode.py Log: Never use raise_unicode_exception_encode when called from the codecs module, otherwise a RPython UnicodeError is raised, and crashes the intepreter. Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/interplevel-codecs/pypy/module/_codecs/test/test_codecs.py Thu Jul 1 20:13:26 2010 @@ -531,6 +531,7 @@ import codecs raises(TypeError, codecs.charmap_encode, u'\xff', "replace", {0xff: 300}) + raises(UnicodeError, codecs.charmap_encode, u"\xff", "replace", {0xff: None}) def test_charmap_encode_replace(self): charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"]) Modified: pypy/branch/interplevel-codecs/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/interplevel-codecs/pypy/rlib/runicode.py (original) +++ pypy/branch/interplevel-codecs/pypy/rlib/runicode.py Thu Jul 1 20:13:26 2010 @@ -785,8 +785,8 @@ for ch2 in res: c2 = mapping.get(unichr(ord(ch2)), '') if len(c2) == 0: - raise_unicode_exception_encode( - errors, "charmap", + errorhandler( + "strict", "charmap", "character maps to ", s, pos, pos + 1) result.append(c2) From getxsick at codespeak.net Thu Jul 1 20:38:24 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 1 Jul 2010 20:38:24 +0200 (CEST) Subject: [pypy-svn] r75733 - pypy/build/ubuntu/1.2.0 Message-ID: <20100701183824.C8F29282B9E@codespeak.net> Author: getxsick Date: Thu Jul 1 20:38:22 2010 New Revision: 75733 Removed: pypy/build/ubuntu/1.2.0/ Log: remove 1.2.0 as we already released 1.3.0 From benjamin at codespeak.net Thu Jul 1 20:50:04 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 20:50:04 +0200 (CEST) Subject: [pypy-svn] r75734 - pypy/branch/fast-forward/pypy/annotation Message-ID: <20100701185004.C2EFF282B9E@codespeak.net> Author: benjamin Date: Thu Jul 1 20:50:02 2010 New Revision: 75734 Modified: pypy/branch/fast-forward/pypy/annotation/builtin.py pypy/branch/fast-forward/pypy/annotation/unaryop.py Log: teach the annotator about the isinstance operation Modified: pypy/branch/fast-forward/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/builtin.py (original) +++ pypy/branch/fast-forward/pypy/annotation/builtin.py Thu Jul 1 20:50:02 2010 @@ -175,17 +175,7 @@ # from bool to int, notice that isinstance( , bool|int) # is quite border case for RPython r.const = False - # XXX HACK HACK HACK - # XXX HACK HACK HACK - # XXX HACK HACK HACK bk = getbookkeeper() - if variables is None: - fn, block, i = bk.position_key - op = block.operations[i] - assert op.opname == "simple_call" - assert len(op.args) == 3 - assert op.args[0] == Constant(isinstance) - variables = [op.args[1]] for variable in variables: assert bk.annotator.binding(variable) == s_obj r.knowntypedata = {} Modified: pypy/branch/fast-forward/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/unaryop.py (original) +++ pypy/branch/fast-forward/pypy/annotation/unaryop.py Thu Jul 1 20:50:02 2010 @@ -23,6 +23,7 @@ UNARY_OPERATIONS = set(['len', 'is_true', 'getattr', 'setattr', 'delattr', 'simple_call', 'call_args', 'str', 'repr', 'iter', 'next', 'invert', 'type', 'issubtype', + 'isinstance', 'pos', 'neg', 'nonzero', 'abs', 'hex', 'oct', 'ord', 'int', 'float', 'long', 'hash', 'id', # <== not supported any more @@ -66,6 +67,13 @@ return immutablevalue(issubclass(obj.const, s_cls.const)) return s_Bool + def isinstance(obj, s_cls): + bk = getbookkeeper() + fn, block, i = bk.position_key + op = block.operations[i] + v = op.args[0] + return builtin.builtin_isinstance(obj, s_cls, [v]) + def len(obj): return SomeInteger(nonneg=True) From wildchild at codespeak.net Thu Jul 1 20:51:10 2010 From: wildchild at codespeak.net (wildchild at codespeak.net) Date: Thu, 1 Jul 2010 20:51:10 +0200 (CEST) Subject: [pypy-svn] r75735 - in pypy/trunk: lib_pypy lib_pypy/pypy_test pypy/doc pypy/module/_stackless pypy/module/_stackless/test pypy/rlib Message-ID: <20100701185110.D1C62282B9E@codespeak.net> Author: wildchild Date: Thu Jul 1 20:51:08 2010 New Revision: 75735 Modified: pypy/trunk/lib_pypy/pypy_test/test_coroutine.py pypy/trunk/lib_pypy/pypy_test/test_stackless.py pypy/trunk/lib_pypy/stackless.py pypy/trunk/pypy/doc/stackless.txt pypy/trunk/pypy/module/_stackless/interp_coroutine.py pypy/trunk/pypy/module/_stackless/rcoroutine.py pypy/trunk/pypy/module/_stackless/test/test_coroutine.py pypy/trunk/pypy/rlib/rcoroutine.py Log: - Implementation of application level CoroutineExit and TaskletExit exceptions (now catchable in user code!). - Implementation of coroutine.throw() to raise an exception inside a specific coroutine (catchable in user code). - Enabled a previously skipped test in pypy/module/_stackless/test/test_coroutine.py - Updated documentation about coroutines/stackless. Modified: pypy/trunk/lib_pypy/pypy_test/test_coroutine.py ============================================================================== --- pypy/trunk/lib_pypy/pypy_test/test_coroutine.py (original) +++ pypy/trunk/lib_pypy/pypy_test/test_coroutine.py Thu Jul 1 20:51:08 2010 @@ -112,6 +112,67 @@ co.kill() assert not co.is_alive + def test_catch_coroutineexit(self): + coroutineexit = [] + co_a = coroutine() + co_test = coroutine.getcurrent() + + def a(): + try: + co_test.switch() + except CoroutineExit: + coroutineexit.append(True) + raise + + co_a.bind(a) + co_a.switch() + assert co_a.is_alive + + co_a.kill() + assert coroutineexit == [True] + assert not co_a.is_alive + + def test_throw(self): + exceptions = [] + co = coroutine() + def f(main): + try: + main.switch() + except RuntimeError: + exceptions.append(True) + + co.bind(f, coroutine.getcurrent()) + co.switch() + co.throw(RuntimeError) + assert exceptions == [True] + + def test_propagation(self): + exceptions = [] + co = coroutine() + co2 = coroutine() + def f(main): + main.switch() + + co.bind(f, coroutine.getcurrent()) + co.switch() + + try: + co.throw(RuntimeError) + except RuntimeError: + exceptions.append(1) + + def f2(): + raise RuntimeError + + co2.bind(f2) + + try: + co2.switch() + except RuntimeError: + exceptions.append(2) + + assert exceptions == [1,2] + def test_bogus_bind(self): co = coroutine() def f(): Modified: pypy/trunk/lib_pypy/pypy_test/test_stackless.py ============================================================================== --- pypy/trunk/lib_pypy/pypy_test/test_stackless.py (original) +++ pypy/trunk/lib_pypy/pypy_test/test_stackless.py Thu Jul 1 20:51:08 2010 @@ -226,6 +226,36 @@ t.kill() assert not t.alive + def test_catch_taskletexit(self): + # Tests if TaskletExit can be caught in the tasklet being killed. + global taskletexit + taskletexit = False + + def f(): + try: + stackless.schedule() + except TaskletExit: + global TaskletExit + taskletexit = True + raise + + t = stackless.tasklet(f)() + t.run() + assert t.alive + t.kill() + assert not t.alive + assert taskletexit + + def test_autocatch_taskletexit(self): + # Tests if TaskletExit is caught correctly in stackless.tasklet.setup(). + def f(): + stackless.schedule() + + t = stackless.tasklet(f)() + t.run() + t.kill() + + # tests inspired from simple stackless.com examples def test_construction(self): Modified: pypy/trunk/lib_pypy/stackless.py ============================================================================== --- pypy/trunk/lib_pypy/stackless.py (original) +++ pypy/trunk/lib_pypy/stackless.py Thu Jul 1 20:51:08 2010 @@ -114,7 +114,7 @@ import operator __all__ = 'run getcurrent getmain schedule tasklet channel coroutine \ - TaskletExit greenlet'.split() + greenlet'.split() _global_task_id = 0 _squeue = None @@ -153,12 +153,12 @@ _last_task = next assert not next.blocked if next is not current: - next.switch() + try: + next.switch() + except CoroutineExit: + raise TaskletExit return current - -class TaskletExit(Exception):pass - def set_schedule_callback(callback): global _schedule_callback _schedule_callback = callback @@ -435,6 +435,7 @@ the tasklet will silently die. """ if not self.is_zombie: + # Killing the tasklet by throwing TaskletExit exception. coroutine.kill(self) _scheduler_remove(self) self.alive = False Modified: pypy/trunk/pypy/doc/stackless.txt ============================================================================== --- pypy/trunk/pypy/doc/stackless.txt (original) +++ pypy/trunk/pypy/doc/stackless.txt Thu Jul 1 20:51:08 2010 @@ -124,10 +124,17 @@ * ``coro.kill()`` - Kill ``coro`` by sending an exception to it. (At the moment, the - exception is not visible to app-level, which means that you cannot - catch it, and that ``try: finally:`` clauses are not honored. This - will be fixed in the future.) + Kill ``coro`` by sending a CoroutineExit exception and switching + execution immediately to it. This exception can be caught in the + coroutine itself and can be raised from any call to ``coro.switch()``. + This exception isn't propagated to the parent coroutine. + +* ``coro.throw(type, value)`` + + Insert an exception in ``coro`` an resume switches execution + immediately to it. In the coroutine itself, this exception + will come from any call to ``coro.switch()`` and can be caught. If the + exception isn't caught, it will be propagated to the parent coroutine. Example ~~~~~~~ Modified: pypy/trunk/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/trunk/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/trunk/pypy/module/_stackless/interp_coroutine.py Thu Jul 1 20:51:08 2010 @@ -24,7 +24,9 @@ from pypy.interpreter.function import StaticMethod from pypy.module._stackless.stackless_flags import StacklessFlags -from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState, AbstractThunk +from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState, AbstractThunk, CoroutineExit + +from pypy.module.exceptions.interp_exceptions import W_SystemExit, _new_exception from pypy.rlib import rstack # for resume points from pypy.tool import stdlib_opcode as pythonopcode @@ -48,6 +50,13 @@ rstack.resume_point("appthunk", costate, returns=w_result) costate.w_tempval = w_result +W_CoroutineExit = _new_exception('CoroutineExit', W_SystemExit, + """Coroutine killed manually.""") + +# Should be moved to interp_stackless.py if it's ever implemented... Currently +# used by pypy/lib/stackless.py. +W_TaskletExit = _new_exception('TaskletExit', W_SystemExit, + """Tasklet killed manually.""") class AppCoroutine(Coroutine): # XXX, StacklessFlags): @@ -92,6 +101,13 @@ w_ret, state.w_tempval = state.w_tempval, space.w_None return w_ret + def switch(self): + space = self.space + try: + Coroutine.switch(self) + except CoroutineExit: + raise OperationError(self.costate.w_CoroutineExit, space.w_None) + def w_finished(self, w_excinfo): pass @@ -102,6 +118,9 @@ w_excvalue = operror.get_w_value(space) w_exctraceback = operror.application_traceback w_excinfo = space.newtuple([w_exctype, w_excvalue, w_exctraceback]) + + if w_exctype is self.costate.w_CoroutineExit: + self.coroutine_exit = True else: w_N = space.w_None w_excinfo = space.newtuple([w_N, w_N, w_N]) @@ -118,6 +137,23 @@ def w_kill(self): self.kill() + + def w_throw(self, w_type, w_value=None, w_traceback=None): + space = self.space + + operror = OperationError(w_type, w_value) + operror.normalize_exception(space) + + if not space.is_w(w_traceback, space.w_None): + from pypy.interpreter import pytraceback + tb = space.interpclass_w(w_traceback) + if tb is None or not space.is_true(space.isinstance(tb, + space.gettypeobject(pytraceback.PyTraceback.typedef))): + raise OperationError(space.w_TypeError, + space.wrap("throw: arg 3 must be a traceback or None")) + operror.application_traceback = tb + + self._kill(operror) def _userdel(self): if self.get_is_zombie(): @@ -309,6 +345,7 @@ unwrap_spec=['self', W_Root, Arguments]), switch = interp2app(AppCoroutine.w_switch), kill = interp2app(AppCoroutine.w_kill), + throw = interp2app(AppCoroutine.w_throw), finished = interp2app(AppCoroutine.w_finished), is_alive = GetSetProperty(AppCoroutine.w_get_is_alive), is_zombie = GetSetProperty(AppCoroutine.w_get_is_zombie, @@ -330,6 +367,26 @@ BaseCoState.__init__(self) self.w_tempval = space.w_None self.space = space + + # Exporting new exception to space + self.w_CoroutineExit = space.gettypefor(W_CoroutineExit) + space.setitem( + space.exceptions_module.w_dict, + space.new_interned_str('CoroutineExit'), + self.w_CoroutineExit) + space.setitem(space.builtin.w_dict, + space.new_interned_str('CoroutineExit'), + self.w_CoroutineExit) + + # Should be moved to interp_stackless.py if it's ever implemented... + self.w_TaskletExit = space.gettypefor(W_TaskletExit) + space.setitem( + space.exceptions_module.w_dict, + space.new_interned_str('TaskletExit'), + self.w_TaskletExit) + space.setitem(space.builtin.w_dict, + space.new_interned_str('TaskletExit'), + self.w_TaskletExit) def post_install(self): self.current = self.main = AppCoroutine(self.space, state=self) Modified: pypy/trunk/pypy/module/_stackless/rcoroutine.py ============================================================================== --- pypy/trunk/pypy/module/_stackless/rcoroutine.py (original) +++ pypy/trunk/pypy/module/_stackless/rcoroutine.py Thu Jul 1 20:51:08 2010 @@ -7,3 +7,4 @@ BaseCoState = d['BaseCoState'] AbstractThunk = d['AbstractThunk'] syncstate = d['syncstate'] +CoroutineExit = d['CoroutineExit'] Modified: pypy/trunk/pypy/module/_stackless/test/test_coroutine.py ============================================================================== --- pypy/trunk/pypy/module/_stackless/test/test_coroutine.py (original) +++ pypy/trunk/pypy/module/_stackless/test/test_coroutine.py Thu Jul 1 20:51:08 2010 @@ -84,8 +84,7 @@ assert not co.is_alive def test_kill_running(self): - skip("kill is not really working (there is only CoroutineExit, " - "which is not an app-level exception)") + coroutineexit = [] import _stackless as stackless main = stackless.coroutine.getcurrent() result = [] @@ -96,6 +95,9 @@ result.append(1) main.switch() x = 3 + except CoroutineExit: + coroutineexit.append(True) + raise finally: result.append(x) result.append(4) @@ -107,6 +109,7 @@ co.kill() assert not co.is_alive assert result == [1, 2] + assert coroutineexit == [True] def test_bogus_bind(self): import _stackless as stackless Modified: pypy/trunk/pypy/rlib/rcoroutine.py ============================================================================== --- pypy/trunk/pypy/rlib/rcoroutine.py (original) +++ pypy/trunk/pypy/rlib/rcoroutine.py Thu Jul 1 20:51:08 2010 @@ -32,6 +32,8 @@ from pypy.rlib.rstack import yield_current_frame_to_caller, resume_point from pypy.rlib.objectmodel import we_are_translated +from pypy.interpreter.error import OperationError + try: from greenlet import greenlet main_greenlet = greenlet.getcurrent() @@ -158,6 +160,7 @@ self.costate = state self.parent = None self.thunk = None + self.coroutine_exit = False def __repr__(self): 'NOT_RPYTHON' @@ -236,11 +239,12 @@ self = state.current self.finish(exc) except CoroutineExit: - # ignore a shutdown exception pass except Exception, e: - # redirect all unhandled exceptions to the parent - syncstate.push_exception(e) + if self.coroutine_exit is False: + # redirect all unhandled exceptions to the parent + syncstate.push_exception(e) + while self.parent is not None and self.parent.frame is None: # greenlet behavior is fine self.parent = self.parent.parent @@ -257,10 +261,13 @@ syncstate.switched(incoming_frame) def kill(self): + self._kill(CoroutineExit()) + + def _kill(self, exc): if self.frame is None: return state = self.costate - syncstate.push_exception(CoroutineExit()) + syncstate.push_exception(exc) # careful here - if setting self.parent to state.current would # create a loop, break it. The assumption is that 'self' # will die, so that state.current's chain of parents can be From benjamin at codespeak.net Thu Jul 1 20:51:21 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 20:51:21 +0200 (CEST) Subject: [pypy-svn] r75736 - in pypy/branch/fast-forward/pypy/rlib: . test Message-ID: <20100701185121.871AF282BDE@codespeak.net> Author: benjamin Date: Thu Jul 1 20:51:19 2010 New Revision: 75736 Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py Log: import useful things from the math module when possible; add copysign Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Thu Jul 1 20:51:19 2010 @@ -53,14 +53,25 @@ INFINITY = 1e200 * 1e200 NAN = INFINITY / INFINITY -def isinf(x): - return x != 0.0 and x / 2 == x +try: + from math import isinf, isnan, copysign +except ImportError: + def isinf(x): + return x != 0.0 and x / 2 == x + + # To get isnan, working x-platform and both on 2.3 and 2.4, is a + # horror. I think this works (for reasons I don't really want to talk + # about), and probably when implemented on top of pypy, too. + def isnan(v): + return v != v*1.0 or (v == 1.0 and v == 2.0) + + def copysign(x, y): + """Return x with the sign of y""" + if y > 0. or (y == 0. and math.atan2(y, -1.) > 0.): + return math.fabs(x) + else: + return -math.fabs(x) -# To get isnan, working x-platform and both on 2.3 and 2.4, is a -# horror. I think this works (for reasons I don't really want to talk -# about), and probably when implemented on top of pypy, too. -def isnan(v): - return v != v*1.0 or (v == 1.0 and v == 2.0) def intmask(n): if isinstance(n, int): Modified: pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py Thu Jul 1 20:51:19 2010 @@ -372,3 +372,10 @@ def test_int_real_union(): from pypy.rpython.lltypesystem.rffi import r_int_real assert compute_restype(r_int_real, r_int_real) is r_int_real + +def test_copysign(): + assert copysign(1, 1) == 1 + assert copysign(-1, 1) == 1 + assert copysign(-1, -1) == -1 + assert copysign(1, -1) == -1 + assert copysign(1, -0.) == -1 From afa at codespeak.net Thu Jul 1 22:24:33 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 22:24:33 +0200 (CEST) Subject: [pypy-svn] r75737 - in pypy/trunk/pypy: interpreter interpreter/pyparser module/_codecs module/_codecs/test objspace/std rlib Message-ID: <20100701202433.CCF4E282BE0@codespeak.net> Author: afa Date: Thu Jul 1 22:24:32 2010 New Revision: 75737 Removed: pypy/trunk/pypy/module/_codecs/app_codecs.py Modified: pypy/trunk/pypy/interpreter/pyparser/parsestring.py pypy/trunk/pypy/interpreter/unicodehelper.py pypy/trunk/pypy/module/_codecs/__init__.py pypy/trunk/pypy/module/_codecs/interp_codecs.py pypy/trunk/pypy/module/_codecs/test/test_codecs.py pypy/trunk/pypy/objspace/std/marshal_impl.py pypy/trunk/pypy/objspace/std/unicodeobject.py pypy/trunk/pypy/rlib/rstring.py pypy/trunk/pypy/rlib/runicode.py Log: Merge branch/interplevel-codecs: Rewrite all encodings implemented at applevel, move most of them to rlib.runicode where they may be useful for RPython programs. - This fixes translation with -O0: this option disables geninterp, and uses the pypy compiler instead to compile applevel code. But unicode literals need to be decoded with the codec module... - This also removes some huge geninterp'd code: app_codecs.py used to contain 3 or 4 functions each rendered with 20000 lines of C code! + use StringBuilder everywhere Modified: pypy/trunk/pypy/interpreter/pyparser/parsestring.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/parsestring.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/parsestring.py Thu Jul 1 22:24:32 2010 @@ -72,12 +72,12 @@ bufp = 0 bufq = len(buf) assert 0 <= bufp <= bufq - w_substr = space.wrap(buf[bufp : bufq]) + substr = buf[bufp:bufq] if rawmode: - w_v = unicodehelper.PyUnicode_DecodeRawUnicodeEscape(space, w_substr) + v = unicodehelper.PyUnicode_DecodeRawUnicodeEscape(space, substr) else: - w_v = unicodehelper.PyUnicode_DecodeUnicodeEscape(space, w_substr) - return w_v + v = unicodehelper.PyUnicode_DecodeUnicodeEscape(space, substr) + return space.wrap(v) need_encoding = (encoding is not None and encoding != "utf-8" and encoding != "iso-8859-1") @@ -86,7 +86,7 @@ substr = s[ps : q] if rawmode or '\\' not in s[ps:]: if need_encoding: - w_u = unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(substr)) + w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, substr)) #w_v = space.wrap(space.unwrap(w_u).encode(encoding)) this works w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding)) return w_v @@ -96,7 +96,7 @@ enc = None if need_encoding: enc = encoding - v = PyString_DecodeEscape(space, substr, unicode, enc) + v = PyString_DecodeEscape(space, substr, enc) return space.wrap(v) def hexbyte(val): @@ -105,10 +105,9 @@ result = "0" + result return result -def PyString_DecodeEscape(space, s, unicode, recode_encoding): +def PyString_DecodeEscape(space, s, recode_encoding): """ - Unescape a backslash-escaped string. If unicode is non-zero, - the string is a u-literal. If recode_encoding is non-zero, + Unescape a backslash-escaped string. If recode_encoding is non-zero, the string is UTF-8 encoded and should be re-encoded in the specified encoding. """ @@ -171,9 +170,6 @@ raise_app_valueerror(space, 'invalid \\x escape') # ignored replace and ignore for now - elif unicode and (ch == 'u' or ch == 'U' or ch == 'N'): - raise_app_valueerror(space, 'Unicode escapes not legal ' - 'when Unicode disabled') else: # this was not an escape, so the backslash # has to be added, and we start over in @@ -200,7 +196,7 @@ # while (s < end && *s != '\\') s++; */ /* inefficient for u".." while ps < end and ord(s[ps]) & 0x80: ps += 1 - w_u = unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(s[pt : ps])) + w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, s[pt:ps])) w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding)) v = space.str_w(w_v) return v, ps Modified: pypy/trunk/pypy/interpreter/unicodehelper.py ============================================================================== --- pypy/trunk/pypy/interpreter/unicodehelper.py (original) +++ pypy/trunk/pypy/interpreter/unicodehelper.py Thu Jul 1 22:24:32 2010 @@ -1,30 +1,10 @@ -from pypy.interpreter import gateway +from pypy.module._codecs import interp_codecs -app = gateway.applevel(r''' - def PyUnicode_DecodeUnicodeEscape(data): - import _codecs - return _codecs.unicode_escape_decode(data)[0] +def PyUnicode_AsEncodedString(space, w_data, w_encoding): + return interp_codecs.encode(space, w_data, w_encoding) - def PyUnicode_DecodeRawUnicodeEscape(data): - import _codecs - return _codecs.raw_unicode_escape_decode(data)[0] - - def PyUnicode_DecodeUTF8(data): - import _codecs - return _codecs.utf_8_decode(data)[0] - - def PyUnicode_AsEncodedString(data, encoding): - import _codecs - return _codecs.encode(data, encoding) - - def PyUnicode_EncodeUTF8(data): - import _codecs - return _codecs.utf_8_encode(data)[0] - -''') - -PyUnicode_DecodeUnicodeEscape = app.interphook('PyUnicode_DecodeUnicodeEscape') -PyUnicode_DecodeRawUnicodeEscape = app.interphook('PyUnicode_DecodeRawUnicodeEscape') -PyUnicode_DecodeUTF8 = app.interphook('PyUnicode_DecodeUTF8') -PyUnicode_AsEncodedString = app.interphook('PyUnicode_AsEncodedString') -PyUnicode_EncodeUTF8 = app.interphook('PyUnicode_EncodeUTF8') +# These functions take and return unwrapped rpython strings and unicodes +PyUnicode_DecodeUnicodeEscape = interp_codecs.make_raw_decoder('unicode_escape') +PyUnicode_DecodeRawUnicodeEscape = interp_codecs.make_raw_decoder('raw_unicode_escape') +PyUnicode_DecodeUTF8 = interp_codecs.make_raw_decoder('utf_8') +PyUnicode_EncodeUTF8 = interp_codecs.make_raw_encoder('utf_8') Modified: pypy/trunk/pypy/module/_codecs/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_codecs/__init__.py (original) +++ pypy/trunk/pypy/module/_codecs/__init__.py Thu Jul 1 22:24:32 2010 @@ -3,22 +3,43 @@ from pypy.module._codecs import interp_codecs class Module(MixedModule): - appleveldefs = { - '__doc__' : 'app_codecs.__doc__', - '__name__' : 'app_codecs.__name__', - 'charmap_encode' : 'app_codecs.charmap_encode', - 'escape_decode' : 'app_codecs.escape_decode', - 'escape_encode' : 'app_codecs.escape_encode', - 'raw_unicode_escape_decode' : 'app_codecs.raw_unicode_escape_decode', - 'raw_unicode_escape_encode' : 'app_codecs.raw_unicode_escape_encode', - 'unicode_escape_decode' : 'app_codecs.unicode_escape_decode', - 'unicode_escape_encode' : 'app_codecs.unicode_escape_encode', - 'unicode_internal_decode' : 'app_codecs.unicode_internal_decode', - 'unicode_internal_encode' : 'app_codecs.unicode_internal_encode', - 'utf_7_decode' : 'app_codecs.utf_7_decode', - 'utf_7_encode' : 'app_codecs.utf_7_encode', - 'charmap_build' : 'app_codecs.charmap_build' - } + """ + _codecs -- Provides access to the codec registry and the builtin + codecs. + + This module should never be imported directly. The standard library + module "codecs" wraps this builtin module for use within Python. + + The codec registry is accessible via: + + register(search_function) -> None + + lookup(encoding) -> (encoder, decoder, stream_reader, stream_writer) + + The builtin Unicode codecs use the following interface: + + _encode(Unicode_object[,errors='strict']) -> + (string object, bytes consumed) + + _decode(char_buffer_obj[,errors='strict']) -> + (Unicode object, bytes consumed) + + _encode() interfaces also accept non-Unicode object as + input. The objects are then converted to Unicode using + PyUnicode_FromObject() prior to applying the conversion. + + These s are available: utf_8, unicode_escape, + raw_unicode_escape, unicode_internal, latin_1, ascii (7-bit), + mbcs (on win32). + + +Written by Marc-Andre Lemburg (mal at lemburg.com). + +Copyright (c) Corporation for National Research Initiatives. +""" + + appleveldefs = {} + interpleveldefs = { 'encode': 'interp_codecs.encode', 'decode': 'interp_codecs.decode', @@ -26,12 +47,15 @@ 'lookup_error': 'interp_codecs.lookup_error', 'register': 'interp_codecs.register_codec', 'register_error': 'interp_codecs.register_error', + 'charmap_build' : 'interp_codecs.charmap_build', # encoders and decoders 'ascii_decode' : 'interp_codecs.ascii_decode', 'ascii_encode' : 'interp_codecs.ascii_encode', 'latin_1_decode' : 'interp_codecs.latin_1_decode', 'latin_1_encode' : 'interp_codecs.latin_1_encode', + 'utf_7_decode' : 'interp_codecs.utf_7_decode', + 'utf_7_encode' : 'interp_codecs.utf_7_encode', 'utf_8_decode' : 'interp_codecs.utf_8_decode', 'utf_8_encode' : 'interp_codecs.utf_8_encode', 'utf_16_be_decode' : 'interp_codecs.utf_16_be_decode', @@ -44,6 +68,15 @@ 'charbuffer_encode': 'interp_codecs.buffer_encode', 'readbuffer_encode': 'interp_codecs.buffer_encode', 'charmap_decode' : 'interp_codecs.charmap_decode', + 'charmap_encode' : 'interp_codecs.charmap_encode', + 'escape_encode' : 'interp_codecs.escape_encode', + 'escape_decode' : 'interp_codecs.escape_decode', + 'unicode_escape_decode' : 'interp_codecs.unicode_escape_decode', + 'unicode_escape_encode' : 'interp_codecs.unicode_escape_encode', + 'raw_unicode_escape_decode' : 'interp_codecs.raw_unicode_escape_decode', + 'raw_unicode_escape_encode' : 'interp_codecs.raw_unicode_escape_encode', + 'unicode_internal_decode' : 'interp_codecs.unicode_internal_decode', + 'unicode_internal_encode' : 'interp_codecs.unicode_internal_encode', } def __init__(self, space, *args): Modified: pypy/trunk/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/trunk/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/trunk/pypy/module/_codecs/interp_codecs.py Thu Jul 1 22:24:32 2010 @@ -1,5 +1,6 @@ from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import ObjSpace, NoneNotWrapped, applevel +from pypy.interpreter.gateway import ObjSpace, NoneNotWrapped, interp2app +from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.baseobjspace import W_Root from pypy.rlib.rstring import StringBuilder, UnicodeBuilder from pypy.rlib.objectmodel import we_are_translated @@ -13,6 +14,8 @@ self.decode_error_handler = self.make_errorhandler(space, True) self.encode_error_handler = self.make_errorhandler(space, False) + self.unicodedata_handler = None + def make_errorhandler(self, space, decode): def unicode_call_errorhandler(errors, encoding, reason, input, startpos, endpos): @@ -53,6 +56,21 @@ return replace, newpos return unicode_call_errorhandler + def get_unicodedata_handler(self, space): + if self.unicodedata_handler: + return self.unicodedata_handler + try: + w_builtin = space.getbuiltinmodule('__builtin__') + w_import = space.getattr(w_builtin, space.wrap("__import__")) + w_unicodedata = space.call_function(w_import, + space.wrap("unicodedata")) + w_getcode = space.getattr(w_unicodedata, space.wrap("_get_code")) + except OperationError: + return None + else: + self.unicodedata_handler = UnicodeData_Handler(space, w_getcode) + return self.unicodedata_handler + def _freeze_(self): assert not self.codec_search_path return False @@ -114,78 +132,125 @@ "unknown encoding: %s", encoding) lookup_codec.unwrap_spec = [ObjSpace, str] -app_errors = applevel(""" -def check_exception(exc): +# ____________________________________________________________ +# Register standard error handlers + +def check_exception(space, w_exc): try: - delta = exc.end - exc.start - if delta < 0 or not isinstance(exc.object, (unicode, str)): - raise TypeError("wrong exception") - except AttributeError: - raise TypeError("wrong exception") - -def strict_errors(exc): - if isinstance(exc, Exception): - raise exc - else: - raise TypeError("codec must pass exception instance") - -def ignore_errors(exc): - check_exception(exc) - if isinstance(exc, UnicodeEncodeError): - return u'', exc.end - elif isinstance(exc, (UnicodeDecodeError, UnicodeTranslateError)): - return u'', exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%exc) - -Py_UNICODE_REPLACEMENT_CHARACTER = u"\ufffd" - -def replace_errors(exc): - check_exception(exc) - if isinstance(exc, UnicodeEncodeError): - return u'?'*(exc.end-exc.start), exc.end - elif isinstance(exc, (UnicodeTranslateError, UnicodeDecodeError)): - return Py_UNICODE_REPLACEMENT_CHARACTER*(exc.end-exc.start), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%exc) - -def xmlcharrefreplace_errors(exc): - if isinstance(exc, UnicodeEncodeError): - res = [] - for ch in exc.object[exc.start:exc.end]: - res += '&#' - res += str(ord(ch)) - res += ';' - return u''.join(res), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) - -def backslashreplace_errors(exc): - if isinstance(exc, UnicodeEncodeError): - p = [] - for c in exc.object[exc.start:exc.end]: - p += '\\\\' - oc = ord(c) - if (oc >= 0x00010000): - p += 'U' - p += "%.8x" % ord(c) + w_start = space.getattr(w_exc, space.wrap('start')) + w_end = space.getattr(w_exc, space.wrap('end')) + w_obj = space.getattr(w_exc, space.wrap('object')) + except OperationError, e: + if not e.match(space, space.w_AttributeError): + raise + raise OperationError(space.w_TypeError, space.wrap( + "wrong exception")) + + delta = space.int_w(w_end) - space.int_w(w_start) + if delta < 0 or not (space.isinstance_w(w_obj, space.w_str) or + space.isinstance_w(w_obj, space.w_unicode)): + raise OperationError(space.w_TypeError, space.wrap( + "wrong exception")) + +def strict_errors(space, w_exc): + check_exception(space, w_exc) + if space.isinstance_w(w_exc, space.w_BaseException): + raise OperationError(space.type(w_exc), w_exc) + else: + raise OperationError(space.w_TypeError, space.wrap( + "codec must pass exception instance")) + +def ignore_errors(space, w_exc): + check_exception(space, w_exc) + w_end = space.getattr(w_exc, space.wrap('end')) + if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): + return space.newtuple([space.wrap(''), w_end]) + elif (space.isinstance_w(w_exc, space.w_UnicodeDecodeError) or + space.isinstance_w(w_exc, space.w_UnicodeTranslateError)): + return space.newtuple([space.wrap(u''), w_end]) + else: + typename = space.type(w_exc).getname(space, '?') + raise operationerrfmt(space.w_TypeError, + "don't know how to handle %s in error callback", typename) + +def replace_errors(space, w_exc): + check_exception(space, w_exc) + w_start = space.getattr(w_exc, space.wrap('start')) + w_end = space.getattr(w_exc, space.wrap('end')) + size = space.int_w(w_end) - space.int_w(w_start) + if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): + text = '?' * size + return space.newtuple([space.wrap(text), w_end]) + elif (space.isinstance_w(w_exc, space.w_UnicodeDecodeError) or + space.isinstance_w(w_exc, space.w_UnicodeTranslateError)): + text = u'\ufffd' * size + return space.newtuple([space.wrap(text), w_end]) + else: + typename = space.type(w_exc).getname(space, '?') + raise operationerrfmt(space.w_TypeError, + "don't know how to handle %s in error callback", typename) + +def xmlcharrefreplace_errors(space, w_exc): + check_exception(space, w_exc) + if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): + obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object'))) + start = space.int_w(space.getattr(w_exc, space.wrap('start'))) + w_end = space.getattr(w_exc, space.wrap('end')) + end = space.int_w(w_end) + builder = UnicodeBuilder() + pos = start + while pos < end: + ch = obj[pos] + builder.append(u"&#") + builder.append(unicode(str(ord(ch)))) + builder.append(u";") + pos += 1 + return space.newtuple([space.wrap(builder.build()), w_end]) + else: + typename = space.type(w_exc).getname(space, '?') + raise operationerrfmt(space.w_TypeError, + "don't know how to handle %s in error callback", typename) + +def backslashreplace_errors(space, w_exc): + check_exception(space, w_exc) + if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): + obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object'))) + start = space.int_w(space.getattr(w_exc, space.wrap('start'))) + w_end = space.getattr(w_exc, space.wrap('end')) + end = space.int_w(w_end) + builder = UnicodeBuilder() + pos = start + while pos < end: + oc = ord(obj[pos]) + num = hex(oc) + if (oc >= 0x10000): + builder.append(u"\\U") + zeros = 8 elif (oc >= 0x100): - p += 'u' - p += "%.4x" % ord(c) + builder.append(u"\\u") + zeros = 4 else: - p += 'x' - p += "%.2x" % ord(c) - return u''.join(p), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) -""") + builder.append(u"\\x") + zeros = 2 + lnum = len(num) + nb = zeros + 2 - lnum # num starts with '0x' + if nb > 0: + builder.append_multiple_char(u'0', nb) + builder.append_slice(unicode(num), 2, lnum) + pos += 1 + return space.newtuple([space.wrap(builder.build()), w_end]) + else: + typename = space.type(w_exc).getname(space, '?') + raise operationerrfmt(space.w_TypeError, + "don't know how to handle %s in error callback", typename) def register_builtin_error_handlers(space): + "NOT_RPYTHON" state = space.fromcache(CodecState) for error in ("strict", "ignore", "replace", "xmlcharrefreplace", "backslashreplace"): name = error + "_errors" - state.codec_error_registry[error] = app_errors.wget(space, name) + state.codec_error_registry[error] = space.wrap(interp2app(globals()[name])) def lookup_error(space, errors): @@ -279,6 +344,38 @@ from pypy.rlib import runicode +def make_raw_encoder(name): + rname = "unicode_encode_%s" % (name.replace("_encode", ""), ) + assert hasattr(runicode, rname) + def raw_encoder(space, uni): + state = space.fromcache(CodecState) + func = getattr(runicode, rname) + errors = "strict" + return func(uni, len(uni), errors, state.encode_error_handler) + raw_encoder.func_name = rname + return raw_encoder + +def make_raw_decoder(name): + rname = "str_decode_%s" % (name.replace("_decode", ""), ) + assert hasattr(runicode, rname) + def raw_decoder(space, string): + final = True + errors = "strict" + state = space.fromcache(CodecState) + func = getattr(runicode, rname) + kwargs = {} + if name == 'unicode_escape': + unicodedata_handler = state.get_unicodedata_handler(space) + result, consumed = func(string, len(string), errors, + final, state.decode_error_handler, + unicodedata_handler=unicodedata_handler) + else: + result, consumed = func(string, len(string), errors, + final, state.decode_error_handler) + return result + raw_decoder.func_name = rname + return raw_decoder + def make_encoder_wrapper(name): rname = "unicode_encode_%s" % (name.replace("_encode", ""), ) assert hasattr(runicode, rname) @@ -308,20 +405,26 @@ for encoders in [ "ascii_encode", "latin_1_encode", + "utf_7_encode", "utf_8_encode", "utf_16_encode", "utf_16_be_encode", "utf_16_le_encode", + "unicode_escape_encode", + "raw_unicode_escape_encode", + "unicode_internal_encode", ]: make_encoder_wrapper(encoders) for decoders in [ "ascii_decode", "latin_1_decode", + "utf_7_decode", "utf_8_decode", "utf_16_decode", "utf_16_be_decode", "utf_16_le_decode", + "raw_unicode_escape_decode", ]: make_decoder_wrapper(decoders) @@ -330,8 +433,6 @@ make_decoder_wrapper('mbcs_decode') def utf_16_ex_decode(space, data, errors='strict', byteorder=0, w_final=False): - """None - """ final = space.is_true(w_final) state = space.fromcache(CodecState) if byteorder == 0: @@ -349,77 +450,213 @@ space.wrap(byteorder)]) utf_16_ex_decode.unwrap_spec = [ObjSpace, str, str, int, W_Root] -def _extract_from_mapping(space, mapping_w, w_mapping, ch): - if mapping_w is not None: +# ____________________________________________________________ +# Charmap + +class Charmap_Decode: + def __init__(self, space, w_mapping): + self.space = space + self.w_mapping = w_mapping + + # fast path for all the stuff in the encodings module + if space.is_true(space.isinstance(w_mapping, space.w_tuple)): + self.mapping_w = space.fixedview(w_mapping) + else: + self.mapping_w = None + + def get(self, ch, errorchar): + space = self.space + + # get the character from the mapping + if self.mapping_w is not None: + w_ch = self.mapping_w[ord(ch)] + else: + try: + w_ch = space.getitem(self.w_mapping, space.newint(ord(ch))) + except OperationError, e: + if not e.match(space, space.w_LookupError): + raise + return errorchar + + # Charmap may return a unicode string try: - return mapping_w[ord(ch)] - except IndexError: - pass - else: + x = space.unicode_w(w_ch) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + else: + return x + + # Charmap may return a number + try: + x = space.int_w(w_ch) + except OperationError: + if not e.match(space, space.w_TypeError): + raise + else: + if 0 <= x < 65536: # Even on wide unicode builds... + return unichr(x) + else: + raise OperationError(space.w_TypeError, space.wrap( + "character mapping must be in range(65536)")) + + # Charmap may return None + if space.is_w(w_ch, space.w_None): + return errorchar + + raise OperationError(space.w_TypeError, space.wrap("invalid mapping")) + +class Charmap_Encode: + def __init__(self, space, w_mapping): + self.space = space + self.w_mapping = w_mapping + + def get(self, ch, errorchar): + space = self.space + + # get the character from the mapping try: - return space.getitem(w_mapping, space.newint(ord(ch))) + w_ch = space.getitem(self.w_mapping, space.newint(ord(ch))) except OperationError, e: - if (not e.match(space, space.w_KeyError) and - not e.match(space, space.w_IndexError)): + if not e.match(space, space.w_LookupError): raise - pass + return errorchar -def _append_unicode(space, builder, w_x): - try: - x = space.unicode_w(w_x) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - else: - if x != u"\ufffe": - builder.append(x) - return True - return False - try: - x = space.int_w(w_x) - except OperationError: - if not e.match(space, space.w_TypeError): - raise - else: - if x < 65536: - builder.append(unichr(x)) + # Charmap may return a string + try: + x = space.realstr_w(w_ch) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise else: - raise OperationError(space.w_TypeError, space.wrap("character mapping must be in range(65536)")) - return True - if not space.is_true(w_x): - return False + return x + + # Charmap may return a number + try: + x = space.int_w(w_ch) + except OperationError: + if not e.match(space, space.w_TypeError): + raise + else: + if 0 <= x < 256: + return chr(x) + else: + raise OperationError(space.w_TypeError, space.wrap( + "character mapping must be in range(256)")) + + # Charmap may return None + if space.is_w(w_ch, space.w_None): + return errorchar + + raise OperationError(space.w_TypeError, space.wrap("invalid mapping")) + + + at unwrap_spec(ObjSpace, str, str, W_Root) +def charmap_decode(space, string, errors="strict", w_mapping=None): + if len(string) == 0: + return space.newtuple([space.wrap(u''), space.wrap(0)]) + + if space.is_w(w_mapping, space.w_None): + mapping = None else: - raise OperationError(space.w_TypeError, space.w_None) + mapping = Charmap_Decode(space, w_mapping) + + final = True + state = space.fromcache(CodecState) + result, consumed = runicode.str_decode_charmap( + string, len(string), errors, + final, state.decode_error_handler, mapping) + return space.newtuple([space.wrap(result), space.wrap(consumed)]) + + at unwrap_spec(ObjSpace, unicode, str, W_Root) +def charmap_encode(space, uni, errors="strict", w_mapping=None): + if space.is_w(w_mapping, space.w_None): + mapping = None + else: + mapping = Charmap_Encode(space, w_mapping) + + state = space.fromcache(CodecState) + result = runicode.unicode_encode_charmap( + uni, len(uni), errors, + state.encode_error_handler, mapping) + return space.newtuple([space.wrap(result), space.wrap(len(uni))]) -def charmap_decode(space, s, errors="strict", w_mapping=None): - size = len(s) - # Default to Latin-1 - if space.is_true(space.is_(w_mapping, space.w_None)): - return latin_1_decode(space, s, errors, space.w_False) + at unwrap_spec(ObjSpace, unicode) +def charmap_build(space, chars): + # XXX CPython sometimes uses a three-level trie + w_charmap = space.newdict() + for num in range(len(chars)): + elem = chars[num] + space.setitem(w_charmap, space.newint(ord(elem)), space.newint(num)) + return w_charmap - if (size == 0): +# ____________________________________________________________ +# Unicode escape + +class UnicodeData_Handler: + def __init__(self, space, w_getcode): + self.space = space + self.w_getcode = w_getcode + + def call(self, name): + space = self.space + try: + w_code = space.call_function(self.w_getcode, space.wrap(name)) + except OperationError, e: + if not e.match(space, space.w_KeyError): + raise + return -1 + return space.int_w(w_code) + + at unwrap_spec(ObjSpace, 'bufferstr', str, W_Root) +def unicode_escape_decode(space, string, errors="strict", w_final=False): + final = space.is_true(w_final) + state = space.fromcache(CodecState) + errorhandler=state.decode_error_handler + + unicode_name_handler = state.get_unicodedata_handler(space) + + result, consumed = runicode.str_decode_unicode_escape( + string, len(string), errors, + final, state.decode_error_handler, + unicode_name_handler) + + return space.newtuple([space.wrap(result), space.wrap(consumed)]) + +# ____________________________________________________________ +# Unicode-internal + + at unwrap_spec(ObjSpace, W_Root, str) +def unicode_internal_decode(space, w_string, errors="strict"): + # special case for this codec: unicodes are returned as is + if space.isinstance_w(w_string, space.w_unicode): + return space.newtuple([w_string, space.len(w_string)]) + + string = space.str_w(w_string) + + if len(string) == 0: return space.newtuple([space.wrap(u''), space.wrap(0)]) - - # fast path for all the stuff in the encodings module - if space.is_true(space.isinstance(w_mapping, space.w_tuple)): - mapping_w = space.fixedview(w_mapping) - else: - mapping_w = None - - builder = UnicodeBuilder(size) - inpos = 0 - while (inpos < len(s)): - #/* Get mapping_w (char ordinal -> integer, Unicode char or None) */ - ch = s[inpos] - w_x = _extract_from_mapping(space, mapping_w, w_mapping, ch) - if w_x is not None and _append_unicode(space, builder, w_x): - inpos += 1 - continue - state = space.fromcache(CodecState) - next, inpos = state.decode_error_handler(errors, "charmap", - "character maps to ", s, inpos, inpos+1) - builder.append(next) - res = builder.build() - return space.newtuple([space.wrap(res), space.wrap(size)]) -charmap_decode.unwrap_spec = [ObjSpace, str, str, W_Root] + + final = True + state = space.fromcache(CodecState) + result, consumed = runicode.str_decode_unicode_internal( + string, len(string), errors, + final, state.decode_error_handler) + return space.newtuple([space.wrap(result), space.wrap(consumed)]) + +# ____________________________________________________________ +# support for the "string escape" codec +# This is a bytes-to bytes transformation + + at unwrap_spec(ObjSpace, W_Root, str) +def escape_encode(space, w_string, errors='strict'): + w_repr = space.repr(w_string) + w_result = space.getslice(w_repr, space.wrap(1), space.wrap(-1)) + return space.newtuple([w_result, space.len(w_string)]) + + at unwrap_spec(ObjSpace, str, str) +def escape_decode(space, data, errors='strict'): + from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape + result = PyString_DecodeEscape(space, data, None) + return space.newtuple([space.wrap(result), space.wrap(len(data))]) Modified: pypy/trunk/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/trunk/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/trunk/pypy/module/_codecs/test/test_codecs.py Thu Jul 1 22:24:32 2010 @@ -1,7 +1,5 @@ import autopath from pypy.conftest import gettestobjspace -from pypy.module._codecs.app_codecs import unicode_escape_encode,\ - charmap_encode, unicode_escape_decode class AppTestCodecs: @@ -14,26 +12,16 @@ raises(TypeError, _codecs.register, 1) def test_bigU_codecs(self): - import sys - oldmaxunicode = sys.maxunicode - if sys.maxunicode <= 0xffff: - return # this test cannot run on UCS2 builds u = u'\U00010001\U00020002\U00030003\U00040004\U00050005' for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', 'raw_unicode_escape', 'unicode_escape', 'unicode_internal'): assert unicode(u.encode(encoding),encoding) == u - sys.maxunicode = oldmaxunicode def test_ucs4(self): - import sys - oldmaxunicode = sys.maxunicode - if sys.maxunicode <= 0xffff: - sys.maxunicode = 0xffffffff x = u'\U00100000' y = x.encode("raw-unicode-escape").decode("raw-unicode-escape") assert x == y - sys.maxunicode = oldmaxunicode def test_named_unicode(self): assert unicode('\\N{SPACE}','unicode-escape') == u" " @@ -118,12 +106,20 @@ def test_charmap_decode(self): from _codecs import charmap_decode + import sys assert charmap_decode('', 'strict', 'blablabla') == ('', 0) assert charmap_decode('xxx') == ('xxx', 3) assert charmap_decode('xxx', 'strict', {ord('x'): u'XX'}) == ('XXXXXX', 3) map = tuple([unichr(i) for i in range(256)]) assert charmap_decode('xxx\xff', 'strict', map) == (u'xxx\xff', 4) + raises(TypeError, charmap_decode, '\xff', "replace", {0xff: 0x10001}) + + def test_unicode_escape(self): + from _codecs import unicode_escape_encode, unicode_escape_decode + assert unicode_escape_encode(u'abc') == (u'abc'.encode('unicode_escape'), 3) + assert unicode_escape_decode('abc') == (u'abc'.decode('unicode_escape'), 3) + assert unicode_escape_decode('\\x61\\x62\\x63') == (u'abc', 12) class AppTestPartialEvaluation: @@ -377,6 +373,9 @@ def test_charmap_decode_1(self): import codecs + assert codecs.charmap_encode(u'xxx') == ('xxx', 3) + assert codecs.charmap_encode(u'xxx', 'strict', {ord('x'): 'XX'}) == ('XXXXXX', 3) + res = codecs.charmap_decode("\x00\x01\x02", "replace", u"ab") assert res == (u"ab\ufffd", 3) res = codecs.charmap_decode("\x00\x01\x02", "replace", u"ab\ufffe") @@ -464,6 +463,9 @@ assert '\xff'.decode('utf-7', 'ignore') == '' assert '\x00'.decode('unicode-internal', 'ignore') == '' + def test_backslahreplace(self): + assert u'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == 'a\\xac\u1234\u20ac\u8000' + def test_badhandler(self): import codecs results = ( 42, u"foo", (1,2,3), (u"foo", 1, 3), (u"foo", None), (u"foo",), ("foo", 1, 3), ("foo", None), ("foo",) ) @@ -527,9 +529,26 @@ def test_charmap_encode(self): assert 'xxx'.encode('charmap') == 'xxx' + import codecs + raises(TypeError, codecs.charmap_encode, u'\xff', "replace", {0xff: 300}) + raises(UnicodeError, codecs.charmap_encode, u"\xff", "replace", {0xff: None}) + + def test_charmap_encode_replace(self): + charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"]) + charmap[ord("?")] = "XYZ" + import codecs + sin = u"abcDEF" + sout = codecs.charmap_encode(sin, "replace", charmap)[0] + assert sout == "AABBCCXYZXYZXYZ" + def test_charmap_decode_2(self): assert 'foo'.decode('charmap') == 'foo' + def test_charmap_build(self): + import codecs + assert codecs.charmap_build(u'123456') == {49: 0, 50: 1, 51: 2, + 52: 3, 53: 4, 54: 5} + def test_utf7_start_end_in_exception(self): try: '+IC'.decode('utf-7') @@ -537,6 +556,9 @@ assert exc.start == 0 assert exc.end == 3 + def test_utf7_surrogate(self): + raises(UnicodeDecodeError, '+3ADYAA-'.decode, 'utf-7') + def test_utf_16_encode_decode(self): import codecs x = u'123abc' @@ -546,6 +568,8 @@ def test_unicode_escape(self): assert u'\\'.encode('unicode-escape') == '\\\\' assert '\\\\'.decode('unicode-escape') == u'\\' + assert u'\ud801'.encode('unicode-escape') == '\\ud801' + assert u'\u0013'.encode('unicode-escape') == '\\x13' def test_mbcs(self): import sys @@ -555,14 +579,3 @@ assert u'caf\xe9'.encode('mbcs') == 'caf\xe9' assert u'\u040a'.encode('mbcs') == '?' # some cyrillic letter assert 'cafx\e9'.decode('mbcs') == u'cafx\e9' - - -class TestDirect: - def test_charmap_encode(self): - assert charmap_encode(u'xxx') == ('xxx', 3) - assert charmap_encode(u'xxx', 'strict', {ord('x'): 'XX'}) == ('XXXXXX', 6) - - def test_unicode_escape(self): - assert unicode_escape_encode(u'abc') == (u'abc'.encode('unicode_escape'), 3) - assert unicode_escape_decode('abc') == (u'abc'.decode('unicode_escape'), 3) - assert unicode_escape_decode('\\x61\\x62\\x63') == (u'abc', 12) Modified: pypy/trunk/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/trunk/pypy/objspace/std/marshal_impl.py (original) +++ pypy/trunk/pypy/objspace/std/marshal_impl.py Thu Jul 1 22:24:32 2010 @@ -447,11 +447,11 @@ register(TYPE_CODE, unmarshal_pycode) def marshal_w__Unicode(space, w_unicode, m): - s = space.str_w(unicodehelper.PyUnicode_EncodeUTF8(space, w_unicode)) + s = unicodehelper.PyUnicode_EncodeUTF8(space, space.unicode_w(w_unicode)) m.atom_str(TYPE_UNICODE, s) def unmarshal_Unicode(space, u, tc): - return unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(u.get_str())) + return space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, u.get_str())) register(TYPE_UNICODE, unmarshal_Unicode) app = gateway.applevel(r''' Modified: pypy/trunk/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/unicodeobject.py (original) +++ pypy/trunk/pypy/objspace/std/unicodeobject.py Thu Jul 1 22:24:32 2010 @@ -12,6 +12,7 @@ from pypy.rlib.rarithmetic import intmask, ovfcheck from pypy.rlib.objectmodel import compute_hash from pypy.rlib.rstring import string_repeat +from pypy.rlib.runicode import unicode_encode_unicode_escape from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb from pypy.tool.sourcetools import func_with_new_name @@ -892,101 +893,11 @@ space.wrap("character mapping must return integer, None or unicode")) return W_UnicodeObject(u''.join(result)) -# Move this into the _codecs module as 'unicodeescape_string (Remember to cater for quotes)' def repr__Unicode(space, w_unicode): - hexdigits = "0123456789abcdef" chars = w_unicode._value size = len(chars) - - singlequote = doublequote = False - for c in chars: - if c == u'\'': - singlequote = True - elif c == u'"': - doublequote = True - if singlequote and not doublequote: - quote = '"' - else: - quote = '\'' - result = ['u', quote] - j = 0 - while j= 0x10000: - # Resize if needed - result.extend(['\\', "U", - hexdigits[(code >> 28) & 0xf], - hexdigits[(code >> 24) & 0xf], - hexdigits[(code >> 20) & 0xf], - hexdigits[(code >> 16) & 0xf], - hexdigits[(code >> 12) & 0xf], - hexdigits[(code >> 8) & 0xf], - hexdigits[(code >> 4) & 0xf], - hexdigits[(code >> 0) & 0xf], - ]) - j += 1 - continue - if code >= 0xD800 and code < 0xDC00: - if j < size - 1: - ch2 = chars[j+1] - code2 = ord(ch2) - if code2 >= 0xDC00 and code2 <= 0xDFFF: - code = (((code & 0x03FF) << 10) | (code2 & 0x03FF)) + 0x00010000 - result.extend(['\\', "U", - hexdigits[(code >> 28) & 0xf], - hexdigits[(code >> 24) & 0xf], - hexdigits[(code >> 20) & 0xf], - hexdigits[(code >> 16) & 0xf], - hexdigits[(code >> 12) & 0xf], - hexdigits[(code >> 8) & 0xf], - hexdigits[(code >> 4) & 0xf], - hexdigits[(code >> 0) & 0xf], - ]) - j += 2 - continue - - if code >= 0x100: - result.extend(['\\', "u", - hexdigits[(code >> 12) & 0xf], - hexdigits[(code >> 8) & 0xf], - hexdigits[(code >> 4) & 0xf], - hexdigits[(code >> 0) & 0xf], - ]) - j += 1 - continue - if code == ord('\\') or code == ord(quote): - result.append('\\') - result.append(chr(code)) - j += 1 - continue - if code == ord('\t'): - result.append('\\') - result.append('t') - j += 1 - continue - if code == ord('\r'): - result.append('\\') - result.append('r') - j += 1 - continue - if code == ord('\n'): - result.append('\\') - result.append('n') - j += 1 - continue - if code < ord(' ') or code >= 0x7f: - result.extend(['\\', "x", - hexdigits[(code >> 4) & 0xf], - hexdigits[(code >> 0) & 0xf], - ]) - j += 1 - continue - result.append(chr(code)) - j += 1 - result.append(quote) - return space.wrap(''.join(result)) - + s = unicode_encode_unicode_escape(chars, size, "strict", quotes=True) + return space.wrap(s) def mod__Unicode_ANY(space, w_format, w_values): return mod_format(space, w_format, w_values, do_unicode=True) Modified: pypy/trunk/pypy/rlib/rstring.py ============================================================================== --- pypy/trunk/pypy/rlib/rstring.py (original) +++ pypy/trunk/pypy/rlib/rstring.py Thu Jul 1 22:24:32 2010 @@ -56,6 +56,7 @@ self.l.append(s) def append_slice(self, s, start, end): + assert 0 <= start <= end <= len(s) self.l.append(s[start:end]) def append_multiple_char(self, c, times): Modified: pypy/trunk/pypy/rlib/runicode.py ============================================================================== --- pypy/trunk/pypy/rlib/runicode.py (original) +++ pypy/trunk/pypy/rlib/runicode.py Thu Jul 1 22:24:32 2010 @@ -1,7 +1,9 @@ import sys from pypy.rlib.bitmanipulation import splitter from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize +from pypy.rlib.rstring import StringBuilder, UnicodeBuilder +from pypy.rlib.rarithmetic import r_uint if rffi.sizeof(lltype.UniChar) == 4: MAXUNICODE = 0x10ffff @@ -42,8 +44,6 @@ UNICHR = unichr ORD = ord -# XXX review the functions below and think about using stringbuilders for them - def raise_unicode_exception_decode(errors, encoding, msg, s, startingpos, endingpos): @@ -55,8 +55,8 @@ assert isinstance(u, unicode) raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg) -# ____________________________________________________________ -# unicode decoding +# ____________________________________________________________ +# utf-8 utf8_code_length = [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -81,9 +81,10 @@ errorhandler=None): if errorhandler is None: errorhandler = raise_unicode_exception_decode - if (size == 0): + if size == 0: return u'', 0 - result = [] + + result = UnicodeBuilder(size) pos = 0 while pos < size: ch = s[pos] @@ -94,14 +95,14 @@ continue n = utf8_code_length[ordch1] - if (pos + n > size): + if pos + n > size: if not final: break else: r, pos = errorhandler(errors, "utf-8", "unexpected end of data", s, pos, size) result.append(r) - if (pos + n > size): + if pos + n > size: break if n == 0: r, pos = errorhandler(errors, "utf-8", "unexpected code byte", @@ -116,7 +117,7 @@ z, two = splitter[6, 2](ordch2) y, six = splitter[5, 3](ordch1) assert six == 6 - if (two != 2): + if two != 2: r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 2) result.append(r) @@ -137,7 +138,7 @@ y, two2 = splitter[6, 2](ordch2) x, fourteen = splitter[4, 4](ordch1) assert fourteen == 14 - if (two1 != 2 or two2 != 2): + if two1 != 2 or two2 != 2: r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 3) result.append(r) @@ -166,7 +167,7 @@ x, two3 = splitter[6, 2](ordch2) w, thirty = splitter[3, 5](ordch1) assert thirty == 30 - if (two1 != 2 or two2 != 2 or two3 != 2): + if two1 != 2 or two2 != 2 or two3 != 2: r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 4) result.append(r) @@ -174,7 +175,7 @@ c = (w << 18) + (x << 12) + (y << 6) + z # minimum value allowed for 4 byte encoding # maximum value allowed for UTF-16 - if ((c < 0x10000) or (c > 0x10ffff)): + if c < 0x10000 or c > 0x10ffff: r, pos = errorhandler(errors, "utf-8", "illegal encoding", s, pos, pos + 4) result.append(r) @@ -197,8 +198,53 @@ s, pos, pos + n) result.append(r) - return u"".join(result), pos + return result.build(), pos +def _encodeUCS4(result, ch): + # Encode UCS4 Unicode ordinals + result.append((chr((0xf0 | (ch >> 18))))) + result.append((chr((0x80 | ((ch >> 12) & 0x3f))))) + result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) + result.append((chr((0x80 | (ch & 0x3f))))) + +def unicode_encode_utf_8(s, size, errors, errorhandler=None): + assert(size >= 0) + result = StringBuilder(size) + i = 0 + while i < size: + ch = ord(s[i]) + i += 1 + if ch < 0x80: + # Encode ASCII + result.append(chr(ch)) + elif ch < 0x0800: + # Encode Latin-1 + result.append(chr((0xc0 | (ch >> 6)))) + result.append(chr((0x80 | (ch & 0x3f)))) + else: + # Encode UCS2 Unicode ordinals + if ch < 0x10000: + # Special case: check for high surrogate + if 0xD800 <= ch <= 0xDBFF and i != size: + ch2 = ord(s[i]) + # Check for low surrogate and combine the two to + # form a UCS4 value + if 0xDC00 <= ch2 <= 0xDFFF: + ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000 + i += 1 + _encodeUCS4(result, ch3) + continue + # Fall through: handles isolated high surrogates + result.append((chr((0xe0 | (ch >> 12))))) + result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) + result.append((chr((0x80 | (ch & 0x3f))))) + continue + else: + _encodeUCS4(result, ch) + return result.build() + +# ____________________________________________________________ +# utf-16 def str_decode_utf_16(s, size, errors, final=True, errorhandler=None): @@ -238,12 +284,11 @@ # mark is skipped, in all other modes, it is copied to the output # stream as-is (giving a ZWNBSP character). pos = 0 - result = [] if byteorder == 'native': - if (size >= 2): + if size >= 2: bom = (ord(s[ihi]) << 8) | ord(s[ilo]) if BYTEORDER == 'little': - if (bom == 0xFEFF): + if bom == 0xFEFF: pos += 2 bo = -1 elif bom == 0xFFFE: @@ -260,20 +305,22 @@ bo = -1 else: bo = 1 - if (size == 0): + if size == 0: return u'', 0, bo - if (bo == -1): + if bo == -1: # force little endian ihi = 1 ilo = 0 - elif (bo == 1): + elif bo == 1: # force big endian ihi = 0 ilo = 1 + result = UnicodeBuilder(size // 2) + #XXX I think the errors are not correctly handled here - while (pos < len(s)): + while pos < size: # remaining bytes at the end? (size should be even) if len(s) - pos < 2: if not final: @@ -285,7 +332,7 @@ break ch = (ord(s[pos + ihi]) << 8) | ord(s[pos + ilo]) pos += 2 - if (ch < 0xD800 or ch > 0xDFFF): + if ch < 0xD800 or ch > 0xDFFF: result.append(unichr(ch)) continue # UTF-16 code pair: @@ -297,10 +344,10 @@ result.append(r) if len(s) - pos < 2: break - elif (0xD800 <= ch and ch <= 0xDBFF): + elif 0xD800 <= ch <= 0xDBFF: ch2 = (ord(s[pos+ihi]) << 8) | ord(s[pos+ilo]) pos += 2 - if (0xDC00 <= ch2 and ch2 <= 0xDFFF): + if 0xDC00 <= ch2 <= 0xDFFF: if MAXUNICODE < 65536: result.append(unichr(ch)) result.append(unichr(ch2)) @@ -318,17 +365,305 @@ "illegal encoding", s, pos - 2, pos) result.append(r) - return u"".join(result), pos, bo + return result.build(), pos, bo + +def _STORECHAR(result, CH, byteorder): + hi = chr(((CH) >> 8) & 0xff) + lo = chr((CH) & 0xff) + if byteorder == 'little': + result.append(lo) + result.append(hi) + else: + result.append(hi) + result.append(lo) + +def unicode_encode_utf_16_helper(s, size, errors, + errorhandler=None, + byteorder='little'): + if size == 0: + return "" + + result = StringBuilder(size * 2 + 2) + if byteorder == 'native': + _STORECHAR(result, 0xFEFF, BYTEORDER) + byteorder = BYTEORDER + + i = 0 + while i < size: + ch = ord(s[i]) + i += 1 + ch2 = 0 + if ch >= 0x10000: + ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) + ch = 0xD800 | ((ch-0x10000) >> 10) + + _STORECHAR(result, ch, byteorder) + if ch2: + _STORECHAR(result, ch2, byteorder) + + return result.build() + +def unicode_encode_utf_16(s, size, errors, + errorhandler=None): + return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "native") + + +def unicode_encode_utf_16_be(s, size, errors, + errorhandler=None): + return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "big") + + +def unicode_encode_utf_16_le(s, size, errors, + errorhandler=None): + return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "little") + + +# ____________________________________________________________ +# utf-7 + +## indicate whether a UTF-7 character is special i.e. cannot be directly +## encoded: +## 0 - not special +## 1 - special +## 2 - whitespace (optional) +## 3 - RFC2152 Set O (optional) + +_utf7_special = [ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, + 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3, + 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1, +] + +def _utf7_SPECIAL(oc, encodeO=False, encodeWS=False): + return (oc > 127 or _utf7_special[oc] == 1 or + (encodeWS and _utf7_special[oc] == 2) or + (encodeO and _utf7_special[oc] == 3)) + +def _utf7_B64CHAR(oc): + if oc > 127: + return False + c = chr(oc) + return c.isalnum() or c == '+' or c == '/' +def _utf7_TO_BASE64(n): + "Returns the base-64 character of the bottom 6 bits of n" + return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[n & 0x3f] +def _utf7_FROM_BASE64(c): + "Retuns the base-64 value of a base-64 character" + if c == '+': + return 62 + elif c == '/': + return 63 + elif c >= 'a': + return ord(c) - 71 + elif c >= 'A': + return ord(c) - 65 + else: + return ord(c) + 4 + +def _utf7_ENCODE(result, ch, bits): + while bits >= 6: + result.append(_utf7_TO_BASE64(ch >> (bits - 6))) + bits -= 6 + return bits + +def _utf7_DECODE(s, result, errorhandler, errors, + pos, charsleft, bitsleft, surrogate): + while bitsleft >= 16: + outCh = (charsleft >> (bitsleft-16)) & 0xffff + bitsleft -= 16 + + if surrogate: + ## We have already generated an error for the high + ## surrogate so let's not bother seeing if the low + ## surrogate is correct or not + surrogate = False + elif 0xDC00 <= outCh <= 0xDFFF: + ## This is a surrogate pair. Unfortunately we can't + ## represent it in a 16-bit character + surrogate = True + msg = "code pairs are not supported" + res, pos = errorhandler(errors, 'utf-7', + msg, s, pos-1, pos) + result.append(res) + bitsleft = 0 + break + else: + result.append(unichr(outCh)) + return pos, charsleft, bitsleft, surrogate + + +def str_decode_utf_7(s, size, errors, final=False, + errorhandler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode + if size == 0: + return u'', 0 + + inShift = False + bitsleft = 0 + startinpos = 0 + charsleft = 0 + surrogate = False + + result = UnicodeBuilder(size) + pos = 0 + while pos < size: + ch = s[pos] + oc = ord(ch) + + if inShift: + if ch == '-' or not _utf7_B64CHAR(oc): + inShift = 0 + pos += 1 + + pos, charsleft, bitsleft, surrogate = _utf7_DECODE( + s, result, errorhandler, errors, + pos, charsleft, bitsleft, surrogate) + if bitsleft >= 6: + ## The shift sequence has a partial character in it. If + ## bitsleft < 6 then we could just classify it as padding + ## but that is not the case here + msg = "partial character in shift sequence" + res, pos = errorhandler(errors, 'utf-7', + msg, s, pos-1, pos) + result.append(res) + ## According to RFC2152 the remaining bits should be + ## zero. We choose to signal an error/insert a replacement + ## character here so indicate the potential of a + ## misencoded character. + if ch == '-': + if pos < size and s[pos] == '-': + result.append(u'-') + inShift = True + + elif _utf7_SPECIAL(oc): + msg = "unexpected special character" + res, pos = errorhandler(errors, 'utf-7', + msg, s, pos-1, pos) + result.append(res) + else: + result.append(unichr(ord(ch))) + else: + charsleft = (charsleft << 6) | _utf7_FROM_BASE64(ch) + bitsleft += 6 + pos += 1 + + pos, charsleft, bitsleft, surrogate = _utf7_DECODE( + s, result, errorhandler, errors, + pos, charsleft, bitsleft, surrogate) + elif ch == '+': + startinpos = pos + pos += 1 + if pos < size and s[pos] == '-': + pos += 1 + result.append(u'+') + else: + inShift = 1 + bitsleft = 0 + + elif _utf7_SPECIAL(oc): + pos += 1 + msg = "unexpected special character" + res, pos = errorhandler(errors, 'utf-7', msg, s, pos-1, pos) + result.append(res) + else: + result.append(unichr(oc)) + pos += 1 + + if inShift: + endinpos = size + msg = "unterminated shift sequence" + res, pos = errorhandler(errors, 'utf-7', msg, s, startinpos, pos) + result.append(res) + + return result.build(), pos + +def unicode_encode_utf_7(s, size, errors, errorhandler=None): + if size == 0: + return '' + result = StringBuilder(size) + + encodeSetO = encodeWhiteSpace = False + + inShift = False + bitsleft = 0 + charsleft = 0 + + pos = 0 + while pos < size: + ch = s[pos] + oc = ord(ch) + if not inShift: + if ch == u'+': + result.append('+-') + elif _utf7_SPECIAL(oc, encodeSetO, encodeWhiteSpace): + charsleft = oc + bitsleft = 16 + result.append('+') + bitsleft = _utf7_ENCODE(result, charsleft, bitsleft) + inShift = bitsleft > 0 + else: + result.append(chr(oc)) + else: + if not _utf7_SPECIAL(oc, encodeSetO, encodeWhiteSpace): + result.append(_utf7_TO_BASE64(charsleft << (6-bitsleft))) + charsleft = 0 + bitsleft = 0 + ## Characters not in the BASE64 set implicitly unshift the + ## sequence so no '-' is required, except if the character is + ## itself a '-' + if _utf7_B64CHAR(oc) or ch == u'-': + result.append('-') + inShift = False + result.append(chr(oc)) + else: + bitsleft += 16 + charsleft = (charsleft << 16) | oc + bitsleft = _utf7_ENCODE(result, charsleft, bitsleft) + ## If the next character is special then we dont' need to + ## terminate the shift sequence. If the next character is not + ## a BASE64 character or '-' then the shift sequence will be + ## terminated implicitly and we don't have to insert a '-'. + if bitsleft == 0: + if pos + 1 < size: + ch2 = s[pos + 1] + oc2 = ord(ch2) + + if _utf7_SPECIAL(oc2, encodeSetO, encodeWhiteSpace): + pass + elif _utf7_B64CHAR(oc2) or ch2 == u'-': + result.append('-') + inShift = False + else: + inShift = False + else: + result.append('-') + inShift = False + pos += 1 + + if bitsleft: + result.append(_utf7_TO_BASE64(charsleft << (6 - bitsleft))) + result.append('-') + + return result.build() + +# ____________________________________________________________ +# ascii and latin-1 def str_decode_latin_1(s, size, errors, final=False, errorhandler=None): # latin1 is equivalent to the first 256 ordinals in Unicode. pos = 0 - result = [] - while (pos < size): + result = UnicodeBuilder(size) + while pos < size: result.append(unichr(ord(s[pos]))) pos += 1 - return u"".join(result), pos + return result.build(), pos def str_decode_ascii(s, size, errors, final=False, @@ -336,9 +671,9 @@ if errorhandler is None: errorhandler = raise_unicode_exception_decode # ASCII is equivalent to the first 128 ordinals in Unicode. - result = [] + result = UnicodeBuilder(size) pos = 0 - while pos < len(s): + while pos < size: c = s[pos] if ord(c) < 128: result.append(unichr(ord(c))) @@ -347,55 +682,7 @@ r, pos = errorhandler(errors, "ascii", "ordinal not in range(128)", s, pos, pos + 1) result.append(r) - return u"".join(result), pos - - -# ____________________________________________________________ -# unicode encoding - - -def unicode_encode_utf_8(s, size, errors, errorhandler=None): - assert(size >= 0) - result = [] - i = 0 - while i < size: - ch = ord(s[i]) - i += 1 - if (ch < 0x80): - # Encode ASCII - result.append(chr(ch)) - elif (ch < 0x0800) : - # Encode Latin-1 - result.append(chr((0xc0 | (ch >> 6)))) - result.append(chr((0x80 | (ch & 0x3f)))) - else: - # Encode UCS2 Unicode ordinals - if (ch < 0x10000): - # Special case: check for high surrogate - if (0xD800 <= ch and ch <= 0xDBFF and i != size) : - ch2 = ord(s[i]) - # Check for low surrogate and combine the two to - # form a UCS4 value - if (0xDC00 <= ch2 and ch2 <= 0xDFFF) : - ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000 - i += 1 - _encodeUCS4(result, ch3) - continue - # Fall through: handles isolated high surrogates - result.append((chr((0xe0 | (ch >> 12))))) - result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) - result.append((chr((0x80 | (ch & 0x3f))))) - continue - else: - _encodeUCS4(result, ch) - return "".join(result) - -def _encodeUCS4(result, ch): - # Encode UCS4 Unicode ordinals - result.append((chr((0xf0 | (ch >> 18))))) - result.append((chr((0x80 | ((ch >> 12) & 0x3f))))) - result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) - result.append((chr((0x80 | (ch & 0x3f))))) + return result.build(), pos def unicode_encode_ucs1_helper(p, size, errors, @@ -408,12 +695,12 @@ else: reason = "ordinal not in range(128)" encoding = "ascii" - - if (size == 0): + + if size == 0: return '' - result = [] + result = StringBuilder(size) pos = 0 - while pos < len(p): + while pos < size: ch = p[pos] if ord(ch) < limit: @@ -427,9 +714,9 @@ collend += 1 r, pos = errorhandler(errors, encoding, reason, p, collstart, collend) - result += r # extend 'result' as a list of characters + result.append(r) - return "".join(result) + return result.build() def unicode_encode_latin_1(p, size, errors, errorhandler=None): res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 256) @@ -439,57 +726,479 @@ res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 128) return res +# ____________________________________________________________ +# Charmap -def _STORECHAR(result, CH, byteorder): - hi = chr(((CH) >> 8) & 0xff) - lo = chr((CH) & 0xff) - if byteorder == 'little': - result.append(lo) - result.append(hi) +ERROR_CHAR = u'\ufffe' + + at specialize.argtype(5) +def str_decode_charmap(s, size, errors, final=False, + errorhandler=None, mapping=None): + "mapping can be a rpython dictionary, or a dict-like object." + + # Default to Latin-1 + if mapping is None: + return str_decode_latin_1(s, size, errors, final=final, + errorhandler=errorhandler) + if errorhandler is None: + errorhandler = raise_unicode_exception_decode + if size == 0: + return u'', 0 + + pos = 0 + result = UnicodeBuilder(size) + while pos < size: + ch = s[pos] + + c = mapping.get(ch, ERROR_CHAR) + if c == ERROR_CHAR: + r, pos = errorhandler(errors, "charmap", + "character maps to ", + s, pos, pos + 1) + result.append(r) + continue + result.append(c) + pos += 1 + return result.build(), pos + +def unicode_encode_charmap(s, size, errors, errorhandler=None, + mapping=None): + if mapping is None: + return unicode_encode_latin_1(s, size, errors, + errorhandler=errorhandler) + + if errorhandler is None: + errorhandler = raise_unicode_exception_encode + + if size == 0: + return '' + result = StringBuilder(size) + pos = 0 + while pos < size: + ch = s[pos] + + c = mapping.get(ch, '') + if len(c) == 0: + res, pos = errorhandler(errors, "charmap", + "character maps to ", + s, pos, pos + 1) + for ch2 in res: + c2 = mapping.get(unichr(ord(ch2)), '') + if len(c2) == 0: + errorhandler( + "strict", "charmap", + "character maps to ", + s, pos, pos + 1) + result.append(c2) + continue + result.append(c) + pos += 1 + return result.build() + +# ____________________________________________________________ +# Unicode escape + +hexdigits = "0123456789ABCDEFabcdef" + +def hexescape(builder, s, pos, digits, + encoding, errorhandler, message, errors): + import sys + chr = 0 + if pos + digits > len(s): + message = "end of string in escape sequence" + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-2, len(s)) + builder.append(res) else: - result.append(hi) - result.append(lo) + try: + chr = r_uint(int(s[pos:pos+digits], 16)) + except ValueError: + endinpos = pos + while s[endinpos] in hexdigits: + endinpos += 1 + res, pos = errorhandler(errors, encoding, + message, s, pos-2, endinpos+1) + builder.append(res) + else: + # when we get here, chr is a 32-bit unicode character + if chr <= MAXUNICODE: + builder.append(unichr(chr)) + pos += digits + + elif chr <= 0x10ffff: + chr -= 0x10000L + builder.append(unichr(0xD800 + (chr >> 10))) + builder.append(unichr(0xDC00 + (chr & 0x03FF))) + pos += digits + else: + message = "illegal Unicode character" + res, pos = errorhandler(errors, encoding, + message, s, pos-2, pos+digits) + builder.append(res) + return pos + +def str_decode_unicode_escape(s, size, errors, final=False, + errorhandler=False, + unicodedata_handler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode -def unicode_encode_utf_16_helper(s, size, errors, - errorhandler=None, - byteorder='little'): - result = [] - if (byteorder == 'native'): - _STORECHAR(result, 0xFEFF, BYTEORDER) - byteorder = BYTEORDER - if size == 0: - return "" + return u'', 0 - i = 0 - while i < size: - ch = ord(s[i]) - i += 1 - ch2 = 0 - if (ch >= 0x10000) : - ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) - ch = 0xD800 | ((ch-0x10000) >> 10) + builder = UnicodeBuilder(size) + pos = 0 + while pos < size: + ch = s[pos] - _STORECHAR(result, ch, byteorder) - if ch2: - _STORECHAR(result, ch2, byteorder) + # Non-escape characters are interpreted as Unicode ordinals + if ch != '\\': + builder.append(unichr(ord(ch))) + pos += 1 + continue - return "".join(result) + # - Escapes + pos += 1 + if pos >= size: + message = "\\ at end of string" + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, size) + builder.append(res) + continue -def unicode_encode_utf_16(s, size, errors, - errorhandler=None): - return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "native") + ch = s[pos] + pos += 1 + # \x escapes + if ch == '\n': pass + elif ch == '\\': builder.append(u'\\') + elif ch == '\'': builder.append(u'\'') + elif ch == '\"': builder.append(u'\"') + elif ch == 'b' : builder.append(u'\b') + elif ch == 'f' : builder.append(u'\f') + elif ch == 't' : builder.append(u'\t') + elif ch == 'n' : builder.append(u'\n') + elif ch == 'r' : builder.append(u'\r') + elif ch == 'v' : builder.append(u'\v') + elif ch == 'a' : builder.append(u'\a') + elif '0' <= ch <= '7': + x = ord(ch) - ord('0') + if pos < size: + ch = s[pos] + if '0' <= ch <= '7': + pos += 1 + x = (x<<3) + ord(ch) - ord('0') + if pos < size: + ch = s[pos] + if '0' <= ch <= '7': + pos += 1 + x = (x<<3) + ord(ch) - ord('0') + builder.append(unichr(x)) + # hex escapes + # \xXX + elif ch == 'x': + digits = 2 + message = "truncated \\xXX escape" + pos = hexescape(builder, s, pos, digits, + "unicodeescape", errorhandler, message, errors) + + # \uXXXX + elif ch == 'u': + digits = 4 + message = "truncated \\uXXXX escape" + pos = hexescape(builder, s, pos, digits, + "unicodeescape", errorhandler, message, errors) + + # \UXXXXXXXX + elif ch == 'U': + digits = 8 + message = "truncated \\UXXXXXXXX escape" + pos = hexescape(builder, s, pos, digits, + "unicodeescape", errorhandler, message, errors) + + # \N{name} + elif ch == 'N': + message = "malformed \\N character escape" + look = pos + if unicodedata_handler is None: + message = ("\\N escapes not supported " + "(can't load unicodedata module)") + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, size) + builder.append(res) + continue + + if look < size and s[look] == '{': + # look for the closing brace + while look < size and s[look] != '}': + look += 1 + if look < size and s[look] == '}': + # found a name. look it up in the unicode database + message = "unknown Unicode character name" + name = s[pos+1:look] + code = unicodedata_handler.call(name) + if code < 0: + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, look+1) + builder.append(res) + continue + pos = look + 1 + if code <= MAXUNICODE: + builder.append(unichr(code)) + else: + code -= 0x10000L + builder.append(unichr(0xD800 + (code >> 10))) + builder.append(unichr(0xDC00 + (code & 0x03FF))) + else: + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, look+1) + builder.append(res) + else: + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, look+1) + builder.append(res) + else: + builder.append(u'\\') + builder.append(unichr(ord(ch))) + return builder.build(), pos -def unicode_encode_utf_16_be(s, size, errors, - errorhandler=None): - return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "big") +def unicode_encode_unicode_escape(s, size, errors, errorhandler=None, quotes=False): + # errorhandler is not used: this function cannot cause Unicode errors + result = StringBuilder(size) + + if quotes: + if s.find(u'\'') != -1 and s.find(u'\"') == -1: + quote = ord('\"') + result.append('u"') + else: + quote = ord('\'') + result.append('u\'') + else: + quote = 0 + if size == 0: + return '' -def unicode_encode_utf_16_le(s, size, errors, - errorhandler=None): - return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "little") + pos = 0 + while pos < size: + ch = s[pos] + oc = ord(ch) + + # Escape quotes + if quotes and (oc == quote or ch == '\\'): + result.append('\\') + result.append(chr(oc)) + pos += 1 + continue + + if 0xD800 <= oc < 0xDC00 and pos + 1 < size: + # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes + pos += 1 + oc2 = ord(s[pos]) + + if 0xDC00 <= oc2 <= 0xDFFF: + ucs = (((oc & 0x03FF) << 10) | (oc2 & 0x03FF)) + 0x00010000 + raw_unicode_escape_helper(result, ucs) + pos += 1 + continue + # Fall through: isolated surrogates are copied as-is + pos -= 1 + + # Map special whitespace to '\t', \n', '\r' + if ch == '\t': + result.append('\\t') + elif ch == '\n': + result.append('\\n') + elif ch == '\r': + result.append('\\r') + elif ch == '\\': + result.append('\\\\') + + # Map non-printable or non-ascii to '\xhh' or '\uhhhh' + elif oc < 32 or oc >= 0x7F: + raw_unicode_escape_helper(result, oc) + + # Copy everything else as-is + else: + result.append(chr(oc)) + pos += 1 + + if quotes: + result.append(chr(quote)) + return result.build() + +# ____________________________________________________________ +# Raw unicode escape + +def str_decode_raw_unicode_escape(s, size, errors, final=False, + errorhandler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode + if size == 0: + return u'', 0 + + result = UnicodeBuilder(size) + pos = 0 + while pos < size: + ch = s[pos] + + # Non-escape characters are interpreted as Unicode ordinals + if ch != '\\': + result.append(unichr(ord(ch))) + pos += 1 + continue + + startinpos = pos + # \u-escapes are only interpreted iff the number of leading + # backslashes is odd + bs = pos + while pos < size: + pos += 1 + if pos == size or s[pos] != '\\': + break + result.append(u'\\') + + # we have a backslash at the end of the string, stop here + if pos >= size: + result.append(u'\\') + break + + if ((pos - bs) & 1 == 0 or + pos >= size or + (s[pos] != 'u' and s[pos] != 'U')): + result.append(u'\\') + result.append(unichr(ord(s[pos]))) + pos += 1 + continue + + if s[pos] == 'u': + digits = 4 + message = "truncated \\uXXXX escape" + else: + digits = 8 + message = "truncated \\UXXXXXXXX escape" + pos += 1 + pos = hexescape(result, s, pos, digits, + "rawunicodeescape", errorhandler, message, errors) + + return result.build(), pos + +def raw_unicode_escape_helper(result, char): + num = hex(char) + if char >= 0x10000: + result.append("\\U") + zeros = 8 + elif char >= 0x100: + result.append("\\u") + zeros = 4 + else: + result.append("\\x") + zeros = 2 + lnum = len(num) + nb = zeros + 2 - lnum # num starts with '0x' + if nb > 0: + result.append_multiple_char('0', nb) + result.append_slice(num, 2, lnum) + +def unicode_encode_raw_unicode_escape(s, size, errors, errorhandler=None): + # errorhandler is not used: this function cannot cause Unicode errors + if size == 0: + return '' + result = StringBuilder(size) + pos = 0 + while pos < size: + oc = ord(s[pos]) + if oc < 0x100: + result.append(chr(oc)) + else: + raw_unicode_escape_helper(result, oc) + pos += 1 + + return result.build() + +# ____________________________________________________________ +# unicode-internal + +def str_decode_unicode_internal(s, size, errors, final=False, + errorhandler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode + if size == 0: + return u'', 0 + + if MAXUNICODE < 65536: + unicode_bytes = 2 + else: + unicode_bytes = 4 + if BYTEORDER == "little": + start = 0 + stop = unicode_bytes + step = 1 + else: + start = unicode_bytes - 1 + stop = -1 + step = -1 + + result = UnicodeBuilder(size // unicode_bytes) + pos = 0 + while pos < size: + if pos > size - unicode_bytes: + res, pos = errorhandler(errors, "unicode_internal", + "truncated input", + s, pos, size) + result.append(res) + if pos > size - unicode_bytes: + break + continue + t = r_uint(0) + h = 0 + for j in range(start, stop, step): + t += r_uint(ord(s[pos + j])) << (h*8) + h += 1 + if t > MAXUNICODE: + res, pos = errorhandler(errors, "unicode_internal", + "unichr(%d) not in range" % (t,), + s, pos, pos + unicode_bytes) + result.append(res) + continue + result.append(unichr(t)) + pos += unicode_bytes + return result.build(), pos + +def unicode_encode_unicode_internal(s, size, errors, errorhandler=None): + if size == 0: + return '' + + if MAXUNICODE < 65536: + unicode_bytes = 2 + else: + unicode_bytes = 4 + + result = StringBuilder(size * unicode_bytes) + pos = 0 + while pos < size: + oc = ord(s[pos]) + if MAXUNICODE < 65536: + if BYTEORDER == "little": + result.append(chr(oc & 0xFF)) + result.append(chr(oc >> 8 & 0xFF)) + else: + result.append(chr(oc >> 8 & 0xFF)) + result.append(chr(oc & 0xFF)) + else: + if BYTEORDER == "little": + result.append(chr(oc & 0xFF)) + result.append(chr(oc >> 8 & 0xFF)) + result.append(chr(oc >> 16 & 0xFF)) + result.append(chr(oc >> 24 & 0xFF)) + else: + result.append(chr(oc >> 24 & 0xFF)) + result.append(chr(oc >> 16 & 0xFF)) + result.append(chr(oc >> 8 & 0xFF)) + result.append(chr(oc & 0xFF)) + pos += 1 + return result.build() # ____________________________________________________________ # MBCS codecs for Windows From afa at codespeak.net Thu Jul 1 22:25:15 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 22:25:15 +0200 (CEST) Subject: [pypy-svn] r75738 - pypy/branch/interplevel-codecs Message-ID: <20100701202515.DEB00282BE0@codespeak.net> Author: afa Date: Thu Jul 1 22:25:14 2010 New Revision: 75738 Removed: pypy/branch/interplevel-codecs/ Log: Remove merged branch From getxsick at codespeak.net Thu Jul 1 22:58:28 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 1 Jul 2010 22:58:28 +0200 (CEST) Subject: [pypy-svn] r75739 - pypy/build/ubuntu/trunk/debian Message-ID: <20100701205828.6F6A6282B9E@codespeak.net> Author: getxsick Date: Thu Jul 1 22:58:27 2010 New Revision: 75739 Removed: pypy/build/ubuntu/trunk/debian/pypy-lib.links Modified: pypy/build/ubuntu/trunk/debian/pypy-lib.install pypy/build/ubuntu/trunk/debian/rules Log: update scripts to the current version of trunk structure of the repository has changed since sys-prefix branch was merged (r75556) Modified: pypy/build/ubuntu/trunk/debian/pypy-lib.install ============================================================================== --- pypy/build/ubuntu/trunk/debian/pypy-lib.install (original) +++ pypy/build/ubuntu/trunk/debian/pypy-lib.install Thu Jul 1 22:58:27 2010 @@ -1,2 +1,3 @@ -lib-python usr/share/pypy-1.3 -pypy/lib usr/share/pypy-1.3/pypy +lib-python usr/share/pypy-1.3 +lib_pypy usr/share/pypy-1.3 +site-packages usr/share/pypy-1.3 Modified: pypy/build/ubuntu/trunk/debian/rules ============================================================================== --- pypy/build/ubuntu/trunk/debian/rules (original) +++ pypy/build/ubuntu/trunk/debian/rules Thu Jul 1 22:58:27 2010 @@ -53,8 +53,7 @@ find debian/pypy-lib -name "*.pyc" | xargs rm -rf find debian/pypy-dev -name "*.pyc" | xargs rm -rf rm debian/pypy-dev/usr/share/pypy-$(VERSION)/pypy/module/unicodedata/LICENSE - rm debian/pypy-lib/usr/share/pypy-$(VERSION)/pypy/lib/py - rm debian/pypy-lib/usr/share/pypy-$(VERSION)/pypy/lib/app_test/ctypes_tests/_ctypes_test.o + rm debian/pypy-lib/usr/share/pypy-$(VERSION)/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.o chmod a-x debian/pypy-lib/usr/share/pypy-$(VERSION)/lib-python/2.5.2/idlelib/idle.bat install-arch: From getxsick at codespeak.net Thu Jul 1 23:15:02 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 1 Jul 2010 23:15:02 +0200 (CEST) Subject: [pypy-svn] r75740 - in pypy/trunk: . pypy/module/unicodedata Message-ID: <20100701211502.9512F282B9E@codespeak.net> Author: getxsick Date: Thu Jul 1 23:15:01 2010 New Revision: 75740 Removed: pypy/trunk/pypy/module/unicodedata/LICENSE Modified: pypy/trunk/LICENSE Log: kill the local LICENCE file and move the content to a global LICENSE file Modified: pypy/trunk/LICENSE ============================================================================== --- pypy/trunk/LICENSE (original) +++ pypy/trunk/LICENSE Thu Jul 1 23:15:01 2010 @@ -152,3 +152,26 @@ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License for 'pypy/module/unicodedata/' +====================================== + +The following files are from the website of The Unicode Consortium +at http://www.unicode.org/. For the terms of use of these files, see +http://www.unicode.org/terms_of_use.html + + CompositionExclusions-3.2.0.txt + CompositionExclusions-4.1.0.txt + CompositionExclusions-5.0.0.txt + EastAsianWidth-3.2.0.txt + EastAsianWidth-4.1.0.txt + EastAsianWidth-5.0.0.txt + UnicodeData-3.2.0.txt + UnicodeData-4.1.0.txt + UnicodeData-5.0.0.txt + +The following files are derived from files from the above website. The same +terms of use apply. + UnihanNumeric-3.2.0.txt + UnihanNumeric-4.1.0.txt + UnihanNumeric-5.0.0.txt From getxsick at codespeak.net Thu Jul 1 23:16:57 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 1 Jul 2010 23:16:57 +0200 (CEST) Subject: [pypy-svn] r75741 - pypy/build/ubuntu/trunk/debian Message-ID: <20100701211657.A7497282B9E@codespeak.net> Author: getxsick Date: Thu Jul 1 23:16:56 2010 New Revision: 75741 Modified: pypy/build/ubuntu/trunk/debian/copyright pypy/build/ubuntu/trunk/debian/rules Log: this file is already removed from the trunk Modified: pypy/build/ubuntu/trunk/debian/copyright ============================================================================== --- pypy/build/ubuntu/trunk/debian/copyright (original) +++ pypy/build/ubuntu/trunk/debian/copyright Thu Jul 1 23:16:56 2010 @@ -237,7 +237,7 @@ TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -License for 'pypy/module/unicodedata' +License for 'pypy/module/unicodedata/' ===================================== The following files are from the website of The Unicode Consortium Modified: pypy/build/ubuntu/trunk/debian/rules ============================================================================== --- pypy/build/ubuntu/trunk/debian/rules (original) +++ pypy/build/ubuntu/trunk/debian/rules Thu Jul 1 23:16:56 2010 @@ -52,7 +52,6 @@ dh_install -i find debian/pypy-lib -name "*.pyc" | xargs rm -rf find debian/pypy-dev -name "*.pyc" | xargs rm -rf - rm debian/pypy-dev/usr/share/pypy-$(VERSION)/pypy/module/unicodedata/LICENSE rm debian/pypy-lib/usr/share/pypy-$(VERSION)/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.o chmod a-x debian/pypy-lib/usr/share/pypy-$(VERSION)/lib-python/2.5.2/idlelib/idle.bat From afa at codespeak.net Thu Jul 1 23:19:39 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 23:19:39 +0200 (CEST) Subject: [pypy-svn] r75742 - in pypy/trunk/pypy: interpreter/pyparser interpreter/pyparser/test module/_codecs/test Message-ID: <20100701211939.CB24E282B9E@codespeak.net> Author: afa Date: Thu Jul 1 23:19:38 2010 New Revision: 75742 Modified: pypy/trunk/pypy/interpreter/pyparser/parsestring.py pypy/trunk/pypy/interpreter/pyparser/test/test_parsestring.py pypy/trunk/pypy/module/_codecs/test/test_codecs.py Log: Fixed issue531: CPython oddities when parsing octal escapes in string literals '\9' == ('\\' + '9'), '\400' == chr(0) Noticed by santagada. Modified: pypy/trunk/pypy/interpreter/pyparser/parsestring.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/parsestring.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/parsestring.py Thu Jul 1 23:19:38 2010 @@ -160,11 +160,17 @@ span = ps span += (span < end) and (s[span] in '01234567') span += (span < end) and (s[span] in '01234567') - lis.append(chr(int(s[prevps : span], 8))) + octal = s[prevps : span] + # emulate a strange wrap-around behavior of CPython: + # \400 is the same as \000 because 0400 == 256 + num = int(octal, 8) & 0xFF + lis.append(chr(num)) ps = span elif ch == 'x': if ps+2 <= end and isxdigit(s[ps]) and isxdigit(s[ps + 1]): - lis.append(chr(int(s[ps : ps + 2], 16))) + hexa = s[ps : ps + 2] + num = int(hexa, 16) + lis.append(chr(num)) ps += 2 else: raise_app_valueerror(space, 'invalid \\x escape') Modified: pypy/trunk/pypy/interpreter/pyparser/test/test_parsestring.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/test/test_parsestring.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/test/test_parsestring.py Thu Jul 1 23:19:38 2010 @@ -22,6 +22,15 @@ s = r'"\123"' w_ret = parsestring.parsestr(space, None, s) assert space.str_w(w_ret) == chr(0123) + s = r'"\400"' + w_ret = parsestring.parsestr(space, None, s) + assert space.str_w(w_ret) == chr(0) + s = r'"\9"' + w_ret = parsestring.parsestr(space, None, s) + assert space.str_w(w_ret) == '\\9' + s = r'"\08"' + w_ret = parsestring.parsestr(space, None, s) + assert space.str_w(w_ret) == chr(0) + '8' s = r'"\x"' space.raises_w(space.w_ValueError, parsestring.parsestr, space, None, s) s = r'"\x7"' Modified: pypy/trunk/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/trunk/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/trunk/pypy/module/_codecs/test/test_codecs.py Thu Jul 1 23:19:38 2010 @@ -271,7 +271,6 @@ assert u"\u0663".encode("raw-unicode-escape") == "\u0663" def test_escape_decode(self): - test = 'a\n\\b\x00c\td\u2045'.encode('string_escape') assert test.decode('string_escape') =='a\n\\b\x00c\td\u2045' assert '\\077'.decode('string_escape') == '?' @@ -279,6 +278,14 @@ assert '\\253'.decode('string_escape') == chr(0253) assert '\\312'.decode('string_escape') == chr(0312) + def test_escape_decode_wrap_around(self): + assert '\\400'.decode('string_escape') == chr(0) + + def test_escape_decode_ignore_invalid(self): + assert '\\9'.decode('string_escape') == '\\9' + assert '\\01'.decode('string_escape') == chr(01) + assert '\\0f'.decode('string_escape') == chr(0) + 'f' + assert '\\08'.decode('string_escape') == chr(0) + '8' def test_decode_utf8_different_case(self): constant = u"a" From afa at codespeak.net Thu Jul 1 23:40:53 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 23:40:53 +0200 (CEST) Subject: [pypy-svn] r75743 - pypy/trunk/pypy/interpreter/pyparser/test Message-ID: <20100701214053.19AE9282B9E@codespeak.net> Author: afa Date: Thu Jul 1 23:40:51 2010 New Revision: 75743 Modified: pypy/trunk/pypy/interpreter/pyparser/test/test_parsestring.py Log: Factor common code in tests Modified: pypy/trunk/pypy/interpreter/pyparser/test/test_parsestring.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/test/test_parsestring.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/test/test_parsestring.py Thu Jul 1 23:40:51 2010 @@ -2,81 +2,56 @@ import py class TestParsetring: + def parse_and_compare(self, literal, value): + space = self.space + w_ret = parsestring.parsestr(space, None, literal) + if isinstance(value, str): + assert space.type(w_ret) == space.w_str + assert space.str_w(w_ret) == value + elif isinstance(value, unicode): + assert space.type(w_ret) == space.w_unicode + assert space.unicode_w(w_ret) == value + else: + assert False + def test_simple(self): space = self.space - s = 'hello world' - w_ret = parsestring.parsestr(space, None, repr(s)) - assert space.str_w(w_ret) == s - s = 'hello\n world' - w_ret = parsestring.parsestr(space, None, repr(s)) - assert space.str_w(w_ret) == s - s = "'''hello\\x42 world'''" - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == 'hello\x42 world' - s = r'"\0"' - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == chr(0) - s = r'"\07"' - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == chr(7) - s = r'"\123"' - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == chr(0123) - s = r'"\400"' - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == chr(0) - s = r'"\9"' - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == '\\9' - s = r'"\08"' - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == chr(0) + '8' - s = r'"\x"' - space.raises_w(space.w_ValueError, parsestring.parsestr, space, None, s) - s = r'"\x7"' - space.raises_w(space.w_ValueError, parsestring.parsestr, space, None, s) - s = r'"\x7g"' - space.raises_w(space.w_ValueError, parsestring.parsestr, space, None, s) - s = r'"\xfF"' - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == chr(0xFF) + for s in ['hello world', 'hello\n world']: + self.parse_and_compare(repr(s), s) + + self.parse_and_compare("'''hello\\x42 world'''", 'hello\x42 world') + + # octal + self.parse_and_compare(r'"\0"', chr(0)) + self.parse_and_compare(r'"\07"', chr(7)) + self.parse_and_compare(r'"\123"', chr(0123)) + self.parse_and_compare(r'"\400"', chr(0)) + self.parse_and_compare(r'"\9"', '\\' + '9') + self.parse_and_compare(r'"\08"', chr(0) + '8') + + # hexadecimal + self.parse_and_compare(r'"\xfF"', chr(0xFF)) + self.parse_and_compare(r'"\""', '"') + self.parse_and_compare(r"'\''", "'") + for s in (r'"\x"', r'"\x7"', r'"\x7g"'): + space.raises_w(space.w_ValueError, + parsestring.parsestr, space, None, s) - s = r'"\""' - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == '"' - - s = r"'\''" - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == "'" - - def test_unicode(self): space = self.space - s = u'hello world' - w_ret = parsestring.parsestr(space, None, repr(s)) - ret = space.unwrap(w_ret) - assert isinstance(ret, unicode) - assert ret == s - s = u'hello\n world' - w_ret = parsestring.parsestr(self.space, None, repr(s)) - ret = space.unwrap(w_ret) - assert isinstance(ret, unicode) - assert ret == s - s = "u'''hello\\x42 world'''" - w_ret = parsestring.parsestr(self.space, None, s) - ret = space.unwrap(w_ret) - assert isinstance(ret, unicode) - assert ret == u'hello\x42 world' - s = "u'''hello\\u0842 world'''" - w_ret = parsestring.parsestr(self.space, None, s) - ret = space.unwrap(w_ret) - assert isinstance(ret, unicode) - assert ret == u'hello\u0842 world' + for s in [u'hello world', u'hello\n world']: + self.parse_and_compare(repr(s), s) + + self.parse_and_compare("u'''hello\\x42 world'''", + u'hello\x42 world') + self.parse_and_compare("u'''hello\\u0842 world'''", + u'hello\u0842 world') + s = "u'\x81'" s = s.decode("koi8-u").encode("utf8") w_ret = parsestring.parsestr(self.space, 'koi8-u', s) ret = space.unwrap(w_ret) - assert ret == eval("# -*- coding: koi8-u -*-\nu'\x81'") + assert ret == eval("# -*- coding: koi8-u -*-\nu'\x81'") def test_simple_enc_roundtrip(self): #py.test.skip("crashes in app_codecs, but when cheating using .encode at interp-level passes?!") From afa at codespeak.net Thu Jul 1 23:41:50 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Jul 2010 23:41:50 +0200 (CEST) Subject: [pypy-svn] r75744 - pypy/trunk/pypy/interpreter/pyparser/test Message-ID: <20100701214150.B63F0282B9E@codespeak.net> Author: afa Date: Thu Jul 1 23:41:49 2010 New Revision: 75744 Modified: pypy/trunk/pypy/interpreter/pyparser/test/test_parsestring.py Log: Remove outdated comment Modified: pypy/trunk/pypy/interpreter/pyparser/test/test_parsestring.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/test/test_parsestring.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/test/test_parsestring.py Thu Jul 1 23:41:49 2010 @@ -54,7 +54,6 @@ assert ret == eval("# -*- coding: koi8-u -*-\nu'\x81'") def test_simple_enc_roundtrip(self): - #py.test.skip("crashes in app_codecs, but when cheating using .encode at interp-level passes?!") space = self.space s = "'\x81'" s = s.decode("koi8-u").encode("utf8") From benjamin at codespeak.net Thu Jul 1 23:44:53 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 23:44:53 +0200 (CEST) Subject: [pypy-svn] r75745 - in pypy/branch/fast-forward/pypy: annotation translator translator/test Message-ID: <20100701214453.1B12F282B9E@codespeak.net> Author: benjamin Date: Thu Jul 1 23:44:51 2010 New Revision: 75745 Modified: pypy/branch/fast-forward/pypy/annotation/builtin.py pypy/branch/fast-forward/pypy/annotation/unaryop.py pypy/branch/fast-forward/pypy/translator/simplify.py pypy/branch/fast-forward/pypy/translator/test/test_simplify.py Log: simply rewrite the isinstance operation into a builtin call Modified: pypy/branch/fast-forward/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/builtin.py (original) +++ pypy/branch/fast-forward/pypy/annotation/builtin.py Thu Jul 1 23:44:51 2010 @@ -175,7 +175,17 @@ # from bool to int, notice that isinstance( , bool|int) # is quite border case for RPython r.const = False + # XXX HACK HACK HACK + # XXX HACK HACK HACK + # XXX HACK HACK HACK bk = getbookkeeper() + if variables is None: + fn, block, i = bk.position_key + op = block.operations[i] + assert op.opname == "simple_call" + assert len(op.args) == 3 + assert op.args[0] == Constant(isinstance) + variables = [op.args[1]] for variable in variables: assert bk.annotator.binding(variable) == s_obj r.knowntypedata = {} Modified: pypy/branch/fast-forward/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/unaryop.py (original) +++ pypy/branch/fast-forward/pypy/annotation/unaryop.py Thu Jul 1 23:44:51 2010 @@ -23,7 +23,6 @@ UNARY_OPERATIONS = set(['len', 'is_true', 'getattr', 'setattr', 'delattr', 'simple_call', 'call_args', 'str', 'repr', 'iter', 'next', 'invert', 'type', 'issubtype', - 'isinstance', 'pos', 'neg', 'nonzero', 'abs', 'hex', 'oct', 'ord', 'int', 'float', 'long', 'hash', 'id', # <== not supported any more @@ -67,13 +66,6 @@ return immutablevalue(issubclass(obj.const, s_cls.const)) return s_Bool - def isinstance(obj, s_cls): - bk = getbookkeeper() - fn, block, i = bk.position_key - op = block.operations[i] - v = op.args[0] - return builtin.builtin_isinstance(obj, s_cls, [v]) - def len(obj): return SomeInteger(nonneg=True) Modified: pypy/branch/fast-forward/pypy/translator/simplify.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/simplify.py (original) +++ pypy/branch/fast-forward/pypy/translator/simplify.py Thu Jul 1 23:44:51 2010 @@ -73,6 +73,21 @@ # ____________________________________________________________ +def desugar_isinstance(graph): + """Replace isinstance operation with a call to isinstance.""" + constant_isinstance = Constant(isinstance) + def visit(block): + if not isinstance(block, Block): + return + for i in range(len(block.operations) - 1, -1, -1): + op = block.operations[i] + insert = [] + if op.opname == "isinstance": + args = [constant_isinstance, op.args[0], op.args[1]] + new_op = SpaceOperation("simple_call", args, op.result) + block.operations[i] = new_op + traverse(visit, graph) + def eliminate_empty_blocks(graph): """Eliminate basic blocks that do not contain any operations. When this happens, we need to replace the preceeding link with the @@ -985,6 +1000,7 @@ # ____ all passes & simplify_graph all_passes = [ + desugar_isinstance, eliminate_empty_blocks, remove_assertion_errors, join_blocks, Modified: pypy/branch/fast-forward/pypy/translator/test/test_simplify.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/test/test_simplify.py (original) +++ pypy/branch/fast-forward/pypy/translator/test/test_simplify.py Thu Jul 1 23:44:51 2010 @@ -1,8 +1,9 @@ import py from pypy.translator.translator import TranslationContext, graphof from pypy.translator.backendopt.all import backend_optimizations -from pypy.translator.simplify import get_graph, transform_dead_op_vars -from pypy.objspace.flow.model import traverse, Block, summary +from pypy.translator.simplify import (get_graph, transform_dead_op_vars, + desugar_isinstance) +from pypy.objspace.flow.model import traverse, Block, Constant, summary from pypy import conftest def translate(func, argtypes, backend_optimize=True): @@ -280,6 +281,20 @@ e = py.test.raises(LLException, 'interp.eval_graph(graph, [])') assert 'ValueError' in str(e.value) +def test_desugar_isinstance(): + class X(object): + pass + def f(): + x = X() + return isinstance(x, X()) + graph = TranslationContext().buildflowgraph(f) + desugar_isinstance(graph) + assert len(graph.startblock.operations) == 3 + block = graph.startblock + assert block.operations[2].opname == "simple_call" + assert isinstance(block.operations[2].args[0], Constant) + assert block.operations[2].args[0].value is isinstance + class TestDetectListComprehension: def check(self, f1, expected): t = TranslationContext(list_comprehension_operations=True) From benjamin at codespeak.net Thu Jul 1 23:59:29 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Jul 2010 23:59:29 +0200 (CEST) Subject: [pypy-svn] r75746 - in pypy/branch/fast-forward/pypy: objspace/std rlib/rstruct rlib/rstruct/test rlib/test Message-ID: <20100701215929.5B856282B9E@codespeak.net> Author: benjamin Date: Thu Jul 1 23:59:27 2010 New Revision: 75746 Added: pypy/branch/fast-forward/pypy/rlib/rstruct/test/ pypy/branch/fast-forward/pypy/rlib/rstruct/test/test_ieee.py (contents, props changed) Removed: pypy/branch/fast-forward/pypy/rlib/test/test_ieee.py Modified: pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Log: correct code for packing and unpacking floats This is courtesy Mark Dickinson, to whom we now owe much. Modified: pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py Thu Jul 1 23:59:27 2010 @@ -169,11 +169,11 @@ def pack_float(f): result = [] - ieee.pack_float(result, f, 8, False) + ieee.pack_float8(result, f) return ''.join(result) def unpack_float(s): - return ieee.unpack_float(s, False) + return ieee.unpack_float8(s) def marshal_w__Float(space, w_float, m): if m.version > 1: Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Thu Jul 1 23:59:27 2010 @@ -3,107 +3,135 @@ """ import math -from pypy.rlib.rarithmetic import r_longlong, isinf, isnan, INFINITY, NAN -def pack_float(result, number, size, bigendian): - """Append to 'result' the 'size' characters of the 32-bit or 64-bit - IEEE representation of the number. +from pypy.rlib import rarithmetic + + +def round_to_nearest(x): + """Python 3 style round: round a float x to the nearest int, but + unlike the builtin Python 2.x round function: + + - return an int, not a float + - do round-half-to-even, not round-half-away-from-zero. + + We assume that x is finite and nonnegative; except wrong results + if you use this for negative x. + """ - if size == 4: - bias = 127 - exp = 8 - prec = 23 - else: - bias = 1023 - exp = 11 - prec = 52 - - if isnan(number): - sign = 0x80 - man, e = 1.5, bias + 1 + int_part = int(x) + frac_part = x - int_part + if frac_part > 0.5 or frac_part == 0.5 and int_part & 1 == 1: + int_part += 1 + return int_part + + +def float_unpack(Q, size): + """Convert a 32-bit or 64-bit integer created + by float_pack into a Python float.""" + + if size == 8: + MIN_EXP = -1021 # = sys.float_info.min_exp + MAX_EXP = 1024 # = sys.float_info.max_exp + MANT_DIG = 53 # = sys.float_info.mant_dig + BITS = 64 + elif size == 4: + MIN_EXP = -125 # C's FLT_MIN_EXP + MAX_EXP = 128 # FLT_MAX_EXP + MANT_DIG = 24 # FLT_MANT_DIG + BITS = 32 else: - if number < 0: - sign = 0x80 - number *= -1 - elif number == 0.0: - for i in range(size): - result.append('\x00') - return - else: - sign = 0x00 - if isinf(number): - man, e = 1.0, bias + 1 - else: - man, e = math.frexp(number) + raise ValueError("invalid size value") - if 0.5 <= man and man < 1.0: - man *= 2 - e -= 1 - man -= 1 - e += bias - power_of_two = r_longlong(1) << prec - mantissa = r_longlong(power_of_two * man + 0.5) - if mantissa >> prec : - mantissa = 0 - e += 1 - - for i in range(size-2): - result.append(chr(mantissa & 0xff)) - mantissa >>= 8 - x = (mantissa & ((1<<(15-exp))-1)) | ((e & ((1<<(exp-7))-1))<<(15-exp)) - result.append(chr(x)) - x = sign | e >> (exp - 7) - result.append(chr(x)) - if bigendian: - first = len(result) - size - last = len(result) - 1 - for i in range(size // 2): - (result[first + i], result[last - i]) = ( - result[last - i], result[first + i]) - -def unpack_float(input, bigendian): - """Interpret the 'input' string into a 32-bit or 64-bit - IEEE representation a the number. - """ - size = len(input) - bytes = [] - if bigendian: - reverse_mask = size - 1 + if Q >> BITS: + raise ValueError("input out of range") + + # extract pieces + sign = Q >> BITS - 1 + exp = (Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1 + mant = Q & ((1 << MANT_DIG - 1) - 1) + + if exp == MAX_EXP - MIN_EXP + 2: + # nan or infinity + result = float('nan') if mant else float('inf') + elif exp == 0: + # subnormal or zero + result = math.ldexp(float(mant), MIN_EXP - MANT_DIG) else: - reverse_mask = 0 - nonzero = False - for i in range(size): - x = ord(input[i ^ reverse_mask]) - bytes.append(x) - nonzero |= x - if not nonzero: - return 0.0 - if size == 4: - bias = 127 - exp = 8 - prec = 23 + # normal + mant += 1 << MANT_DIG - 1 + result = math.ldexp(float(mant), exp + MIN_EXP - MANT_DIG - 1) + return -result if sign else result + + +def float_pack(x, size): + """Convert a Python float x into a 64-bit unsigned integer + with the same byte representation.""" + + if size == 8: + MIN_EXP = -1021 # = sys.float_info.min_exp + MAX_EXP = 1024 # = sys.float_info.max_exp + MANT_DIG = 53 # = sys.float_info.mant_dig + BITS = 64 + elif size == 4: + MIN_EXP = -125 # C's FLT_MIN_EXP + MAX_EXP = 128 # FLT_MAX_EXP + MANT_DIG = 24 # FLT_MANT_DIG + BITS = 32 else: - bias = 1023 - exp = 11 - prec = 52 - mantissa_scale_factor = 0.5 ** prec # this is constant-folded if it's - # right after the 'if' - mantissa = r_longlong(bytes[size-2] & ((1<<(15-exp))-1)) - for i in range(size-3, -1, -1): - mantissa = mantissa << 8 | bytes[i] - mantissa = 1 + mantissa * mantissa_scale_factor - mantissa *= 0.5 - e = (bytes[-1] & 0x7f) << (exp - 7) - e += (bytes[size-2] >> (15 - exp)) & ((1<<(exp - 7)) -1) - e -= bias - e += 1 - sign = bytes[-1] & 0x80 - if e == bias + 2: - if mantissa == 0.5: - number = INFINITY - else: - return NAN + raise ValueError("invalid size value") + + sign = rarithmetic.copysign(1.0, x) < 0.0 + if rarithmetic.isinf(x): + mant = 0 + exp = MAX_EXP - MIN_EXP + 2 + elif rarithmetic.isnan(x): + mant = 1 << (MANT_DIG-2) # other values possible + exp = MAX_EXP - MIN_EXP + 2 + elif x == 0.0: + mant = 0 + exp = 0 else: - number = math.ldexp(mantissa,e) - if sign : number = -number - return number + m, e = math.frexp(abs(x)) # abs(x) == m * 2**e + exp = e - (MIN_EXP - 1) + if exp > 0: + # Normal case. + mant = round_to_nearest(m * (1 << MANT_DIG)) + mant -= 1 << MANT_DIG - 1 + else: + # Subnormal case. + if exp + MANT_DIG - 1 >= 0: + mant = round_to_nearest(m * (1 << exp + MANT_DIG - 1)) + else: + mant = 0 + exp = 0 + + # Special case: rounding produced a MANT_DIG-bit mantissa. + assert 0 <= mant <= 1 << MANT_DIG - 1 + if mant == 1 << MANT_DIG - 1: + mant = 0 + exp += 1 + + # Raise on overflow (in some circumstances, may want to return + # infinity instead). + if exp >= MAX_EXP - MIN_EXP + 2: + raise OverflowError("float too large to pack in this format") + + # check constraints + assert 0 <= mant < 1 << MANT_DIG - 1 + assert 0 <= exp <= MAX_EXP - MIN_EXP + 2 + assert 0 <= sign <= 1 + return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant + + +def pack_float8(result, x): + unsigned = float_pack(x, 8) + print unsigned + for i in range(8): + result.append(chr((unsigned >> (i * 8)) & 0xFF)) + + +def unpack_float8(s): + unsigned = 0 + for i in range(8): + unsigned |= ord(s[i]) << (i * 8) + return float_unpack(unsigned, 8) Added: pypy/branch/fast-forward/pypy/rlib/rstruct/test/test_ieee.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/test/test_ieee.py Thu Jul 1 23:59:27 2010 @@ -0,0 +1,103 @@ +import random +import struct + +from pypy.rlib.rarithmetic import isnan +from pypy.rlib.rstruct.ieee import float_pack, float_unpack + + +class TestFloatPacking: + + def check_float(self, x): + # check roundtrip + Q = float_pack(x, 8) + y = float_unpack(Q, 8) + assert repr(x) == repr(y) + + # check that packing agrees with the struct module + struct_pack8 = struct.unpack(' Author: benjamin Date: Fri Jul 2 00:08:48 2010 New Revision: 75747 Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py pypy/branch/fast-forward/pypy/module/math/interp_math.py Log: add copysign, isinf, and isnan Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/math/__init__.py Fri Jul 2 00:08:48 2010 @@ -11,6 +11,7 @@ 'pi' : 'interp_math.get(space).w_pi', 'pow' : 'interp_math.pow', 'cosh' : 'interp_math.cosh', + 'copysign' : 'interp_math.copysign', 'ldexp' : 'interp_math.ldexp', 'hypot' : 'interp_math.hypot', 'tan' : 'interp_math.tan', @@ -34,5 +35,7 @@ 'modf' : 'interp_math.modf', 'exp' : 'interp_math.exp', 'acos' : 'interp_math.acos', + 'isinf' : 'interp_math.isinf', + 'isnan' : 'interp_math.isnan', } Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Fri Jul 2 00:08:48 2010 @@ -1,5 +1,6 @@ - import math + +from pypy.rlib import rarithmetic from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped @@ -46,6 +47,22 @@ return space.wrap(r) math2._annspecialcase_ = 'specialize:arg(1)' +def copysign(space, x, y): + """Return x with the sign of y.""" + # No exceptions possible. + return space.wrap(rarithmetic.copysign(x, y)) +copysign.unwrap_spec = [ObjSpace, float, float] + +def isinf(space, x): + """Return True if x is infinity.""" + return space.wrap(rarithmetic.isinf(x)) +isinf.unwrap_spec = [ObjSpace, float, float] + +def isnan(space, x): + """Return True if x is not a number.""" + return space.wrap(rarithmetic.isnan(x)) +isnan.unwrap_spec = [ObjSpace, float, float] + def pow(space, x, y): """pow(x,y) From benjamin at codespeak.net Fri Jul 2 00:16:04 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 2 Jul 2010 00:16:04 +0200 (CEST) Subject: [pypy-svn] r75748 - pypy/branch/fast-forward/pypy/module/math Message-ID: <20100701221604.42B63282B9E@codespeak.net> Author: benjamin Date: Fri Jul 2 00:16:02 2010 New Revision: 75748 Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py Log: fix unwrap spec Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Fri Jul 2 00:16:02 2010 @@ -56,12 +56,12 @@ def isinf(space, x): """Return True if x is infinity.""" return space.wrap(rarithmetic.isinf(x)) -isinf.unwrap_spec = [ObjSpace, float, float] +isinf.unwrap_spec = [ObjSpace, float] def isnan(space, x): """Return True if x is not a number.""" return space.wrap(rarithmetic.isnan(x)) -isnan.unwrap_spec = [ObjSpace, float, float] +isnan.unwrap_spec = [ObjSpace, float] def pow(space, x, y): """pow(x,y) From getxsick at codespeak.net Fri Jul 2 00:17:42 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 2 Jul 2010 00:17:42 +0200 (CEST) Subject: [pypy-svn] r75749 - pypy/build/ubuntu/trunk/debian Message-ID: <20100701221742.91863282B9E@codespeak.net> Author: getxsick Date: Fri Jul 2 00:17:41 2010 New Revision: 75749 Modified: pypy/build/ubuntu/trunk/debian/Makefile.in Log: pypy-c is pypy-c. not testing_1 anymore Modified: pypy/build/ubuntu/trunk/debian/Makefile.in ============================================================================== --- pypy/build/ubuntu/trunk/debian/Makefile.in (original) +++ pypy/build/ubuntu/trunk/debian/Makefile.in Fri Jul 2 00:17:41 2010 @@ -44,7 +44,7 @@ mkdir $(TMPDIR) PYPY_USESSION_BASENAME='-' $(TRANSLATE) $(TRANSLATEOPTS) $(TARGET) $(TARGETOPTS) make -C $(TMPDIR)/usession-0/testing_1 - install -D $(TMPDIR)/usession-0/testing_1/testing_1 $@ + install -D $(TMPDIR)/usession-0/testing_1/pypy-c $@ clean: rm -rf bin From getxsick at codespeak.net Fri Jul 2 00:21:22 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 2 Jul 2010 00:21:22 +0200 (CEST) Subject: [pypy-svn] r75750 - pypy/build/ubuntu/1.3.0/debian Message-ID: <20100701222122.4B0E8282B9E@codespeak.net> Author: getxsick Date: Fri Jul 2 00:21:20 2010 New Revision: 75750 Modified: pypy/build/ubuntu/1.3.0/debian/Makefile.in Log: same as r75749 Modified: pypy/build/ubuntu/1.3.0/debian/Makefile.in ============================================================================== --- pypy/build/ubuntu/1.3.0/debian/Makefile.in (original) +++ pypy/build/ubuntu/1.3.0/debian/Makefile.in Fri Jul 2 00:21:20 2010 @@ -44,7 +44,7 @@ mkdir $(TMPDIR) PYPY_USESSION_BASENAME='-' $(TRANSLATE) $(TRANSLATEOPTS) $(TARGET) $(TARGETOPTS) make -C $(TMPDIR)/usession-0/testing_1 - install -D $(TMPDIR)/usession-0/testing_1/testing_1 $@ + install -D $(TMPDIR)/usession-0/testing_1/pypy-c $@ clean: rm -rf bin From benjamin at codespeak.net Fri Jul 2 01:42:43 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 2 Jul 2010 01:42:43 +0200 (CEST) Subject: [pypy-svn] r75751 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100701234243.7BE2F282B9E@codespeak.net> Author: benjamin Date: Fri Jul 2 01:42:41 2010 New Revision: 75751 Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Log: fake float.__getformat__ Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floattype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floattype.py Fri Jul 2 01:42:41 2010 @@ -1,4 +1,5 @@ from pypy.interpreter import gateway +from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.error import OperationError from pypy.objspace.std.stdtypedef import StdTypeDef from pypy.objspace.std.strutil import ParseStringError @@ -35,6 +36,18 @@ W_FloatObject.__init__(w_obj, value) return w_obj + +_float_format = float.__getformat__("float") +_double_format = float.__getformat__("double") +def descr___getformat__(space, w_cls, kind): + if kind == "float": + return space.wrap(_float_format) + elif kind == "double": + return space.wrap(_double_format) + raise OperationError(space.w_ValueError, + space.wrap("only float and double are valid")) + + # ____________________________________________________________ float_typedef = StdTypeDef("float", @@ -42,4 +55,7 @@ Convert a string or number to a floating point number, if possible.''', __new__ = gateway.interp2app(descr__new__), + __getformat__ = gateway.interp2app(descr___getformat__, + unwrap_spec=[ObjSpace, W_Root, str], + as_classmethod=True), ) Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Fri Jul 2 01:42:41 2010 @@ -317,6 +317,11 @@ assert not (nan < x) assert not (nan > x) + def test___getformat__(self): + float.__getformat__("float") + float.__getformat__("double") + raises(ValueError, float.__getformat__, "random") + def test_multimethod_slice(self): assert 5 .__add__(3.14) is NotImplemented assert 3.25 .__add__(5) == 8.25 From benjamin at codespeak.net Fri Jul 2 02:05:54 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 2 Jul 2010 02:05:54 +0200 (CEST) Subject: [pypy-svn] r75752 - in pypy/branch/fast-forward/pypy: interpreter module/math objspace/flow objspace/std objspace/std/test Message-ID: <20100702000554.28FF4282B9E@codespeak.net> Author: benjamin Date: Fri Jul 2 02:05:52 2010 New Revision: 75752 Modified: pypy/branch/fast-forward/pypy/interpreter/baseobjspace.py pypy/branch/fast-forward/pypy/module/math/__init__.py pypy/branch/fast-forward/pypy/module/math/interp_math.py pypy/branch/fast-forward/pypy/objspace/flow/operation.py pypy/branch/fast-forward/pypy/objspace/std/builtinshortcut.py pypy/branch/fast-forward/pypy/objspace/std/floatobject.py pypy/branch/fast-forward/pypy/objspace/std/intobject.py pypy/branch/fast-forward/pypy/objspace/std/longobject.py pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py pypy/branch/fast-forward/pypy/objspace/std/test/test_intobject.py pypy/branch/fast-forward/pypy/objspace/std/test/test_longobject.py Log: add math.trunc and support for floats, ints, and longs Modified: pypy/branch/fast-forward/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/baseobjspace.py Fri Jul 2 02:05:52 2010 @@ -1213,6 +1213,7 @@ ('getslice', 'getslice', 3, ['__getslice__']), ('setslice', 'setslice', 4, ['__setslice__']), ('delslice', 'delslice', 3, ['__delslice__']), + ('trunc', 'trunc', 1, ['__trunc__']), ('pos', 'pos', 1, ['__pos__']), ('neg', 'neg', 1, ['__neg__']), ('nonzero', 'truth', 1, ['__nonzero__']), Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/math/__init__.py Fri Jul 2 02:05:52 2010 @@ -37,5 +37,6 @@ 'acos' : 'interp_math.acos', 'isinf' : 'interp_math.isinf', 'isnan' : 'interp_math.isnan', + 'trunc' : 'interp_math.trunc', } Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Fri Jul 2 02:05:52 2010 @@ -47,6 +47,11 @@ return space.wrap(r) math2._annspecialcase_ = 'specialize:arg(1)' +def trunc(space, w_x): + """Truncate x.""" + return space.trunc(w_x) +trunc.unwrap_spec = [ObjSpace, W_Root] + def copysign(space, x, y): """Return x with the sign of y.""" # No exceptions possible. Modified: pypy/branch/fast-forward/pypy/objspace/flow/operation.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/flow/operation.py (original) +++ pypy/branch/fast-forward/pypy/objspace/flow/operation.py Fri Jul 2 02:05:52 2010 @@ -188,6 +188,7 @@ ('nonzero', operator.truth), ('is_true', bool), ('is_true', operator.truth), + ('trunc', unsupported), ('abs' , abs), ('hex', hex), ('oct', oct), Modified: pypy/branch/fast-forward/pypy/objspace/std/builtinshortcut.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/builtinshortcut.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/builtinshortcut.py Fri Jul 2 02:05:52 2010 @@ -35,7 +35,7 @@ 'setattr', 'delattr', 'userdel', # mostly for non-builtins 'get', 'set', 'delete', # uncommon (except on functions) 'getslice', 'setslice', 'delslice', # see below - 'delitem', # rare stuff? + 'delitem', 'trunc', # rare stuff? 'abs', 'hex', 'oct', # rare stuff? 'pos', 'divmod', 'cmp', # rare stuff? 'float', 'long', 'coerce', # rare stuff? Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Fri Jul 2 02:05:52 2010 @@ -8,7 +8,7 @@ from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.longobject import W_LongObject from pypy.rlib.rarithmetic import ovfcheck_float_to_int, intmask, isinf, isnan -from pypy.rlib.rarithmetic import formatd, LONG_BIT +from pypy.rlib.rarithmetic import formatd, LONG_BIT, FL_MAXINT, FL_MININT from pypy.rlib.rbigint import rbigint from pypy.tool.sourcetools import func_with_new_name @@ -73,6 +73,15 @@ except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("cannot convert float infinity to long")) +def trunc__Float(space, w_floatobj): + whole = math.modf(w_floatobj.floatval)[1] + if FL_MININT < whole < FL_MAXINT: + return space.newint(int(whole)) + try: + return W_LongObject.fromfloat(w_floatobj.floatval) + except OverflowError: + raise OperationError(space.w_OverflowError, + space.wrap("cannot convert infinity to long")) def float_w__Float(space, w_float): return w_float.floatval Modified: pypy/branch/fast-forward/pypy/objspace/std/intobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/intobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/intobject.py Fri Jul 2 02:05:52 2010 @@ -305,6 +305,7 @@ return wrapint(space, a) get_integer = int__Int pos__Int = int__Int +trunc__Int = int__Int def index__Int(space, w_int1): return get_integer(space, w_int1) Modified: pypy/branch/fast-forward/pypy/objspace/std/longobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/longobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/longobject.py Fri Jul 2 02:05:52 2010 @@ -77,6 +77,7 @@ return w_long1 l = w_long1.num return W_LongObject(l) +trunc__Long = long__Long def long__Int(space, w_intobj): return W_LongObject.fromint(space, w_intobj.intval) Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Fri Jul 2 02:05:52 2010 @@ -322,6 +322,17 @@ float.__getformat__("double") raises(ValueError, float.__getformat__, "random") + def test_trunc(self): + import math + assert math.trunc(1.5) == 1 + assert math.trunc(-1.5) == -1 + assert math.trunc(1.999999) == 1 + assert math.trunc(-1.999999) == -1 + assert math.trunc(-0.999999) == -0 + assert math.trunc(-100.999) == -100 + raises(OverflowError, math.trunc, float("inf")) + + def test_multimethod_slice(self): assert 5 .__add__(3.14) is NotImplemented assert 3.25 .__add__(5) == 8.25 Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_intobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_intobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_intobject.py Fri Jul 2 02:05:52 2010 @@ -285,6 +285,11 @@ class AppTestInt: + def test_trunc(self): + import math + assert math.trunc(1) == 1 + assert math.trunc(-1) == -1 + def test_int_callable(self): assert 43 == int(43) Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_longobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_longobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_longobject.py Fri Jul 2 02:05:52 2010 @@ -40,6 +40,12 @@ class AppTestLong: + + def test_trunc(self): + import math + assert math.trunc(1L) == 1L + assert math.trunc(-1L) == -1L + def test_add(self): assert int(123L + 12443L) == 123 + 12443 assert -20 + 2 + 3L + True == -14L From antocuni at codespeak.net Fri Jul 2 10:54:25 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 2 Jul 2010 10:54:25 +0200 (CEST) Subject: [pypy-svn] r75753 - pypy/trunk/pypy/tool/release/test Message-ID: <20100702085425.1BDA6282B9E@codespeak.net> Author: antocuni Date: Fri Jul 2 10:54:22 2010 New Revision: 75753 Modified: pypy/trunk/pypy/tool/release/test/test_package.py Log: fix test_package, for the case when we run the test without having run a translation first Modified: pypy/trunk/pypy/tool/release/test/test_package.py ============================================================================== --- pypy/trunk/pypy/tool/release/test/test_package.py (original) +++ pypy/trunk/pypy/tool/release/test/test_package.py Fri Jul 2 10:54:22 2010 @@ -26,9 +26,16 @@ assert prefix.join('README').check() th = tarfile.open(str(builddir.join('test.tar.bz2'))) assert th.getmember('test/lib_pypy/syslog.py') - assert th.getmember('test/include/Python.h') - assert th.getmember('test/include/modsupport.inl') - assert th.getmember('test/include/pypy_decl.h') + + # the headers file could be not there, because they are copied into + # trunk/include only during translation + includedir = py.path.local(pypydir).dirpath().join('include') + def check_include(name): + if includedir.join(name).check(file=True): + assert th.getmember('test/include/%s' % name) + check_include('Python.h') + check_include('modsupport.inl') + check_include('pypy_decl.h') finally: if fake_pypy_c: pypy_c.remove() From antocuni at codespeak.net Fri Jul 2 11:37:19 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 2 Jul 2010 11:37:19 +0200 (CEST) Subject: [pypy-svn] r75754 - pypy/build/bot2/codespeak-html Message-ID: <20100702093719.D4947282BF4@codespeak.net> Author: antocuni Date: Fri Jul 2 11:37:17 2010 New Revision: 75754 Modified: pypy/build/bot2/codespeak-html/index.html Log: add a link to the nightly build page Modified: pypy/build/bot2/codespeak-html/index.html ============================================================================== --- pypy/build/bot2/codespeak-html/index.html (original) +++ pypy/build/bot2/codespeak-html/index.html Fri Jul 2 11:37:17 2010 @@ -22,9 +22,12 @@
  • the Grid Display will give you a developer-oriented summary of recent buildbot activity.
    • -
  • The Latest Build for each builder is +
  • the Latest Build for each builder is here.
    • +
  • the Nightly Build page contains precompiled PyPy + binaries.
    • +
  • Recent Builds are summarized here, one per line.
  • Builders information
    • From fijal at codespeak.net Fri Jul 2 12:52:55 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Jul 2010 12:52:55 +0200 (CEST) Subject: [pypy-svn] r75756 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20100702105255.1EEF8282BEF@codespeak.net> Author: fijal Date: Fri Jul 2 12:52:54 2010 New Revision: 75756 Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py Log: Disable an attempt to allocate a nonmoving buffer. This speeds up twisted's tests by a considerable margin. Reason - in case we however do copy the buffer, we end up allocating new and new buffers, collecting every now and then instead of reusing freelists of C. Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rffi.py Fri Jul 2 12:52:54 2010 @@ -642,16 +642,8 @@ allows for the process to be performed without an extra copy. Make sure to call keep_buffer_alive_until_here on the returned values. """ - str_chars_offset = (offsetof(STRTYPE, 'chars') + \ - itemoffsetof(STRTYPE.chars, 0)) - gc_buf = rgc.malloc_nonmovable(STRTYPE, count) - if gc_buf: - realbuf = cast_ptr_to_adr(gc_buf) + str_chars_offset - raw_buf = cast(TYPEP, realbuf) - return raw_buf, gc_buf - else: - raw_buf = lltype.malloc(TYPEP.TO, count, flavor='raw') - return raw_buf, lltype.nullptr(STRTYPE) + raw_buf = lltype.malloc(TYPEP.TO, count, flavor='raw') + return raw_buf, lltype.nullptr(STRTYPE) alloc_buffer._always_inline_ = True # to get rid of the returned tuple # (char*, str, int, int) -> None From fijal at codespeak.net Fri Jul 2 12:53:38 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Jul 2010 12:53:38 +0200 (CEST) Subject: [pypy-svn] r75757 - pypy/branch/os-read-speedup Message-ID: <20100702105338.8B053282BEF@codespeak.net> Author: fijal Date: Fri Jul 2 12:53:36 2010 New Revision: 75757 Removed: pypy/branch/os-read-speedup/ Log: Remove "merged" branch. (The only change was applied in 75754) From hakanardo at codespeak.net Fri Jul 2 14:02:21 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Fri, 2 Jul 2010 14:02:21 +0200 (CEST) Subject: [pypy-svn] r75759 - pypy/branch/interplevel-array Message-ID: <20100702120221.5DF65282BE3@codespeak.net> Author: hakanardo Date: Fri Jul 2 14:02:19 2010 New Revision: 75759 Added: pypy/branch/interplevel-array/ (props changed) - copied from r75758, pypy/trunk/ Log: interpreter implementation of array module From hakanardo at codespeak.net Fri Jul 2 14:29:08 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Fri, 2 Jul 2010 14:29:08 +0200 (CEST) Subject: [pypy-svn] r75760 - in pypy/branch/interplevel-array: lib_pypy pypy/module/array pypy/module/array/test Message-ID: <20100702122908.B4813282BF4@codespeak.net> Author: hakanardo Date: Fri Jul 2 14:29:07 2010 New Revision: 75760 Added: pypy/branch/interplevel-array/lib_pypy/oldarray.py (props changed) - copied unchanged from r75759, pypy/branch/interplevel-array/lib_pypy/array.py pypy/branch/interplevel-array/pypy/module/array/ pypy/branch/interplevel-array/pypy/module/array/__init__.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Removed: pypy/branch/interplevel-array/lib_pypy/array.py Log: simple lltype.malloc wrapper Added: pypy/branch/interplevel-array/pypy/module/array/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/__init__.py Fri Jul 2 14:29:07 2010 @@ -0,0 +1,12 @@ +# Package initialisation +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + + appleveldefs = { + } + + interpleveldefs = { + 'array' : 'interp_array.array', + } + Added: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Fri Jul 2 14:29:07 2010 @@ -0,0 +1,44 @@ +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.typedef import TypeDef +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.interpreter.gateway import interp2app, ObjSpace +from pypy.rlib.jit import dont_look_inside +from pypy.rlib import rgc + +FloatArray=lltype.GcArray(lltype.Float) + +class W_Array(Wrappable): + @dont_look_inside + def __init__(self, space, size): + self.space=space + self.size=size + print "malloc" + #self.buffer = lltype.malloc(rffi.DOUBLEP.TO, size, flavor='raw', zero=True) + #self.buffer = rgc.malloc_nonmovable(lltype.GcArray(lltype.Float), size) + self.buffer = lltype.malloc(FloatArray, size, zero=True) + print "buf: ", self.buffer + + def __del__(self): + print "free" + #lltype.free(self.buffer, flavor='raw') + + def descr_getitem(self, idx): + return self.space.wrap(self.buffer[idx]) + descr_getitem.unwrap_spec = ['self', int] + + def descr_setitem(self, idx, val): + self.buffer[idx]=val + descr_setitem.unwrap_spec = ['self', int, float] + +W_Array.typedef = TypeDef( + 'Array', + __getitem__ = interp2app(W_Array.descr_getitem), + __setitem__ = interp2app(W_Array.descr_setitem), +) + + + + +def array(space,size): + return W_Array(space, size) +array.unwrap_spec=(ObjSpace, int) Added: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Fri Jul 2 14:29:07 2010 @@ -0,0 +1,26 @@ +from pypy.conftest import gettestobjspace + +class AppTestcArray: + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array',)) + cls.w_array = cls.space.appexec([], """(): + import array + return array.array + """) + + def test_simple(self): + a=self.array(10) + a[5]=7.42 + assert a[5]==7.42 + + +## space.sys.get('modules') + +## w_import = space.builtin.get('__import__') +## w_array = space.call(w_import, space.newlist([space.wrap('array')])) + +## print w_array +## space.appexec([], "(): import array; print array.array(7)") +## print space.appexec([], "(): import array; return array.array(7)") +## space.appexec([], "(): import array; a=array.array(7); print a[2]") +## space.appexec([], "(): import array; a=array.array(7); a[2]=7.42; print a[2]") From fijal at codespeak.net Fri Jul 2 14:56:49 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Jul 2010 14:56:49 +0200 (CEST) Subject: [pypy-svn] r75763 - in pypy/trunk/pypy/jit/codewriter: . test Message-ID: <20100702125649.AB354282BEF@codespeak.net> Author: fijal Date: Fri Jul 2 14:56:47 2010 New Revision: 75763 Modified: pypy/trunk/pypy/jit/codewriter/jtransform.py pypy/trunk/pypy/jit/codewriter/support.py pypy/trunk/pypy/jit/codewriter/test/test_codewriter.py Log: Start writing support for raw allocs in codewriter Modified: pypy/trunk/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/jtransform.py (original) +++ pypy/trunk/pypy/jit/codewriter/jtransform.py Fri Jul 2 14:56:47 2010 @@ -403,7 +403,12 @@ log.WARNING('ignoring hint %r at %r' % (hints, self.graph)) def rewrite_op_malloc_varsize(self, op): - assert op.args[1].value == {'flavor': 'gc'} + if op.args[1].value['flavor'] == 'raw': + ARRAY = op.args[0].value + return self._do_builtin_call(op, 'raw_malloc', + [op.args[2]], + extra = (ARRAY,), + extrakey = ARRAY) if op.args[0].value == rstr.STR: return SpaceOperation('newstr', [op.args[2]], op.result) elif op.args[0].value == rstr.UNICODE: Modified: pypy/trunk/pypy/jit/codewriter/support.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/support.py (original) +++ pypy/trunk/pypy/jit/codewriter/support.py Fri Jul 2 14:56:47 2010 @@ -282,6 +282,9 @@ # ---------- malloc with del ---------- + def _ll_2_raw_malloc(TP, size): + return lltype.malloc(TP, size, flavor='raw') + def build_ll_0_alloc_with_del(RESULT, vtable): def _ll_0_alloc_with_del(): p = lltype.malloc(RESULT.TO) @@ -289,6 +292,10 @@ return p return _ll_0_alloc_with_del + def build_ll_1_raw_malloc(ARRAY): + def _ll_1_raw_malloc(n): + return lltype.malloc(ARRAY, n, flavor='raw') + return _ll_1_raw_malloc class OOtypeHelpers: Modified: pypy/trunk/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/trunk/pypy/jit/codewriter/test/test_codewriter.py Fri Jul 2 14:56:47 2010 @@ -2,7 +2,7 @@ from pypy.jit.codewriter.codewriter import CodeWriter from pypy.jit.codewriter import support from pypy.jit.metainterp.history import AbstractDescr -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rffi class FakeCallDescr(AbstractDescr): def __init__(self, FUNC, ARGS, RESULT, effectinfo=None): @@ -157,3 +157,22 @@ # s = jitdriver_sd.mainjitcode.dump() assert "inline_call_ir_i " in s + +def test_raw_malloc_and_access(): + TP = rffi.CArray(lltype.Signed) + + def f(n): + a = lltype.malloc(TP, n, flavor='raw') + #a[0] = n + #res = a[0] + #lltype.free(a, flavor='raw') + #return res + + rtyper = support.annotate(f, [35]) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cw = CodeWriter(FakeCPU(rtyper), [jitdriver_sd]) + cw.find_all_graphs(FakePolicy()) + cw.make_jitcodes(verbose=True) + # + s = jitdriver_sd.mainjitcode.dump() + assert 'residual_call_ir_i $<* fn _ll_1_raw_malloc__Signed>' in s From afa at codespeak.net Fri Jul 2 15:05:23 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 2 Jul 2010 15:05:23 +0200 (CEST) Subject: [pypy-svn] r75764 - pypy/trunk/pypy/rlib Message-ID: <20100702130523.595BA282BEF@codespeak.net> Author: afa Date: Fri Jul 2 15:05:21 2010 New Revision: 75764 Modified: pypy/trunk/pypy/rlib/runicode.py Log: UNICHR allows wide unicode chars when testing on top of CPython with a narrow build. Use it to fix tests on the os/x buildbot Modified: pypy/trunk/pypy/rlib/runicode.py ============================================================================== --- pypy/trunk/pypy/rlib/runicode.py (original) +++ pypy/trunk/pypy/rlib/runicode.py Fri Jul 2 15:05:21 2010 @@ -822,13 +822,13 @@ else: # when we get here, chr is a 32-bit unicode character if chr <= MAXUNICODE: - builder.append(unichr(chr)) + builder.append(UNICHR(chr)) pos += digits elif chr <= 0x10ffff: chr -= 0x10000L builder.append(unichr(0xD800 + (chr >> 10))) - builder.append(unichr(0xDC00 + (chr & 0x03FF))) + builder.append(unichr(0xDC00 + (chr & 0x03FF))) pos += digits else: message = "illegal Unicode character" @@ -943,7 +943,7 @@ continue pos = look + 1 if code <= MAXUNICODE: - builder.append(unichr(code)) + builder.append(UNICHR(code)) else: code -= 0x10000L builder.append(unichr(0xD800 + (code >> 10))) From antocuni at codespeak.net Fri Jul 2 15:36:39 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 2 Jul 2010 15:36:39 +0200 (CEST) Subject: [pypy-svn] r75766 - pypy/build/bot2/pypybuildbot Message-ID: <20100702133639.E4331282BEF@codespeak.net> Author: antocuni Date: Fri Jul 2 15:36:38 2010 New Revision: 75766 Added: pypy/build/bot2/pypybuildbot/pypylist.py (contents, props changed) Modified: pypy/build/bot2/pypybuildbot/master.py Log: order the pypy binaries in a way that: - highest revisions are displayed first - they are grouped by platform: linux, then linux64, then win32 - they are ordered by features: jit, notjit, stackless Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Fri Jul 2 15:36:38 2010 @@ -2,7 +2,7 @@ from buildbot.buildslave import BuildSlave from buildbot.status.html import WebStatus from buildbot.process.builder import Builder -from twisted.web.static import File +from pypybuildbot.pypylist import PyPyList # I really wanted to pass logPath to Site from twisted.web.server import Site @@ -45,8 +45,8 @@ 'windows', 'mac', 'benchmark-run', 'other'])) -status.putChild('nightly', File(os.path.expanduser('~/nightly'), - defaultType='application/octet-stream')) +status.putChild('nightly', PyPyList(os.path.expanduser('~/nightly'), + defaultType='application/octet-stream')) pypybuilds = load('pypybuildbot.builds') Added: pypy/build/bot2/pypybuildbot/pypylist.py ============================================================================== --- (empty file) +++ pypy/build/bot2/pypybuildbot/pypylist.py Fri Jul 2 15:36:38 2010 @@ -0,0 +1,36 @@ +from twisted.web.static import File +import re + +# to get the desired order keep in mind that they are reversed at the end, so +# the highest the value, the bigger the priority +FEATURES = { + 'jit': 100, + 'nojit': 50, + 'stackless': 10 + } + +PLATFORMS = { + 'linux': 100, + 'linux64': 50, + 'win32': 10, + } + +def parsename(name): + # name is something like pypy-c-jit-75654-linux.tar.bz2 + try: + name2 = name.replace('.tar.bz2', '') + exe, backend, features, rev, platform = name2.split('-') + except ValueError: + return '', name + else: + return rev, PLATFORMS.get(platform, -1), FEATURES.get(features, -1), name + +class PyPyList(File): + + def listNames(self): + names = File.listNames(self) + items = map(parsename, names) + items.sort() + items.reverse() + return [item[-1] for item in items] + From antocuni at codespeak.net Fri Jul 2 15:43:40 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 2 Jul 2010 15:43:40 +0200 (CEST) Subject: [pypy-svn] r75767 - pypy/build/bot2/pypybuildbot Message-ID: <20100702134340.B03F4282BEF@codespeak.net> Author: antocuni Date: Fri Jul 2 15:43:36 2010 New Revision: 75767 Modified: pypy/build/bot2/pypybuildbot/pypylist.py Log: invert the order on which platform and features are considered for sorting Modified: pypy/build/bot2/pypybuildbot/pypylist.py ============================================================================== --- pypy/build/bot2/pypybuildbot/pypylist.py (original) +++ pypy/build/bot2/pypybuildbot/pypylist.py Fri Jul 2 15:43:36 2010 @@ -23,7 +23,7 @@ except ValueError: return '', name else: - return rev, PLATFORMS.get(platform, -1), FEATURES.get(features, -1), name + return rev, FEATURES.get(features, -1), PLATFORMS.get(platform, -1), name class PyPyList(File): @@ -33,4 +33,3 @@ items.sort() items.reverse() return [item[-1] for item in items] - From arigo at codespeak.net Fri Jul 2 16:04:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 16:04:45 +0200 (CEST) Subject: [pypy-svn] r75768 - pypy/branch/asmgcc-re Message-ID: <20100702140445.2E0C2282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 16:04:44 2010 New Revision: 75768 Removed: pypy/branch/asmgcc-re/ Log: Remove dead branch. From hakanardo at codespeak.net Fri Jul 2 16:27:53 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Fri, 2 Jul 2010 16:27:53 +0200 (CEST) Subject: [pypy-svn] r75769 - in pypy/branch/interplevel-array/pypy: jit/tl module/array/test module/pypyjit Message-ID: <20100702142753.30C97282BEF@codespeak.net> Author: hakanardo Date: Fri Jul 2 16:27:51 2010 New Revision: 75769 Added: pypy/branch/interplevel-array/pypy/module/array/test/Makefile pypy/branch/interplevel-array/pypy/module/array/test/intimg.c pypy/branch/interplevel-array/pypy/module/array/test/inttst.py pypy/branch/interplevel-array/pypy/module/array/test/loop.c pypy/branch/interplevel-array/pypy/module/array/test/tstintimg.c Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit.py pypy/branch/interplevel-array/pypy/module/pypyjit/policy.py Log: enabling jit, performace tests Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/tl/pypyjit.py (original) +++ pypy/branch/interplevel-array/pypy/jit/tl/pypyjit.py Fri Jul 2 16:27:51 2010 @@ -37,6 +37,7 @@ set_opt_level(config, level='jit') config.objspace.allworkingmodules = False config.objspace.usemodules.pypyjit = True +config.objspace.usemodules.array = True config.objspace.usemodules._weakref = False config.objspace.usemodules._sre = False set_pypy_opt_level(config, level='jit') Added: pypy/branch/interplevel-array/pypy/module/array/test/Makefile ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/test/Makefile Fri Jul 2 16:27:51 2010 @@ -0,0 +1,6 @@ +CFLAGS=-O3 -mtune=native -march=native -std=gnu99 + +tst: tstintimg.o intimg.o + gcc -o $@ $^ +loop: loop.o + gcc -o $@ $^ Added: pypy/branch/interplevel-array/pypy/module/array/test/intimg.c ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/test/intimg.c Fri Jul 2 16:27:51 2010 @@ -0,0 +1,18 @@ +void trans(unsigned char *img, unsigned int *intimg) { + int l=0; + for (int i=640; i<640*480; i++) { + l+=img[i]; + intimg[i]=intimg[i-640]+l; + } +} + + +void transf(double *img, double *intimg) { + double l=0; + for (int i=640; i<640*480; i++) { + l+=img[i]; + intimg[i]=intimg[i-640]+l; + } +} + + Added: pypy/branch/interplevel-array/pypy/module/array/test/inttst.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/test/inttst.py Fri Jul 2 16:27:51 2010 @@ -0,0 +1,19 @@ +#!/usr/bin/python +from time import time + +from array import array +#img=array('B',(0,)*640*480); +#intimg=array('I',(0,)*640*480); +img=array(640*480); +intimg=array(640*480); + +def f(): + l=0 + for i in xrange(640,640*480): + l+=img[i] + intimg[i]=intimg[i-640]+l + + +start=time() +for l in range(500): f() +print time()-start Added: pypy/branch/interplevel-array/pypy/module/array/test/loop.c ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/test/loop.c Fri Jul 2 16:27:51 2010 @@ -0,0 +1,9 @@ +int main() { + double s =0.0; + double i =0.0; + while (i < 1000000000) { + s += i; + i += 1.0; + } + return s; +} Added: pypy/branch/interplevel-array/pypy/module/array/test/tstintimg.c ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/test/tstintimg.c Fri Jul 2 16:27:51 2010 @@ -0,0 +1,16 @@ +void trans(unsigned char *img, unsigned int *intimg); + +void main2() { + unsigned char img[640*480]; + unsigned int intimg[640*480]; + + for (int l=0; l<500; l++) trans(img,intimg); +} + +void transf(double *img, double *intimg); +void main() { + double img[640*480]; + double intimg[640*480]; + + for (int l=0; l<500; l++) transf(img,intimg); +} Modified: pypy/branch/interplevel-array/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/interplevel-array/pypy/module/pypyjit/policy.py Fri Jul 2 16:27:51 2010 @@ -11,7 +11,7 @@ if '.' in modname: modname, _ = modname.split('.', 1) if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions', - 'imp', 'sys']: + 'imp', 'sys', 'array']: return True return False From benjamin at codespeak.net Fri Jul 2 16:44:11 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 2 Jul 2010 16:44:11 +0200 (CEST) Subject: [pypy-svn] r75770 - in pypy/branch/fast-forward/pypy/rpython/tool: . test Message-ID: <20100702144411.13D47282BEF@codespeak.net> Author: benjamin Date: Fri Jul 2 16:44:10 2010 New Revision: 75770 Modified: pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py pypy/branch/fast-forward/pypy/rpython/tool/test/test_rffi_platform.py Log: add DefinedConstantDouble Modified: pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py Fri Jul 2 16:44:10 2010 @@ -8,6 +8,7 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator.platform import CompilationError from pypy.tool.udir import udir +from pypy.rlib.rstruct import ieee # ____________________________________________________________ # @@ -45,6 +46,12 @@ DEFINED = Defined(macro) return configure(CConfig)['DEFINED'] +def getdefineddouble(macro, c_header_source): + class CConfig: + _compilation_info_ = eci_from_header(c_header_source) + DEFINED = DefinedConstantDouble(macro) + return configure(CConfig)['DEFINED'] + def has(name, c_header_source, include_dirs=None): class CConfig: _compilation_info_ = eci_from_header(c_header_source, include_dirs) @@ -400,6 +407,30 @@ return info['value'] return None +class DefinedConstantDouble(CConfigEntry): + + def __init__(self, macro): + self.name = self.macro = macro + + def prepare_code(self): + yield '#ifdef %s' % (self.macro,) + yield 'int i;' + yield 'double x = %s;' % (self.macro,) + yield 'unsigned char *p = (unsigned char *)&x;' + yield 'dump("defined", 1);' + yield 'for (i = 0; i < 8; i++) {' + yield ' printf("value_%d: %d\\n", i, p[i]);' + yield '}' + yield '#else' + yield 'dump("defined", 0);' + yield '#endif' + + def build_result(self, info, config_result): + if info["defined"]: + data = [chr(info["value_%d" % (i,)]) for i in range(8)] + return ieee.unpack_float8(''.join(data)) + return None + class DefinedConstantString(CConfigEntry): """ """ Modified: pypy/branch/fast-forward/pypy/rpython/tool/test/test_rffi_platform.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/tool/test/test_rffi_platform.py (original) +++ pypy/branch/fast-forward/pypy/rpython/tool/test/test_rffi_platform.py Fri Jul 2 16:44:10 2010 @@ -5,6 +5,7 @@ from pypy.tool.udir import udir from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator.platform import platform +from pypy.rlib import rarithmetic def import_ctypes(): try: @@ -106,6 +107,18 @@ '#define ALFKJLKJFLKJFKLEJDLKEWMECEE') assert res +def test_defined_constant_float(): + value = rffi_platform.getdefineddouble('BLAH', '#define BLAH 1.0') + assert value == 1.0 + value = rffi_platform.getdefineddouble('BLAH', '#define BLAH 1.5') + assert value == 1.5 + value = rffi_platform.getdefineddouble('BLAH', '#define BLAH 1.0e20') + assert value == 1.0e20 + value = rffi_platform.getdefineddouble('BLAH', '#define BLAH 1.0e50000') + assert value == float("inf") + value = rffi_platform.getdefineddouble('BLAH', '#define BLAH (double)0/0') + assert rarithmetic.isnan(value) + def test_configure(): test_h = udir.join('test_ctypes_platform.h') test_h.write('#define XYZZY 42\n') From benjamin at codespeak.net Fri Jul 2 16:54:59 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 2 Jul 2010 16:54:59 +0200 (CEST) Subject: [pypy-svn] r75771 - pypy/branch/fast-forward/pypy/rpython/tool Message-ID: <20100702145459.44626282BEF@codespeak.net> Author: benjamin Date: Fri Jul 2 16:54:57 2010 New Revision: 75771 Modified: pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py Log: just use stdlib struct Modified: pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py Fri Jul 2 16:54:57 2010 @@ -1,6 +1,9 @@ #! /usr/bin/env python -import os, py, sys +import os +import sys +import struct +import py from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import llmemory @@ -8,7 +11,6 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator.platform import CompilationError from pypy.tool.udir import udir -from pypy.rlib.rstruct import ieee # ____________________________________________________________ # @@ -428,7 +430,8 @@ def build_result(self, info, config_result): if info["defined"]: data = [chr(info["value_%d" % (i,)]) for i in range(8)] - return ieee.unpack_float8(''.join(data)) + # N.B. This depends on IEEE 754 being implemented. + return struct.unpack("d", ''.join(data))[0] return None class DefinedConstantString(CConfigEntry): From arigo at codespeak.net Fri Jul 2 17:01:30 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 17:01:30 +0200 (CEST) Subject: [pypy-svn] r75772 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20100702150130.4801E282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 17:01:28 2010 New Revision: 75772 Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py Log: Test and fix for allocating arrays with Boehm. The length is not at offset zero any more, since a while now. Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Fri Jul 2 17:01:28 2010 @@ -783,12 +783,14 @@ arglocs.append(self.loc(op.args[0])) return self._call(op, arglocs) # boehm GC (XXX kill the following code at some point) - scale_of_field, basesize, _ = self._unpack_arraydescr(op.descr) - return self._malloc_varsize(basesize, 0, scale_of_field, op.args[0], - op.result) + scale_of_field, basesize, ofs_length, _ = ( + self._unpack_arraydescr(op.descr)) + return self._malloc_varsize(basesize, ofs_length, scale_of_field, + op.args[0], op.result) def _unpack_arraydescr(self, arraydescr): assert isinstance(arraydescr, BaseArrayDescr) + ofs_length = arraydescr.get_ofs_length(self.translate_support_code) ofs = arraydescr.get_base_size(self.translate_support_code) size = arraydescr.get_item_size(self.translate_support_code) ptr = arraydescr.is_array_of_pointers() @@ -796,7 +798,7 @@ while (1 << scale) < size: scale += 1 assert (1 << scale) == size - return scale, ofs, ptr + return scale, ofs, ofs_length, ptr def _unpack_fielddescr(self, fielddescr): assert isinstance(fielddescr, BaseFieldDescr) @@ -831,7 +833,7 @@ consider_unicodesetitem = consider_strsetitem def consider_setarrayitem_gc(self, op): - scale, ofs, ptr = self._unpack_arraydescr(op.descr) + scale, ofs, _, ptr = self._unpack_arraydescr(op.descr) base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) if scale == 0: need_lower_byte = True @@ -858,7 +860,7 @@ consider_getfield_gc_pure = consider_getfield_gc def consider_getarrayitem_gc(self, op): - scale, ofs, _ = self._unpack_arraydescr(op.descr) + scale, ofs, _, _ = self._unpack_arraydescr(op.descr) base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) ofs_loc = self.rm.make_sure_var_in_reg(op.args[1], op.args) self.rm.possibly_free_vars(op.args) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py Fri Jul 2 17:01:28 2010 @@ -105,6 +105,12 @@ def test_compile_boehm(): myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) + @dont_look_inside + def see(lst, n): + assert len(lst) == 3 + assert lst[0] == n+10 + assert lst[1] == n+20 + assert lst[2] == n+30 def main(n, x): while n > 0: myjitdriver.can_enter_jit(n=n, x=x) @@ -112,6 +118,7 @@ y = X() y.foo = x.foo n -= y.foo + see([n+10, n+20, n+30], n) res = compile_and_run(get_entry(get_g(main)), "boehm", jit=True) assert int(res) >= 16 From benjamin at codespeak.net Fri Jul 2 17:09:23 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 2 Jul 2010 17:09:23 +0200 (CEST) Subject: [pypy-svn] r75773 - in pypy/branch/fast-forward/pypy/module/sys: . test Message-ID: <20100702150923.2791E282BF4@codespeak.net> Author: benjamin Date: Fri Jul 2 17:09:21 2010 New Revision: 75773 Added: pypy/branch/fast-forward/pypy/module/sys/system.py (contents, props changed) Modified: pypy/branch/fast-forward/pypy/module/sys/__init__.py pypy/branch/fast-forward/pypy/module/sys/test/test_sysmodule.py Log: add sys.float_info Modified: pypy/branch/fast-forward/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/sys/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/sys/__init__.py Fri Jul 2 17:09:21 2010 @@ -73,6 +73,8 @@ 'getdefaultencoding' : 'interp_encoding.getdefaultencoding', 'setdefaultencoding' : 'interp_encoding.setdefaultencoding', 'getfilesystemencoding' : 'interp_encoding.getfilesystemencoding', + + 'float_info' : 'system.get_float_info(space)', } if sys.platform == 'win32': Added: pypy/branch/fast-forward/pypy/module/sys/system.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/pypy/module/sys/system.py Fri Jul 2 17:09:21 2010 @@ -0,0 +1,61 @@ +"""Information about the current system.""" + +from pypy.interpreter import gateway + +from pypy.rpython.tool import rffi_platform +from pypy.translator.tool.cbuild import ExternalCompilationInfo + + +class CConfig: + _compilation_info_ = ExternalCompilationInfo(includes=["float.h"]) + +float_constants = ["DBL_MAX", "DBL_MIN", "DBL_EPSILON"] +int_constants = ["DBL_MAX_EXP", "DBL_MAX_10_EXP", + "DBL_MIN_EXP", "DBL_MIN_10_EXP", + "DBL_DIG", "DBL_MANT_DIG", + "FLT_RADIX", "FLT_ROUNDS"] +for const in float_constants: + setattr(CConfig, const, rffi_platform.DefinedConstantDouble(const)) +for const in int_constants: + setattr(CConfig, const, rffi_platform.DefinedConstantInteger(const)) +del float_constants, int_constants, const + +globals().update(rffi_platform.configure(CConfig)) + + +app = gateway.applevel(""" +"NOT_RPYTHON" +from _structseq import structseqtype, structseqfield +class float_info: + __metaclass__ = structseqtype + + max = structseqfield(0) + max_exp = structseqfield(1) + max_10_exp = structseqfield(2) + min = structseqfield(3) + min_exp = structseqfield(4) + min_10_exp = structseqfield(5) + dig = structseqfield(6) + mant_dig = structseqfield(7) + epsilon = structseqfield(8) + radix = structseqfield(9) + rounds = structseqfield(10) +""") + + +def get_float_info(space): + info_w = [ + space.wrap(DBL_MAX), + space.wrap(DBL_MAX_EXP), + space.wrap(DBL_MAX_10_EXP), + space.wrap(DBL_MIN), + space.wrap(DBL_MIN_EXP), + space.wrap(DBL_MIN_10_EXP), + space.wrap(DBL_DIG), + space.wrap(DBL_MANT_DIG), + space.wrap(DBL_EPSILON), + space.wrap(FLT_RADIX), + space.wrap(FLT_ROUNDS), + ] + w_float_info = app.wget(space, "float_info") + return space.call_function(w_float_info, space.newtuple(info_w)) Modified: pypy/branch/fast-forward/pypy/module/sys/test/test_sysmodule.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/sys/test/test_sysmodule.py (original) +++ pypy/branch/fast-forward/pypy/module/sys/test/test_sysmodule.py Fri Jul 2 17:09:21 2010 @@ -114,6 +114,22 @@ import sys assert sys.getfilesystemencoding() == self.filesystemenc + def test_float_info(self): + import sys + fi = sys.float_info + assert isinstance(fi.epsilon, float) + assert isinstance(fi.dig, int) + assert isinstance(fi.mant_dig, int) + assert isinstance(fi.max, float) + assert isinstance(fi.max_exp, int) + assert isinstance(fi.max_10_exp, int) + assert isinstance(fi.min, float) + assert isinstance(fi.min_exp, int) + assert isinstance(fi.min_10_exp, int) + assert isinstance(fi.radix, int) + assert isinstance(fi.rounds, int) + + class AppTestSysModulePortedFromCPython: def setup_class(cls): From fijal at codespeak.net Fri Jul 2 17:10:50 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Jul 2010 17:10:50 +0200 (CEST) Subject: [pypy-svn] r75774 - in pypy/trunk/pypy/jit/codewriter: . test Message-ID: <20100702151050.358A4282BF4@codespeak.net> Author: fijal Date: Fri Jul 2 17:10:48 2010 New Revision: 75774 Modified: pypy/trunk/pypy/jit/codewriter/jtransform.py pypy/trunk/pypy/jit/codewriter/support.py pypy/trunk/pypy/jit/codewriter/test/test_codewriter.py Log: Support more raw operations: getitem, setitem, malloc and free Modified: pypy/trunk/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/jtransform.py (original) +++ pypy/trunk/pypy/jit/codewriter/jtransform.py Fri Jul 2 17:10:48 2010 @@ -420,9 +420,14 @@ return SpaceOperation('new_array', [arraydescr, op.args[2]], op.result) + def rewrite_op_free(self, op): + assert op.args[1].value == 'raw' + ARRAY = op.args[0].concretetype.TO + return self._do_builtin_call(op, 'raw_free', [op.args[0]], + extra = (ARRAY,), extrakey = ARRAY) + def rewrite_op_getarrayitem(self, op): ARRAY = op.args[0].concretetype.TO - assert ARRAY._gckind == 'gc' if self._array_of_voids(ARRAY): return [] if op.args[0] in self.vable_array_vars: # for virtualizables @@ -436,13 +441,12 @@ # normal case follows arraydescr = self.cpu.arraydescrof(ARRAY) kind = getkind(op.result.concretetype) - return SpaceOperation('getarrayitem_gc_%s' % kind[0], + return SpaceOperation('getarrayitem_%s_%s' % (ARRAY._gckind, kind[0]), [op.args[0], arraydescr, op.args[1]], op.result) def rewrite_op_setarrayitem(self, op): ARRAY = op.args[0].concretetype.TO - assert ARRAY._gckind == 'gc' if self._array_of_voids(ARRAY): return [] if op.args[0] in self.vable_array_vars: # for virtualizables @@ -455,7 +459,7 @@ op.args[1], op.args[2]], None)] arraydescr = self.cpu.arraydescrof(ARRAY) kind = getkind(op.args[2].concretetype) - return SpaceOperation('setarrayitem_gc_%s' % kind[0], + return SpaceOperation('setarrayitem_%s_%s' % (ARRAY._gckind, kind[0]), [op.args[0], arraydescr, op.args[1], op.args[2]], None) Modified: pypy/trunk/pypy/jit/codewriter/support.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/support.py (original) +++ pypy/trunk/pypy/jit/codewriter/support.py Fri Jul 2 17:10:48 2010 @@ -297,6 +297,11 @@ return lltype.malloc(ARRAY, n, flavor='raw') return _ll_1_raw_malloc + def build_ll_1_raw_free(ARRAY): + def _ll_1_raw_free(p): + lltype.free(p, flavor='raw') + return _ll_1_raw_free + class OOtypeHelpers: # ---------- dict ---------- Modified: pypy/trunk/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/trunk/pypy/jit/codewriter/test/test_codewriter.py Fri Jul 2 17:10:48 2010 @@ -24,12 +24,17 @@ def as_vtable_size_descr(self): return self +class FakeArrayDescr(AbstractDescr): + def __init__(self, ARRAY): + self.ARRAY = ARRAY + class FakeCPU: def __init__(self, rtyper): self.rtyper = rtyper calldescrof = FakeCallDescr fielddescrof = FakeFieldDescr sizeof = FakeSizeDescr + arraydescrof = FakeArrayDescr class FakePolicy: def look_inside_graph(self, graph): @@ -163,10 +168,10 @@ def f(n): a = lltype.malloc(TP, n, flavor='raw') - #a[0] = n - #res = a[0] - #lltype.free(a, flavor='raw') - #return res + a[0] = n + res = a[0] + lltype.free(a, flavor='raw') + return res rtyper = support.annotate(f, [35]) jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) @@ -176,3 +181,6 @@ # s = jitdriver_sd.mainjitcode.dump() assert 'residual_call_ir_i $<* fn _ll_1_raw_malloc__Signed>' in s + assert 'setarrayitem_raw_i' in s + assert 'getarrayitem_raw_i' in s + assert 'residual_call_ir_v $<* fn _ll_1_raw_free__arrayPtr>' in s From arigo at codespeak.net Fri Jul 2 17:14:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 17:14:34 +0200 (CEST) Subject: [pypy-svn] r75775 - pypy/trunk/pypy/module/cpyext Message-ID: <20100702151434.E92C2282BF4@codespeak.net> Author: arigo Date: Fri Jul 2 17:14:33 2010 New Revision: 75775 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: The files in pypy-trunk/include are created read-only, but then if they are modified, the copy() fails to overwrite them. In that case, remove the target and try again. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Fri Jul 2 17:14:33 2010 @@ -104,8 +104,12 @@ for name in ("pypy_decl.h", "pypy_macros.h"): headers.append(udir.join(name)) for header in headers: - header.copy(dstdir) target = dstdir.join(header.basename) + try: + header.copy(dstdir) + except py.error.EACCES: + target.remove() # maybe it was a read-only file + header.copy(dstdir) target.chmod(0444) # make the file read-only, to make sure that nobody # edits it by mistake From hakanardo at codespeak.net Fri Jul 2 17:29:49 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Fri, 2 Jul 2010 17:29:49 +0200 (CEST) Subject: [pypy-svn] r75776 - pypy/branch/interplevel-array/pypy/jit/tl Message-ID: <20100702152949.74721282BF4@codespeak.net> Author: hakanardo Date: Fri Jul 2 17:29:47 2010 New Revision: 75776 Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py Log: testcode Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py Fri Jul 2 17:29:47 2010 @@ -1,38 +1,48 @@ -base = object +## base = object -class Number(base): - __slots__ = ('val', ) - def __init__(self, val=0): - self.val = val - - def __add__(self, other): - if not isinstance(other, int): - other = other.val - return Number(val=self.val + other) +## class Number(base): +## __slots__ = ('val', ) +## def __init__(self, val=0): +## self.val = val + +## def __add__(self, other): +## if not isinstance(other, int): +## other = other.val +## return Number(val=self.val + other) - def __cmp__(self, other): - val = self.val - if not isinstance(other, int): - other = other.val - return cmp(val, other) - - def __nonzero__(self): - return bool(self.val) - -def g(x, inc=2): - return x + inc - -def f(n, x, inc): - while x < n: - x = g(x, inc=1) - return x - -import time -#t1 = time.time() -#f(10000000, Number(), 1) -#t2 = time.time() -#print t2 - t1 -t1 = time.time() -f(10000000, 0, 1) -t2 = time.time() -print t2 - t1 +## def __cmp__(self, other): +## val = self.val +## if not isinstance(other, int): +## other = other.val +## return cmp(val, other) + +## def __nonzero__(self): +## return bool(self.val) + +## def g(x, inc=2): +## return x + inc + +## def f(n, x, inc): +## while x < n: +## x = g(x, inc=1) +## return x + +## import time +## #t1 = time.time() +## #f(10000000, Number(), 1) +## #t2 = time.time() +## #print t2 - t1 +## t1 = time.time() +## f(10000000, 0, 1) +## t2 = time.time() +## print t2 - t1 + +from array import array +img=array(4) +def f(): + sa=0 + for i in xrange(4): + sa+=img[i] + return sa + +print f() From arigo at codespeak.net Fri Jul 2 17:38:07 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 17:38:07 +0200 (CEST) Subject: [pypy-svn] r75777 - pypy/trunk/pypy/interpreter Message-ID: <20100702153807.7CC0A282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 17:38:05 2010 New Revision: 75777 Modified: pypy/trunk/pypy/interpreter/pyopcode.py Log: Write this code in a style that tells the annotator that oparg is not negative. Useful e.g. in LOAD_FAST where it is used as an index in a list. Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Fri Jul 2 17:38:05 2010 @@ -14,6 +14,7 @@ from pypy.rlib import jit, rstackovf, rstack from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.debug import check_nonneg from pypy.tool.stdlib_opcode import (bytecode_spec, host_bytecode_spec, unrolling_all_opcode_descs, opmap, host_opmap) @@ -186,7 +187,7 @@ lo = ord(co_code[next_instr]) hi = ord(co_code[next_instr+1]) next_instr += 2 - oparg = (hi << 8) | lo + oparg = (hi * 256) | lo else: oparg = 0 @@ -197,7 +198,7 @@ lo = ord(co_code[next_instr+1]) hi = ord(co_code[next_instr+2]) next_instr += 3 - oparg = (oparg << 16) | (hi << 8) | lo + oparg = (oparg * 65536) | (hi * 256) | lo if opcode == self.opcodedesc.RETURN_VALUE.index: w_returnvalue = self.popvalue() @@ -323,7 +324,9 @@ ## def NOP(self, oparg, next_instr): - pass + # annotation-time check: if it fails, it means that the decoding + # of oparg failed to produce an integer which is annotated as non-neg + check_nonneg(oparg) def LOAD_FAST(self, varindex, next_instr): # access a local variable directly From jcreigh at codespeak.net Fri Jul 2 17:44:55 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 2 Jul 2010 17:44:55 +0200 (CEST) Subject: [pypy-svn] r75778 - in pypy/branch/x86-64-jit-backend: . dotviewer include lib-python lib-python/modified-2.5.2 lib-python/modified-2.5.2/distutils lib-python/modified-2.5.2/distutils/command lib-python/modified-2.5.2/test lib_pypy lib_pypy/_ctypes lib_pypy/ctypes_config_cache lib_pypy/ctypes_config_cache/test lib_pypy/distributed lib_pypy/distributed/demo lib_pypy/pypy_test lib_pypy/xml lib_pypy/xml/dom lib_pypy/xml/etree lib_pypy/xml/parsers lib_pypy/xml/sax py/_process pypy pypy/_interfaces pypy/annotation pypy/annotation/test pypy/bin pypy/config pypy/doc pypy/doc/config pypy/doc/jit pypy/interpreter pypy/interpreter/pyparser pypy/interpreter/pyparser/test pypy/interpreter/test pypy/jit/backend pypy/jit/backend/llgraph pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/backend/x86/tool pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/test pypy/jit/tl pypy/jit/tl/spli pypy/jit/tl/tinyframe pypy/jit/tool pypy/module pypy/module/__builtin__ pypy/module/__builtin__/test pypy/module/_codecs pypy/module/_codecs/test pypy/module/_sre/test pypy/module/_stackless pypy/module/_stackless/test pypy/module/cpyext pypy/module/cpyext/test pypy/module/operator/test pypy/module/sys pypy/module/sys/test pypy/module/test_lib_pypy pypy/module/test_lib_pypy/ctypes_tests pypy/module/test_lib_pypy/test_distributed pypy/module/unicodedata pypy/objspace/flow pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rlib/parsing/test pypy/rlib/rsdl/test pypy/rlib/rsre/test pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/lltypesystem/test pypy/rpython/memory pypy/rpython/microbench pypy/rpython/module pypy/rpython/ootypesystem pypy/tool pypy/tool/algo pypy/tool/algo/test pypy/tool/pytest pypy/tool/release pypy/tool/release/test pypy/tool/test pypy/translator pypy/translator/backendopt/test pypy/translator/benchmark pypy/translator/c pypy/translator/c/gcc pypy/translator/c/src pypy/translator/c/test pypy/translator/cli pypy/translator/cli/test pypy/translator/goal pypy/translator/goal/test2 pypy/translator/jvm pypy/translator/jvm/test pypy/translator/microbench/pybench pypy/translator/oosupport pypy/translator/platform pypy/translator/sandbox pypy/translator/sandbox/test pypy/translator/test pypy/translator/tool site-packages Message-ID: <20100702154455.1C40F282BEF@codespeak.net> Author: jcreigh Date: Fri Jul 2 17:44:46 2010 New Revision: 75778 Added: pypy/branch/x86-64-jit-backend/include/ (props changed) - copied from r75772, pypy/trunk/include/ pypy/branch/x86-64-jit-backend/lib_pypy/ (props changed) - copied from r75772, pypy/trunk/lib_pypy/ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/jitdriver.py - copied unchanged from r75772, pypy/trunk/pypy/jit/metainterp/jitdriver.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_jitdriver.py - copied unchanged from r75772, pypy/trunk/pypy/jit/metainterp/test/test_jitdriver.py pypy/branch/x86-64-jit-backend/pypy/module/conftest.py - copied unchanged from r75772, pypy/trunk/pypy/module/conftest.py pypy/branch/x86-64-jit-backend/pypy/module/sys/test/test_initialpath.py - copied unchanged from r75772, pypy/trunk/pypy/module/sys/test/test_initialpath.py pypy/branch/x86-64-jit-backend/pypy/module/test_lib_pypy/ (props changed) - copied from r75772, pypy/trunk/pypy/module/test_lib_pypy/ pypy/branch/x86-64-jit-backend/pypy/tool/identity_dict.py - copied unchanged from r75772, pypy/trunk/pypy/tool/identity_dict.py pypy/branch/x86-64-jit-backend/pypy/tool/lib_pypy.py - copied unchanged from r75772, pypy/trunk/pypy/tool/lib_pypy.py pypy/branch/x86-64-jit-backend/pypy/tool/test/test_identitydict.py - copied unchanged from r75772, pypy/trunk/pypy/tool/test/test_identitydict.py pypy/branch/x86-64-jit-backend/pypy/tool/test/test_lib_pypy.py - copied unchanged from r75772, pypy/trunk/pypy/tool/test/test_lib_pypy.py pypy/branch/x86-64-jit-backend/site-packages/ - copied from r75772, pypy/trunk/site-packages/ Removed: pypy/branch/x86-64-jit-backend/pypy/_interfaces/ pypy/branch/x86-64-jit-backend/pypy/jit/tl/run_all_tests.py pypy/branch/x86-64-jit-backend/pypy/module/_codecs/app_codecs.py pypy/branch/x86-64-jit-backend/pypy/module/unicodedata/LICENSE pypy/branch/x86-64-jit-backend/pypy/tool/release/pack.py pypy/branch/x86-64-jit-backend/pypy/tool/release/test/test_pack.py Modified: pypy/branch/x86-64-jit-backend/ (props changed) pypy/branch/x86-64-jit-backend/LICENSE pypy/branch/x86-64-jit-backend/dotviewer/sshgraphserver.py pypy/branch/x86-64-jit-backend/lib-python/conftest.py pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/cmd.py pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/command/build_ext.py pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/command/install.py pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/sysconfig.py pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/site.py pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/test/regrtest.py pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/test/test_doctest.py pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/test/test_sys.py pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/ (props changed) pypy/branch/x86-64-jit-backend/lib_pypy/ctypes_config_cache/ (props changed) pypy/branch/x86-64-jit-backend/lib_pypy/ctypes_config_cache/test/ (props changed) pypy/branch/x86-64-jit-backend/lib_pypy/distributed/ (props changed) pypy/branch/x86-64-jit-backend/lib_pypy/distributed/demo/ (props changed) pypy/branch/x86-64-jit-backend/lib_pypy/pypy_test/ (props changed) pypy/branch/x86-64-jit-backend/lib_pypy/xml/ (props changed) pypy/branch/x86-64-jit-backend/lib_pypy/xml/dom/ (props changed) pypy/branch/x86-64-jit-backend/lib_pypy/xml/etree/ (props changed) pypy/branch/x86-64-jit-backend/lib_pypy/xml/parsers/ (props changed) pypy/branch/x86-64-jit-backend/lib_pypy/xml/sax/ (props changed) pypy/branch/x86-64-jit-backend/py/_process/cmdexec.py pypy/branch/x86-64-jit-backend/pypy/annotation/bookkeeper.py pypy/branch/x86-64-jit-backend/pypy/annotation/test/autopath.py pypy/branch/x86-64-jit-backend/pypy/bin/autopath.py pypy/branch/x86-64-jit-backend/pypy/bin/py.py pypy/branch/x86-64-jit-backend/pypy/config/autopath.py pypy/branch/x86-64-jit-backend/pypy/config/pypyoption.py pypy/branch/x86-64-jit-backend/pypy/conftest.py pypy/branch/x86-64-jit-backend/pypy/doc/_ref.txt pypy/branch/x86-64-jit-backend/pypy/doc/coding-guide.txt pypy/branch/x86-64-jit-backend/pypy/doc/config/autopath.py pypy/branch/x86-64-jit-backend/pypy/doc/config/objspace.usemodules.md5.txt pypy/branch/x86-64-jit-backend/pypy/doc/config/objspace.usemodules.sha.txt pypy/branch/x86-64-jit-backend/pypy/doc/config/objspace.usemodules.struct.txt pypy/branch/x86-64-jit-backend/pypy/doc/cpython_differences.txt pypy/branch/x86-64-jit-backend/pypy/doc/ctypes-implementation.txt pypy/branch/x86-64-jit-backend/pypy/doc/docindex.txt pypy/branch/x86-64-jit-backend/pypy/doc/getting-started-python.txt pypy/branch/x86-64-jit-backend/pypy/doc/index.txt pypy/branch/x86-64-jit-backend/pypy/doc/jit/pyjitpl5.txt pypy/branch/x86-64-jit-backend/pypy/doc/maemo.txt pypy/branch/x86-64-jit-backend/pypy/doc/objspace-proxies.txt pypy/branch/x86-64-jit-backend/pypy/doc/release-1.3.0.txt pypy/branch/x86-64-jit-backend/pypy/doc/sandbox.txt pypy/branch/x86-64-jit-backend/pypy/doc/stackless.txt pypy/branch/x86-64-jit-backend/pypy/interpreter/baseobjspace.py pypy/branch/x86-64-jit-backend/pypy/interpreter/pyparser/parsestring.py pypy/branch/x86-64-jit-backend/pypy/interpreter/pyparser/test/test_parsestring.py pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_module.py pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_objspace.py pypy/branch/x86-64-jit-backend/pypy/interpreter/unicodehelper.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/autopath.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/llgraph/llimpl.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/model.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/support.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/autopath.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/autopath.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/call.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/codewriter.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/jitcode.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/jtransform.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/support.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_call.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_flatten.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/blackhole.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/compile.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/history.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/pyjitpl.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/resoperation.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/resume.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_basic.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_blackhole.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_compile.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_list.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_tl.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_ztranslation.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/viewnode.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/virtualizable.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmspot.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmstate.py pypy/branch/x86-64-jit-backend/pypy/jit/tl/autopath.py pypy/branch/x86-64-jit-backend/pypy/jit/tl/spli/autopath.py pypy/branch/x86-64-jit-backend/pypy/jit/tl/spli/targetspli.py pypy/branch/x86-64-jit-backend/pypy/jit/tl/targettlc.py pypy/branch/x86-64-jit-backend/pypy/jit/tl/targettlr.py pypy/branch/x86-64-jit-backend/pypy/jit/tl/tinyframe/targettinyframe.py pypy/branch/x86-64-jit-backend/pypy/jit/tool/autopath.py pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/functional.py pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/test/autopath.py pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/test/test_functional.py pypy/branch/x86-64-jit-backend/pypy/module/_codecs/__init__.py pypy/branch/x86-64-jit-backend/pypy/module/_codecs/interp_codecs.py pypy/branch/x86-64-jit-backend/pypy/module/_codecs/test/autopath.py pypy/branch/x86-64-jit-backend/pypy/module/_codecs/test/test_codecs.py pypy/branch/x86-64-jit-backend/pypy/module/_sre/test/autopath.py pypy/branch/x86-64-jit-backend/pypy/module/_stackless/interp_coroutine.py pypy/branch/x86-64-jit-backend/pypy/module/_stackless/rcoroutine.py pypy/branch/x86-64-jit-backend/pypy/module/_stackless/test/test_coroutine.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/api.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_api.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_cpyext.py pypy/branch/x86-64-jit-backend/pypy/module/operator/test/test_operator.py pypy/branch/x86-64-jit-backend/pypy/module/sys/__init__.py pypy/branch/x86-64-jit-backend/pypy/module/sys/interp_encoding.py pypy/branch/x86-64-jit-backend/pypy/module/sys/state.py pypy/branch/x86-64-jit-backend/pypy/module/sys/test/autopath.py pypy/branch/x86-64-jit-backend/pypy/module/sys/version.py pypy/branch/x86-64-jit-backend/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) pypy/branch/x86-64-jit-backend/pypy/module/test_lib_pypy/test_distributed/ (props changed) pypy/branch/x86-64-jit-backend/pypy/objspace/flow/model.py pypy/branch/x86-64-jit-backend/pypy/objspace/std/floatobject.py pypy/branch/x86-64-jit-backend/pypy/objspace/std/marshal_impl.py pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_floatobject.py pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_shadowtracking.py pypy/branch/x86-64-jit-backend/pypy/objspace/std/unicodeobject.py pypy/branch/x86-64-jit-backend/pypy/pytest-A.cfg pypy/branch/x86-64-jit-backend/pypy/rlib/debug.py pypy/branch/x86-64-jit-backend/pypy/rlib/parsing/test/autopath.py pypy/branch/x86-64-jit-backend/pypy/rlib/rcoroutine.py pypy/branch/x86-64-jit-backend/pypy/rlib/rmd5.py pypy/branch/x86-64-jit-backend/pypy/rlib/rsdl/test/autopath.py pypy/branch/x86-64-jit-backend/pypy/rlib/rsha.py pypy/branch/x86-64-jit-backend/pypy/rlib/rsre/test/targetrsre.py pypy/branch/x86-64-jit-backend/pypy/rlib/rstring.py pypy/branch/x86-64-jit-backend/pypy/rlib/runicode.py pypy/branch/x86-64-jit-backend/pypy/rlib/rzipfile.py pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_debug.py pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_runicode.py pypy/branch/x86-64-jit-backend/pypy/rpython/extfunc.py pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/lltype.py pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rclass.py pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rffi.py pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/test/test_lltype.py pypy/branch/x86-64-jit-backend/pypy/rpython/memory/gctypelayout.py pypy/branch/x86-64-jit-backend/pypy/rpython/microbench/autopath.py pypy/branch/x86-64-jit-backend/pypy/rpython/microbench/microbench.py pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_time.py pypy/branch/x86-64-jit-backend/pypy/rpython/ootypesystem/rclass.py pypy/branch/x86-64-jit-backend/pypy/tool/algo/graphlib.py pypy/branch/x86-64-jit-backend/pypy/tool/algo/test/autopath.py pypy/branch/x86-64-jit-backend/pypy/tool/autopath.py pypy/branch/x86-64-jit-backend/pypy/tool/compat.py pypy/branch/x86-64-jit-backend/pypy/tool/pytest/autopath.py pypy/branch/x86-64-jit-backend/pypy/tool/pytest/confpath.py pypy/branch/x86-64-jit-backend/pypy/tool/release/force-builds.py pypy/branch/x86-64-jit-backend/pypy/tool/release/make_release.py pypy/branch/x86-64-jit-backend/pypy/tool/release/package.py pypy/branch/x86-64-jit-backend/pypy/tool/release/test/test_package.py pypy/branch/x86-64-jit-backend/pypy/tool/stdlib___future__.py pypy/branch/x86-64-jit-backend/pypy/tool/stdlib_opcode.py pypy/branch/x86-64-jit-backend/pypy/tool/test/autopath.py pypy/branch/x86-64-jit-backend/pypy/tool/test/test_killsubprocess.py pypy/branch/x86-64-jit-backend/pypy/translator/autopath.py pypy/branch/x86-64-jit-backend/pypy/translator/backendopt/test/test_mallocprediction.py pypy/branch/x86-64-jit-backend/pypy/translator/benchmark/autopath.py pypy/branch/x86-64-jit-backend/pypy/translator/c/autopath.py pypy/branch/x86-64-jit-backend/pypy/translator/c/database.py pypy/branch/x86-64-jit-backend/pypy/translator/c/funcgen.py pypy/branch/x86-64-jit-backend/pypy/translator/c/gcc/autopath.py pypy/branch/x86-64-jit-backend/pypy/translator/c/gcc/trackgcroot.py pypy/branch/x86-64-jit-backend/pypy/translator/c/genc.py pypy/branch/x86-64-jit-backend/pypy/translator/c/src/support.h pypy/branch/x86-64-jit-backend/pypy/translator/c/test/autopath.py pypy/branch/x86-64-jit-backend/pypy/translator/c/test/test_typed.py pypy/branch/x86-64-jit-backend/pypy/translator/cli/gencli.py pypy/branch/x86-64-jit-backend/pypy/translator/cli/query.py pypy/branch/x86-64-jit-backend/pypy/translator/cli/rte.py pypy/branch/x86-64-jit-backend/pypy/translator/cli/support.py pypy/branch/x86-64-jit-backend/pypy/translator/cli/test/autopath.py pypy/branch/x86-64-jit-backend/pypy/translator/cli/test/runtest.py pypy/branch/x86-64-jit-backend/pypy/translator/driver.py pypy/branch/x86-64-jit-backend/pypy/translator/geninterplevel.py pypy/branch/x86-64-jit-backend/pypy/translator/goal/app_main.py pypy/branch/x86-64-jit-backend/pypy/translator/goal/autopath.py pypy/branch/x86-64-jit-backend/pypy/translator/goal/targetpypystandalone.py pypy/branch/x86-64-jit-backend/pypy/translator/goal/test2/autopath.py pypy/branch/x86-64-jit-backend/pypy/translator/goal/test2/test_app_main.py pypy/branch/x86-64-jit-backend/pypy/translator/jvm/genjvm.py pypy/branch/x86-64-jit-backend/pypy/translator/jvm/test/runtest.py pypy/branch/x86-64-jit-backend/pypy/translator/microbench/pybench/autopath.py pypy/branch/x86-64-jit-backend/pypy/translator/oosupport/function.py pypy/branch/x86-64-jit-backend/pypy/translator/platform/windows.py pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/autopath.py pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/pypy_interact.py pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/sandlib.py pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/test/autopath.py pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/test/test_pypy_interact.py pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/test/test_sandbox.py pypy/branch/x86-64-jit-backend/pypy/translator/test/autopath.py pypy/branch/x86-64-jit-backend/pypy/translator/tool/autopath.py pypy/branch/x86-64-jit-backend/pypy/translator/tool/lltracker.py Log: merged changes from trunk through r75772 Modified: pypy/branch/x86-64-jit-backend/LICENSE ============================================================================== --- pypy/branch/x86-64-jit-backend/LICENSE (original) +++ pypy/branch/x86-64-jit-backend/LICENSE Fri Jul 2 17:44:46 2010 @@ -152,3 +152,26 @@ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License for 'pypy/module/unicodedata/' +====================================== + +The following files are from the website of The Unicode Consortium +at http://www.unicode.org/. For the terms of use of these files, see +http://www.unicode.org/terms_of_use.html + + CompositionExclusions-3.2.0.txt + CompositionExclusions-4.1.0.txt + CompositionExclusions-5.0.0.txt + EastAsianWidth-3.2.0.txt + EastAsianWidth-4.1.0.txt + EastAsianWidth-5.0.0.txt + UnicodeData-3.2.0.txt + UnicodeData-4.1.0.txt + UnicodeData-5.0.0.txt + +The following files are derived from files from the above website. The same +terms of use apply. + UnihanNumeric-3.2.0.txt + UnihanNumeric-4.1.0.txt + UnihanNumeric-5.0.0.txt Modified: pypy/branch/x86-64-jit-backend/dotviewer/sshgraphserver.py ============================================================================== --- pypy/branch/x86-64-jit-backend/dotviewer/sshgraphserver.py (original) +++ pypy/branch/x86-64-jit-backend/dotviewer/sshgraphserver.py Fri Jul 2 17:44:46 2010 @@ -21,7 +21,7 @@ remoteport = random.randrange(10000, 20000) # ^^^ and just hope there is no conflict - args = ['ssh', '-C', '-R%d:127.0.0.1:%d' % (remoteport, localport)] + args = ['ssh', '-S', 'none', '-C', '-R%d:127.0.0.1:%d' % (remoteport, localport)] args = args + sshargs + ['python -u -c "exec input()"'] print ' '.join(args[:-1]) p = subprocess.Popen(args, bufsize=0, Modified: pypy/branch/x86-64-jit-backend/lib-python/conftest.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/conftest.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/conftest.py Fri Jul 2 17:44:46 2010 @@ -20,8 +20,8 @@ regrtestdir, modregrtestdir, testresultdir pytest_plugins = "resultlog", -rsyncdirs = ['.', '../pypy'] - +rsyncdirs = ['.', '../pypy/'] + # # Interfacing/Integrating with py.test's collection process # Modified: pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/cmd.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/cmd.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/cmd.py Fri Jul 2 17:44:46 2010 @@ -17,8 +17,9 @@ def _easy_install_get_site_dirs(): # return a list of 'site' dirs for easy_install from pkg_resources import normalize_path + from distutils.sysconfig import get_python_lib sitedirs = filter(None,os.environ.get('PYTHONPATH','').split(os.pathsep)) - sitedirs.append(os.path.join(sys.pypy_prefix, 'site-packages')) + sitedirs.append(get_python_lib(standard_lib=False)) sitedirs = map(normalize_path, sitedirs) return sitedirs Modified: pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/command/build_ext.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/command/build_ext.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/command/build_ext.py Fri Jul 2 17:44:46 2010 @@ -167,7 +167,7 @@ # for Release and Debug builds. # also Python's library directory must be appended to library_dirs if os.name == 'nt': - self.library_dirs.append(os.path.join(sys.pypy_prefix, 'pypy', '_interfaces')) + self.library_dirs.append(os.path.join(sys.prefix, 'include')) if self.debug: self.build_temp = os.path.join(self.build_temp, "Debug") else: Modified: pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/command/install.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/command/install.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/command/install.py Fri Jul 2 17:44:46 2010 @@ -283,6 +283,7 @@ 'dist_fullname': self.distribution.get_fullname(), 'py_version': py_version, 'py_version_short': py_version[0:3], + 'pypy_version_short': '%d.%d' % sys.pypy_version_info[:2], 'sys_prefix': prefix, 'prefix': prefix, 'sys_exec_prefix': exec_prefix, @@ -391,13 +392,9 @@ if self.exec_prefix is not None: raise DistutilsOptionError, \ "must not supply exec-prefix without prefix" - - if hasattr(sys, 'pypy_prefix'): - self.prefix = os.path.normpath(sys.pypy_prefix) - self.exec_prefix = self.prefix - else: - self.prefix = os.path.normpath(sys.prefix) - self.exec_prefix = os.path.normpath(sys.exec_prefix) + + self.prefix = os.path.normpath(sys.prefix) + self.exec_prefix = os.path.normpath(sys.exec_prefix) else: if self.exec_prefix is None: @@ -417,10 +414,7 @@ self.select_scheme("unix_home") else: if self.prefix is None: - if hasattr(sys, 'pypy_prefix'): - self.prefix = os.path.normpath(sys.pypy_prefix) - else: - self.prefix = os.path.normpath(sys.prefix) + self.prefix = os.path.normpath(sys.prefix) self.install_base = self.install_platbase = self.prefix try: @@ -434,7 +428,7 @@ def select_scheme (self, name): # it's the caller's problem if they supply a bad name! - if hasattr(sys, 'pypy_prefix'): + if hasattr(sys, 'pypy_version_info'): name = 'pypy' scheme = INSTALL_SCHEMES[name] for key in SCHEME_KEYS: Modified: pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/sysconfig.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/sysconfig.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/sysconfig.py Fri Jul 2 17:44:46 2010 @@ -19,5 +19,7 @@ if '__pypy__' in sys.builtin_module_names: from distutils.sysconfig_pypy import * + from distutils.sysconfig_pypy import _config_vars # needed by setuptools else: from distutils.sysconfig_cpython import * + from distutils.sysconfig_pypy import _config_vars # needed by setuptools Modified: pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Fri Jul 2 17:44:46 2010 @@ -7,15 +7,13 @@ from distutils.errors import DistutilsPlatformError -PYPY_PREFIX = os.path.normpath(sys.pypy_prefix) +PREFIX = os.path.normpath(sys.prefix) python_build = False def get_python_inc(plat_specific=0, prefix=None): from os.path import join as j - if plat_specific: - return j(sys.pypy_prefix, "pypy", "_interfaces") - return j(sys.pypy_prefix, 'pypy', 'module', 'cpyext', 'include') + return j(sys.prefix, 'include') def get_python_version(): """Return a string containing the major and minor Python version, @@ -43,8 +41,8 @@ raise DistutilsPlatformError( "calls to get_python_lib(standard_lib=1) cannot succeed") if prefix is None: - prefix = PYPY_PREFIX - return os.path.join(prefix, "site-packages") + prefix = PREFIX + return os.path.join(prefix, 'site-packages') _config_vars = None Modified: pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/site.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/site.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/site.py Fri Jul 2 17:44:46 2010 @@ -175,8 +175,9 @@ def addsitepackages(known_paths): """Add site-packages to sys.path, in a PyPy-specific way.""" - if hasattr(sys, 'pypy_prefix'): - sitedir = os.path.join(sys.pypy_prefix, "site-packages") + if hasattr(sys, 'pypy_version_info'): + from distutils.sysconfig import get_python_lib + sitedir = get_python_lib(standard_lib=False) if os.path.isdir(sitedir): addsitedir(sitedir, known_paths) return None Modified: pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/test/regrtest.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/test/regrtest.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/test/regrtest.py Fri Jul 2 17:44:46 2010 @@ -348,11 +348,8 @@ random.shuffle(tests) if trace: import trace - if hasattr(sys, 'prefix'): - ignoredirs = [sys.prefix, sys.exec_prefix] - else: - ignoredirs = [sys.pypy_prefix] # PyPy only - tracer = trace.Trace(ignoredirs=ignoredirs, trace=False, count=True) + tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix], + trace=False, count=True) test_support.verbose = verbose # Tell tests to be moderately quiet test_support.use_resources = use_resources save_modules = sys.modules.keys() Modified: pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/test/test_doctest.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/test/test_doctest.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/test/test_doctest.py Fri Jul 2 17:44:46 2010 @@ -2422,11 +2422,8 @@ import trace, sys, re, StringIO def test_coverage(coverdir): - if hasattr(sys, 'prefix'): - ignoredirs = [sys.prefix, sys.exec_prefix] - else: - ignoredirs = [sys.pypy_prefix] # PyPy only - tracer = trace.Trace(ignoredirs=ignoredirs, trace=0, count=1) + tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix,], + trace=0, count=1) tracer.run('reload(doctest); test_main()') r = tracer.results() print 'Writing coverage results...' Modified: pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/test/test_sys.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/test/test_sys.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/test/test_sys.py Fri Jul 2 17:44:46 2010 @@ -334,11 +334,8 @@ if test.test_support.have_unicode: self.assert_(isinstance(sys.maxunicode, int)) self.assert_(isinstance(sys.platform, basestring)) - if hasattr(sys, 'prefix'): - self.assert_(isinstance(sys.prefix, basestring)) - self.assert_(isinstance(sys.exec_prefix, basestring)) - else: - self.assert_(isinstance(sys.pypy_prefix, basestring)) # PyPy only + self.assert_(isinstance(sys.prefix, basestring)) + self.assert_(isinstance(sys.exec_prefix, basestring)) self.assert_(isinstance(sys.version, basestring)) vi = sys.version_info self.assert_(isinstance(vi, tuple)) Modified: pypy/branch/x86-64-jit-backend/py/_process/cmdexec.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_process/cmdexec.py (original) +++ pypy/branch/x86-64-jit-backend/py/_process/cmdexec.py Fri Jul 2 17:44:46 2010 @@ -8,18 +8,25 @@ from subprocess import Popen, PIPE def cmdexec(cmd): - """ return output of executing 'cmd' in a separate process. + """ return unicode output of executing 'cmd' in a separate process. raise cmdexec.ExecutionFailed exeception if the command failed. the exception will provide an 'err' attribute containing the error-output from the command. + if the subprocess module does not provide a proper encoding/unicode strings + sys.getdefaultencoding() will be used, if that does not exist, 'UTF-8'. """ process = subprocess.Popen(cmd, shell=True, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() - out = py.builtin._totext(out, sys.stdout.encoding) - err = py.builtin._totext(err, sys.stderr.encoding) + if sys.version_info[0] < 3: # on py3 we get unicode strings, on py2 not + try: + default_encoding = sys.getdefaultencoding() # jython may not have it + except AttributeError: + default_encoding = sys.stdout.encoding or 'UTF-8' + out = unicode(out, process.stdout.encoding or default_encoding) + err = unicode(err, process.stderr.encoding or default_encoding) status = process.poll() if status: raise ExecutionFailed(status, status, cmd, out, err) Modified: pypy/branch/x86-64-jit-backend/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/annotation/bookkeeper.py Fri Jul 2 17:44:46 2010 @@ -22,7 +22,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype from pypy.rpython import extregistry -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict class Stats: Modified: pypy/branch/x86-64-jit-backend/pypy/annotation/test/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/annotation/test/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/annotation/test/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/bin/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/bin/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/bin/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/bin/py.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/bin/py.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/bin/py.py Fri Jul 2 17:44:46 2010 @@ -11,6 +11,7 @@ except ImportError: pass +import pypy from pypy.tool import option from optparse import make_option from pypy.interpreter import main, interactive, error, gateway @@ -84,6 +85,14 @@ space.setitem(space.sys.w_dict, space.wrap('executable'), space.wrap(argv[0])) + # call pypy_initial_path: the side-effect is that it sets sys.prefix and + # sys.exec_prefix + srcdir = os.path.dirname(os.path.dirname(pypy.__file__)) + space.appexec([space.wrap(srcdir)], """(srcdir): + import sys + sys.pypy_initial_path(srcdir) + """) + # set warning control options (if any) warn_arg = interactiveconfig.warn if warn_arg is not None: Modified: pypy/branch/x86-64-jit-backend/pypy/config/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/config/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/config/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/config/pypyoption.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/config/pypyoption.py Fri Jul 2 17:44:46 2010 @@ -8,7 +8,8 @@ modulepath = py.path.local(__file__).dirpath().dirpath().join("module") all_modules = [p.basename for p in modulepath.listdir() if p.check(dir=True, dotfile=False) - and p.join('__init__.py').check()] + and p.join('__init__.py').check() + and not p.basename.startswith('test')] essential_modules = dict.fromkeys( ["exceptions", "_file", "sys", "__builtin__", "posix"] Modified: pypy/branch/x86-64-jit-backend/pypy/conftest.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/conftest.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/conftest.py Fri Jul 2 17:44:46 2010 @@ -10,11 +10,9 @@ # pytest settings pytest_plugins = "resultlog", -rsyncdirs = ['.', '../lib-python', '../demo'] +rsyncdirs = ['.', '../lib-python', '../lib_pypy', '../demo'] rsyncignore = ['_cache'] -collect_ignore = ['./lib/py'] - # PyPy's command line extra options (these are added # to py.test's standard options) # @@ -42,6 +40,13 @@ default="host", callback=_set_platform, help="set up tests to use specified platform as compile/run target") +def pytest_sessionstart(): + # have python subprocesses avoid startup customizations by default + try: + del os.environ['PYTHONSTARTUP'] + except KeyError: + pass + def pytest_funcarg__space(request): spaceconfig = getattr(request.cls, 'spaceconfig', {}) return gettestobjspace(**spaceconfig) @@ -220,8 +225,10 @@ def accept_regular_test(self): if option.runappdirect: - # only collect regular tests if we are in an 'app_test' directory - return "app_test" in self.listnames() + # only collect regular tests if we are in an 'app_test' directory, + # or in test_lib_pypy + names = self.listnames() + return "app_test" in names or "test_lib_pypy" in names else: return True @@ -232,7 +239,7 @@ return True return False - def classnamefilter(self, name): + def classnamefilter(self, name): if name.startswith('Test'): return self.accept_regular_test() if name.startswith('AppTest'): Modified: pypy/branch/x86-64-jit-backend/pypy/doc/_ref.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/_ref.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/_ref.txt Fri Jul 2 17:44:46 2010 @@ -46,11 +46,10 @@ .. _`pypy/interpreter/astcompiler/ast.py`: ../../pypy/interpreter/astcompiler/ast.py .. _`pypy/interpreter/typedef.py`: ../../pypy/interpreter/typedef.py .. _`lib/`: -.. _`pypy/lib/`: ../../pypy/lib -.. _`lib/app_test/`: ../../pypy/lib/app_test -.. _`lib/distributed/`: ../../pypy/lib/distributed -.. _`pypy/lib/stackless.py`: ../../pypy/lib/stackless.py -.. _`pypy/lib/test2`: ../../pypy/lib/test2 +.. _`lib_pypy/`: ../../lib_pypy +.. _`lib/distributed/`: ../../lib_pypy/distributed +.. _`lib_pypy/stackless.py`: ../../lib_pypy/stackless.py +.. _`lib_pypy/pypy_test/`: ../../lib_pypy/pypy_test .. _`module/`: .. _`pypy/module`: .. _`pypy/module/`: ../../pypy/module Modified: pypy/branch/x86-64-jit-backend/pypy/doc/coding-guide.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/coding-guide.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/coding-guide.txt Fri Jul 2 17:44:46 2010 @@ -631,7 +631,7 @@ because they rely on implementation details of CPython. If we don't just modify an original CPython module but need to rewrite -it from scratch we put it into `pypy/lib/`_ as a pure application level +it from scratch we put it into `lib_pypy/`_ as a pure application level module. When we need access to interpreter-level objects we put the module into @@ -653,7 +653,7 @@ >>>> import operator >>>> operator.__file__ - '/home/hpk/pypy-dist/pypy/lib/operator.py' + '/home/hpk/pypy-dist/lib_pypy/operator.py' >>>> import opcode >>>> opcode.__file__ @@ -682,7 +682,7 @@ list of directories, specified in the ``PYTHONPATH`` environment variable. -*pypy/lib/* +*lib_pypy/* contains pure Python reimplementation of modules. @@ -794,14 +794,14 @@ --withoutmod-mymodule (the later being the default)) for py.py and translate.py. -Testing modules in ``pypy/lib`` +Testing modules in ``lib_pypy/`` -------------------------------- -You can go to the `pypy/lib/test2`_ directory and invoke the testing tool -("py.test" or "python ../../test_all.py") to run tests against the -pypy/lib hierarchy. Note, that tests in `pypy/lib/test2`_ are allowed +You can go to the `lib_pypy/pypy_test/`_ directory and invoke the testing tool +("py.test" or "python ../../pypy/test_all.py") to run tests against the +lib_pypy hierarchy. Note, that tests in `lib_pypy/pypy_test/`_ are allowed and encouraged to let their tests run at interpreter level although -`pypy/lib/`_ modules eventually live at PyPy's application level. +`lib_pypy/`_ modules eventually live at PyPy's application level. This allows us to quickly test our python-coded reimplementations against CPython. Modified: pypy/branch/x86-64-jit-backend/pypy/doc/config/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/config/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/config/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/doc/config/objspace.usemodules.md5.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/config/objspace.usemodules.md5.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/config/objspace.usemodules.md5.txt Fri Jul 2 17:44:46 2010 @@ -1,5 +1,5 @@ Use the built-in 'md5' module. This module is expected to be working and is included by default. -There is also a pure Python version in pypy/lib which is used +There is also a pure Python version in lib_pypy which is used if the built-in is disabled, but it is several orders of magnitude slower. Modified: pypy/branch/x86-64-jit-backend/pypy/doc/config/objspace.usemodules.sha.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/config/objspace.usemodules.sha.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/config/objspace.usemodules.sha.txt Fri Jul 2 17:44:46 2010 @@ -1,5 +1,5 @@ Use the built-in 'sha' module. This module is expected to be working and is included by default. -There is also a pure Python version in pypy/lib which is used +There is also a pure Python version in lib_pypy which is used if the built-in is disabled, but it is several orders of magnitude slower. Modified: pypy/branch/x86-64-jit-backend/pypy/doc/config/objspace.usemodules.struct.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/config/objspace.usemodules.struct.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/config/objspace.usemodules.struct.txt Fri Jul 2 17:44:46 2010 @@ -1,5 +1,5 @@ Use the built-in 'struct' module. This module is expected to be working and is included by default. -There is also a pure Python version in pypy/lib which is used +There is also a pure Python version in lib_pypy which is used if the built-in is disabled, but it is several orders of magnitude slower. Modified: pypy/branch/x86-64-jit-backend/pypy/doc/cpython_differences.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/cpython_differences.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/cpython_differences.txt Fri Jul 2 17:44:46 2010 @@ -71,7 +71,7 @@ _stackless * Supported by being rewritten in pure Python (possibly using ``ctypes``): - see the `pypy/lib/`_ directory. Examples of modules that we + see the `lib_pypy/`_ directory. Examples of modules that we support this way: ``ctypes``, ``array``, ``cPickle``, ``cStringIO``, ``cmath``, ``dbm`` (?), ``datetime``, ``binascii``... Note that some modules are both in there and in the list above; @@ -79,7 +79,7 @@ at translation time). The extension modules (i.e. modules written in C, in the standard CPython) -that are neither mentioned above nor in `pypy/lib/`_ are not available in PyPy. +that are neither mentioned above nor in `lib_pypy/`_ are not available in PyPy. .. the nonstandard modules are listed below... .. _`__pypy__`: __pypy__-module.html Modified: pypy/branch/x86-64-jit-backend/pypy/doc/ctypes-implementation.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/ctypes-implementation.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/ctypes-implementation.txt Fri Jul 2 17:44:46 2010 @@ -48,9 +48,9 @@ High-level parts ================= -The reused ``ctypes`` package lives in ``pypy/lib/ctypes``. ``_ctypes`` +The reused ``ctypes`` package lives in ``lib_pypy/ctypes``. ``_ctypes`` implementing the same interface as ``_ctypes`` in CPython is in -``pypy/lib/_ctypes``. +``lib_pypy/_ctypes``. Discussion and limitations ============================= @@ -123,8 +123,8 @@ To run the tests then:: - $ cd ../.. # back to pypy/ - $ ./translator/goal/pypy-c test_all.py lib/app_test/ctypes_tests + $ cd ../../.. # back to pypy-trunk + $ ./pypy/translator/goal/pypy-c pypy/test_all.py lib/pypy1.2/lib_pypy/pypy_test/ctypes_tests There should be 36 skipped tests and all other tests should pass. Modified: pypy/branch/x86-64-jit-backend/pypy/doc/docindex.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/docindex.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/docindex.txt Fri Jul 2 17:44:46 2010 @@ -207,13 +207,6 @@ `interpreter/astcompiler/`_ interpreter-level bytecode compiler, via an AST representation -`lib/`_ PyPy's wholesale reimplementations of CPython modules_ - and experimental new application-level modules - -`lib/app_test/`_ tests for the reimplementations, running on top of CPython - -`lib/distributed/`_ distributed execution prototype, based on `transparent proxies`_ - `module/`_ contains `mixed modules`_ implementing core modules with both application and interpreter level code. Not all are finished and working. Use the ``--withmod-xxx`` Modified: pypy/branch/x86-64-jit-backend/pypy/doc/getting-started-python.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/getting-started-python.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/getting-started-python.txt Fri Jul 2 17:44:46 2010 @@ -207,18 +207,18 @@ For installation purposes, note that the executable needs to be able to find its version of the Python standard library in the following three directories: ``lib-python/2.5.2``, ``lib-python/modified-2.5.2`` and -``pypy/lib``. They are located by "looking around" starting from the +``lib_pypy``. They are located by "looking around" starting from the directory in which the executable resides. The current logic is to try to find a ``PREFIX`` from which the directories ``PREFIX/lib-python/2.5.2`` and ``PREFIX/lib-python/modified.2.5.2`` and -``PREFIX/pypy/lib`` can all be found. The prefixes that are tried are:: +``PREFIX/lib_pypy`` can all be found. The prefixes that are tried are:: . - ./share/pypy-1.2 + ./lib/pypy1.2 .. - ../share/pypy-1.2 + ../lib/pypy1.2 ../.. - ../../share/pypy-1.2 + ../../lib/pypy-1.2 ../../.. etc. Modified: pypy/branch/x86-64-jit-backend/pypy/doc/index.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/index.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/index.txt Fri Jul 2 17:44:46 2010 @@ -8,7 +8,7 @@ Getting into PyPy ... ============================================= -* `Release 1.2`_: the latest official release +* `Release 1.3`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -56,4 +56,4 @@ .. _`Documentation`: docindex.html .. _`Getting Started`: getting-started.html .. _papers: extradoc.html -.. _`Release 1.2`: release-1.2.0.html +.. _`Release 1.3`: http://pypy.org/download.html Modified: pypy/branch/x86-64-jit-backend/pypy/doc/jit/pyjitpl5.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/jit/pyjitpl5.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/jit/pyjitpl5.txt Fri Jul 2 17:44:46 2010 @@ -8,15 +8,15 @@ Implementation of the JIT ========================= -The JIT's `theory`_ is great in principle, but actual code is a different +The JIT's `theory`_ is great in principle, but the actual code is a different story. This section tries to give a high level overview of how PyPy's JIT is -implemented. It's helpful to have a basic understanding of how the PyPy -`translation tool chain`_ works before digging into the sources. +implemented. It's helpful to have an understanding of how the PyPy `translation +tool chain`_ works before digging into the sources. -Almost all JIT specific code is found in the two pypy/jit subdirectories, -metainterp, and backend. The metainterp directory holds platform independent -code including the translation generator, the tracer, and the optimizer. Code -in the backend directory is responsible for generating machine code. +Almost all JIT specific code is found in pypy/jit subdirectories. Translation +time code is in the codewriter directory. The metainterp directory holds +platform independent code including the the tracer and the optimizer. Code in +the backend directory is responsible for generating machine code. .. _`theory`: overview.html .. _`translation tool chain`: ../translation.html @@ -34,7 +34,7 @@ interpreter defines its hints in pypy/module/pypyjit/interp_jit.py in a few overriden methods of the defaut interpreter loop. -The interpreter wishing to use the PyPy's JIT must define a list of *green* +An interpreter wishing to use the PyPy's JIT must define a list of *green* variables and a list of *red* variables. The *green* variables are loop constants. They are used to identify the current loop. Red variables are for everything else used in the execution loop. For example, the Python interpreter @@ -58,11 +58,11 @@ is replaced with a call to a function, maybe_compile_and_run in warmstate.py, that checks if current loop is "hot" and should be compiled. -Next, starting with the portal graph, metainterp/codewriter.py converts the -graphs of the interpreter into JIT bytecode. Since this bytecode is stored in -the final binary, it's designed to be concise rather than fast. The bytecode -codewriter doesn't "see" (what it sees is defined by the JIT's policy) every -part of the interpreter. In these cases, it simply inserts an opaque call. +Next, starting with the portal graph, codewriter/\*.py converts the graphs of the +interpreter into JIT bytecode. Since this bytecode is stored in the final +binary, it's designed to be concise rather than fast. The bytecode codewriter +doesn't "see" (what it sees is defined by the JIT's policy) every part of the +interpreter. In these cases, it simply inserts an opaque call. Finally, translation finishes, including the bytecode of the interpreter in the final binary, and interpreter is ready to use the runtime component of the JIT. Modified: pypy/branch/x86-64-jit-backend/pypy/doc/maemo.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/maemo.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/maemo.txt Fri Jul 2 17:44:46 2010 @@ -157,11 +157,11 @@ You can just copy your whole pypy-trunk directory over to your mobile device - however, only these should be needed:: - pypy/lib - lib-python + lib/pypy1.2/lib_pypy + lib/pypy1.2/lib-python pypy/translator/goal/pypy-c -It is neccessary that the ``pypy-c`` can find a "lib-python" and "pypy/lib" directory +It is neccessary that the ``pypy-c`` can find a "lib-python" and "lib_pypy" directory if you want to successfully startup the interpreter on the device. Start ``pypy-c`` on the device. If you see an error like "setupterm: could not find terminal" Modified: pypy/branch/x86-64-jit-backend/pypy/doc/objspace-proxies.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/objspace-proxies.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/objspace-proxies.txt Fri Jul 2 17:44:46 2010 @@ -610,7 +610,7 @@ .. _`proxy_helpers.py`: ../../pypy/objspace/std/proxy_helpers.py .. _`proxyobject.py`: ../../pypy/objspace/std/proxyobject.py .. _`transparent.py`: ../../pypy/objspace/std/transparent.py -.. _`tputil.py`: ../../pypy/lib/tputil.py +.. _`tputil.py`: ../../lib_pypy/tputil.py .. [D12.1] `High-Level Backends and Interpreter Feature Prototypes`, PyPy EU-Report, 2007, http://codespeak.net/pypy/extradoc/eu-report/D12.1_H-L-Backends_and_Feature_Prototypes-2007-03-22.pdf Modified: pypy/branch/x86-64-jit-backend/pypy/doc/release-1.3.0.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/release-1.3.0.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/release-1.3.0.txt Fri Jul 2 17:44:46 2010 @@ -5,20 +5,40 @@ Hello. We're please to announce release of PyPy 1.3. This release has two major -points. First of all, we stabilized the JIT compiler since 1.2 release, -answered user issues, fixed bugs and generally improved speed. +improvements. First of all, we stabilized the JIT compiler since 1.2 release, +answered user issues, fixed bugs, and generally improved speed. -The other point is that we're pleased to announce alpha support for loading -CPython extension modules written in C. While the main point of the release -is increased stability, this part is in alpha stage and it is not yet suited -for production environments. - -Highlight of this release -========================= - -XXX a rough list -- cpyext (some details, in particular: the extension module must be - recompiled, and typically "bug-fixed" w.r.t. refcounting) -- bugfixes -- blackhole-improvements -- minor jit improvements +We're also pleased to announce alpha support for loading CPython extension +modules written in C. While the main purpose of this release is increased +stability, this feature is in alpha stage and it is not yet suited for +production environments. + +Highlights of this release +========================== + +* We introduced support for CPython extension modules written in C. As of now, + this support is in alpha, and it's very unlikely unaltered C extensions will + work out of the box, due to missing functions or refcounting details. The + support is disable by default, so you have to do:: + + import cpyext + + before trying to import any .so file. Also, libraries are source-compatible + and not binary-compatible. That means you need to recompile binaries, using + for example:: + + python setup.py build + + Details may vary, depending on your build system. Make sure you include + the above line at the beginning of setup.py or put it in your PYTHONSTARTUP. + + This is alpha feature. It'll likely segfault. You have been warned! + +* JIT bugfixes. A lot of bugs reported for the JIT have been fixed, and its + stability greatly improved since 1.2 release. + +* Various small improvements have been added to the JIT code, as well as a great + speedup of compiling time. + +Cheers, +Maciej Fijalkowski, Armin Rigo, Alex Gaynor, Amaury Forgeot d'Arc and the PyPy team Modified: pypy/branch/x86-64-jit-backend/pypy/doc/sandbox.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/sandbox.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/sandbox.txt Fri Jul 2 17:44:46 2010 @@ -72,7 +72,8 @@ should do. I've successfully tried it on the JS interpreter. The controller is only called "pypy_interact" because it emulates a file hierarchy that makes pypy-c-sandbox happy - it contains (read-only) -virtual directories like /bin/lib-python and /bin/pypy/lib and it +virtual directories like /bin/lib/pypy1.2/lib-python and +/bin/lib/pypy1.2/lib_pypy and it pretends that the executable is /bin/pypy-c. Howto Modified: pypy/branch/x86-64-jit-backend/pypy/doc/stackless.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/stackless.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/stackless.txt Fri Jul 2 17:44:46 2010 @@ -124,10 +124,17 @@ * ``coro.kill()`` - Kill ``coro`` by sending an exception to it. (At the moment, the - exception is not visible to app-level, which means that you cannot - catch it, and that ``try: finally:`` clauses are not honored. This - will be fixed in the future.) + Kill ``coro`` by sending a CoroutineExit exception and switching + execution immediately to it. This exception can be caught in the + coroutine itself and can be raised from any call to ``coro.switch()``. + This exception isn't propagated to the parent coroutine. + +* ``coro.throw(type, value)`` + + Insert an exception in ``coro`` an resume switches execution + immediately to it. In the coroutine itself, this exception + will come from any call to ``coro.switch()`` and can be caught. If the + exception isn't caught, it will be propagated to the parent coroutine. Example ~~~~~~~ @@ -181,7 +188,7 @@ website. Note that Tasklets and Channels are implemented at application-level in -`pypy/lib/stackless.py`_ on top of coroutines_. You can refer to this +`lib_pypy/stackless.py`_ on top of coroutines_. You can refer to this module for more details and API documentation. The stackless.py code tries to resemble the stackless C code as much Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/baseobjspace.py Fri Jul 2 17:44:46 2010 @@ -385,6 +385,7 @@ def get_builtinmodule_to_install(self): """NOT_RPYTHON""" + from pypy.tool.lib_pypy import LIB_PYPY try: return self._builtinmodule_list except AttributeError: @@ -404,9 +405,7 @@ if not self.config.objspace.nofaking: for modname in self.ALL_BUILTIN_MODULES: - if not (os.path.exists( - os.path.join(os.path.dirname(pypy.__file__), - 'lib', modname+'.py'))): + if not LIB_PYPY.join(modname+'.py').check(file=True): modules.append('faked+'+modname) self._builtinmodule_list = modules Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/pyparser/parsestring.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/pyparser/parsestring.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/pyparser/parsestring.py Fri Jul 2 17:44:46 2010 @@ -72,12 +72,12 @@ bufp = 0 bufq = len(buf) assert 0 <= bufp <= bufq - w_substr = space.wrap(buf[bufp : bufq]) + substr = buf[bufp:bufq] if rawmode: - w_v = unicodehelper.PyUnicode_DecodeRawUnicodeEscape(space, w_substr) + v = unicodehelper.PyUnicode_DecodeRawUnicodeEscape(space, substr) else: - w_v = unicodehelper.PyUnicode_DecodeUnicodeEscape(space, w_substr) - return w_v + v = unicodehelper.PyUnicode_DecodeUnicodeEscape(space, substr) + return space.wrap(v) need_encoding = (encoding is not None and encoding != "utf-8" and encoding != "iso-8859-1") @@ -86,7 +86,7 @@ substr = s[ps : q] if rawmode or '\\' not in s[ps:]: if need_encoding: - w_u = unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(substr)) + w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, substr)) #w_v = space.wrap(space.unwrap(w_u).encode(encoding)) this works w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding)) return w_v @@ -96,7 +96,7 @@ enc = None if need_encoding: enc = encoding - v = PyString_DecodeEscape(space, substr, unicode, enc) + v = PyString_DecodeEscape(space, substr, enc) return space.wrap(v) def hexbyte(val): @@ -105,10 +105,9 @@ result = "0" + result return result -def PyString_DecodeEscape(space, s, unicode, recode_encoding): +def PyString_DecodeEscape(space, s, recode_encoding): """ - Unescape a backslash-escaped string. If unicode is non-zero, - the string is a u-literal. If recode_encoding is non-zero, + Unescape a backslash-escaped string. If recode_encoding is non-zero, the string is UTF-8 encoded and should be re-encoded in the specified encoding. """ @@ -161,19 +160,22 @@ span = ps span += (span < end) and (s[span] in '01234567') span += (span < end) and (s[span] in '01234567') - lis.append(chr(int(s[prevps : span], 8))) + octal = s[prevps : span] + # emulate a strange wrap-around behavior of CPython: + # \400 is the same as \000 because 0400 == 256 + num = int(octal, 8) & 0xFF + lis.append(chr(num)) ps = span elif ch == 'x': if ps+2 <= end and isxdigit(s[ps]) and isxdigit(s[ps + 1]): - lis.append(chr(int(s[ps : ps + 2], 16))) + hexa = s[ps : ps + 2] + num = int(hexa, 16) + lis.append(chr(num)) ps += 2 else: raise_app_valueerror(space, 'invalid \\x escape') # ignored replace and ignore for now - elif unicode and (ch == 'u' or ch == 'U' or ch == 'N'): - raise_app_valueerror(space, 'Unicode escapes not legal ' - 'when Unicode disabled') else: # this was not an escape, so the backslash # has to be added, and we start over in @@ -200,7 +202,7 @@ # while (s < end && *s != '\\') s++; */ /* inefficient for u".." while ps < end and ord(s[ps]) & 0x80: ps += 1 - w_u = unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(s[pt : ps])) + w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, s[pt:ps])) w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding)) v = space.str_w(w_v) return v, ps Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/pyparser/test/test_parsestring.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/pyparser/test/test_parsestring.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/pyparser/test/test_parsestring.py Fri Jul 2 17:44:46 2010 @@ -2,75 +2,58 @@ import py class TestParsetring: + def parse_and_compare(self, literal, value): + space = self.space + w_ret = parsestring.parsestr(space, None, literal) + if isinstance(value, str): + assert space.type(w_ret) == space.w_str + assert space.str_w(w_ret) == value + elif isinstance(value, unicode): + assert space.type(w_ret) == space.w_unicode + assert space.unicode_w(w_ret) == value + else: + assert False + def test_simple(self): space = self.space - s = 'hello world' - w_ret = parsestring.parsestr(space, None, repr(s)) - assert space.str_w(w_ret) == s - s = 'hello\n world' - w_ret = parsestring.parsestr(space, None, repr(s)) - assert space.str_w(w_ret) == s - s = "'''hello\\x42 world'''" - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == 'hello\x42 world' - s = r'"\0"' - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == chr(0) - s = r'"\07"' - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == chr(7) - s = r'"\123"' - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == chr(0123) - s = r'"\x"' - space.raises_w(space.w_ValueError, parsestring.parsestr, space, None, s) - s = r'"\x7"' - space.raises_w(space.w_ValueError, parsestring.parsestr, space, None, s) - s = r'"\x7g"' - space.raises_w(space.w_ValueError, parsestring.parsestr, space, None, s) - s = r'"\xfF"' - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == chr(0xFF) + for s in ['hello world', 'hello\n world']: + self.parse_and_compare(repr(s), s) + + self.parse_and_compare("'''hello\\x42 world'''", 'hello\x42 world') + + # octal + self.parse_and_compare(r'"\0"', chr(0)) + self.parse_and_compare(r'"\07"', chr(7)) + self.parse_and_compare(r'"\123"', chr(0123)) + self.parse_and_compare(r'"\400"', chr(0)) + self.parse_and_compare(r'"\9"', '\\' + '9') + self.parse_and_compare(r'"\08"', chr(0) + '8') + + # hexadecimal + self.parse_and_compare(r'"\xfF"', chr(0xFF)) + self.parse_and_compare(r'"\""', '"') + self.parse_and_compare(r"'\''", "'") + for s in (r'"\x"', r'"\x7"', r'"\x7g"'): + space.raises_w(space.w_ValueError, + parsestring.parsestr, space, None, s) - s = r'"\""' - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == '"' - - s = r"'\''" - w_ret = parsestring.parsestr(space, None, s) - assert space.str_w(w_ret) == "'" - - def test_unicode(self): space = self.space - s = u'hello world' - w_ret = parsestring.parsestr(space, None, repr(s)) - ret = space.unwrap(w_ret) - assert isinstance(ret, unicode) - assert ret == s - s = u'hello\n world' - w_ret = parsestring.parsestr(self.space, None, repr(s)) - ret = space.unwrap(w_ret) - assert isinstance(ret, unicode) - assert ret == s - s = "u'''hello\\x42 world'''" - w_ret = parsestring.parsestr(self.space, None, s) - ret = space.unwrap(w_ret) - assert isinstance(ret, unicode) - assert ret == u'hello\x42 world' - s = "u'''hello\\u0842 world'''" - w_ret = parsestring.parsestr(self.space, None, s) - ret = space.unwrap(w_ret) - assert isinstance(ret, unicode) - assert ret == u'hello\u0842 world' + for s in [u'hello world', u'hello\n world']: + self.parse_and_compare(repr(s), s) + + self.parse_and_compare("u'''hello\\x42 world'''", + u'hello\x42 world') + self.parse_and_compare("u'''hello\\u0842 world'''", + u'hello\u0842 world') + s = "u'\x81'" s = s.decode("koi8-u").encode("utf8") w_ret = parsestring.parsestr(self.space, 'koi8-u', s) ret = space.unwrap(w_ret) - assert ret == eval("# -*- coding: koi8-u -*-\nu'\x81'") + assert ret == eval("# -*- coding: koi8-u -*-\nu'\x81'") def test_simple_enc_roundtrip(self): - #py.test.skip("crashes in app_codecs, but when cheating using .encode at interp-level passes?!") space = self.space s = "'\x81'" s = s.decode("koi8-u").encode("utf8") Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_module.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_module.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_module.py Fri Jul 2 17:44:46 2010 @@ -54,11 +54,11 @@ r = repr(sys) assert r == "" - import _pypy_interact # known to be in pypy/lib + import _pypy_interact # known to be in lib_pypy r = repr(_pypy_interact) assert (r.startswith("')) nofile = type(_pypy_interact)('nofile', 'foo') assert repr(nofile) == "" Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_objspace.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_objspace.py Fri Jul 2 17:44:46 2010 @@ -259,3 +259,23 @@ def test_sys_import(self): from pypy.interpreter.main import run_string run_string('import sys', space=self.space) + + def test_get_builtinmodule_to_install(self): + space = self.space + try: + # force rebuilding with this fake builtin + space.ALL_BUILTIN_MODULES.append('this_doesnt_exist') + del space._builtinmodule_list + mods = space.get_builtinmodule_to_install() + + assert '__pypy__' in mods # real builtin + assert 'array' not in mods # in lib_pypy + assert 'faked+array' not in mods # in lib_pypy + assert 'this_doesnt_exist' not in mods # not in lib_pypy + assert 'faked+this_doesnt_exist' in mods # not in lib_pypy, but in + # ALL_BUILTIN_MODULES + finally: + # rebuild the original list + space.ALL_BUILTIN_MODULES.pop() + del space._builtinmodule_list + mods = space.get_builtinmodule_to_install() Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/unicodehelper.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/unicodehelper.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/unicodehelper.py Fri Jul 2 17:44:46 2010 @@ -1,30 +1,10 @@ -from pypy.interpreter import gateway +from pypy.module._codecs import interp_codecs -app = gateway.applevel(r''' - def PyUnicode_DecodeUnicodeEscape(data): - import _codecs - return _codecs.unicode_escape_decode(data)[0] +def PyUnicode_AsEncodedString(space, w_data, w_encoding): + return interp_codecs.encode(space, w_data, w_encoding) - def PyUnicode_DecodeRawUnicodeEscape(data): - import _codecs - return _codecs.raw_unicode_escape_decode(data)[0] - - def PyUnicode_DecodeUTF8(data): - import _codecs - return _codecs.utf_8_decode(data)[0] - - def PyUnicode_AsEncodedString(data, encoding): - import _codecs - return _codecs.encode(data, encoding) - - def PyUnicode_EncodeUTF8(data): - import _codecs - return _codecs.utf_8_encode(data)[0] - -''') - -PyUnicode_DecodeUnicodeEscape = app.interphook('PyUnicode_DecodeUnicodeEscape') -PyUnicode_DecodeRawUnicodeEscape = app.interphook('PyUnicode_DecodeRawUnicodeEscape') -PyUnicode_DecodeUTF8 = app.interphook('PyUnicode_DecodeUTF8') -PyUnicode_AsEncodedString = app.interphook('PyUnicode_AsEncodedString') -PyUnicode_EncodeUTF8 = app.interphook('PyUnicode_EncodeUTF8') +# These functions take and return unwrapped rpython strings and unicodes +PyUnicode_DecodeUnicodeEscape = interp_codecs.make_raw_decoder('unicode_escape') +PyUnicode_DecodeRawUnicodeEscape = interp_codecs.make_raw_decoder('raw_unicode_escape') +PyUnicode_DecodeUTF8 = interp_codecs.make_raw_decoder('utf_8') +PyUnicode_EncodeUTF8 = interp_codecs.make_raw_encoder('utf_8') Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/llgraph/llimpl.py Fri Jul 2 17:44:46 2010 @@ -832,13 +832,16 @@ raise Exception("Nonsense type %s" % TYPE) failindex = self.cpu._execute_token(loop_token) + jd = loop_token.outermost_jitdriver_sd + assert jd is not None, ("call_assembler(): the loop_token needs " + "to have 'outermost_jitdriver_sd'") + if jd.index_of_virtualizable != -1: + vable = args[jd.index_of_virtualizable] + else: + vable = lltype.nullptr(llmemory.GCREF.TO) + assembler_helper_ptr = jd.assembler_helper_adr.ptr # fish try: - if self.cpu.index_of_virtualizable != -1: - return self.cpu.assembler_helper_ptr(failindex, - args[self.cpu.index_of_virtualizable]) - else: - return self.cpu.assembler_helper_ptr(failindex, - lltype.nullptr(llmemory.GCREF.TO)) + return assembler_helper_ptr(failindex, vable) except LLException, lle: assert _last_exception is None, "exception left behind" _last_exception = lle Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/model.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/model.py Fri Jul 2 17:44:46 2010 @@ -3,10 +3,6 @@ class AbstractCPU(object): supports_floats = False - _got_exception = None - # assembler_helper_ptr - a pointer to helper to call after a direct - # assembler call - portal_calldescr = None done_with_this_frame_void_v = -1 done_with_this_frame_int_v = -1 done_with_this_frame_ref_v = -1 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py Fri Jul 2 17:44:46 2010 @@ -1717,16 +1717,22 @@ assert self.cpu.get_latest_value_int(0) == 10 called.append(failindex) return 4 + 9 - self.cpu.index_of_virtualizable = -1 - self.cpu.assembler_helper_ptr = llhelper(lltype.Ptr(lltype.FuncType - ([lltype.Signed, llmemory.GCREF], lltype.Signed)), assembler_helper) - + + FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF], + lltype.Signed)) + class FakeJitDriverSD: + index_of_virtualizable = -1 + _assembler_helper_ptr = llhelper(FUNCPTR, assembler_helper) + assembler_helper_adr = llmemory.cast_ptr_to_adr( + _assembler_helper_ptr) + ops = ''' [i0, i1] i2 = int_add(i0, i1) finish(i2)''' loop = parse(ops) looptoken = LoopToken() + looptoken.outermost_jitdriver_sd = FakeJitDriverSD() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) ARGS = [lltype.Signed, lltype.Signed] RES = lltype.Signed @@ -1758,9 +1764,15 @@ assert self.cpu.get_latest_value_float(0) == 1.2 + 3.2 called.append(failindex) return 13.5 - self.cpu.index_of_virtualizable = -1 - self.cpu.assembler_helper_ptr = llhelper(lltype.Ptr(lltype.FuncType - ([lltype.Signed, llmemory.GCREF], lltype.Float)), assembler_helper) + + FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF], + lltype.Float)) + class FakeJitDriverSD: + index_of_virtualizable = -1 + _assembler_helper_ptr = llhelper(FUNCPTR, assembler_helper) + assembler_helper_adr = llmemory.cast_ptr_to_adr( + _assembler_helper_ptr) + ARGS = [lltype.Float, lltype.Float] RES = lltype.Float self.cpu.portal_calldescr = self.cpu.calldescrof( @@ -1772,6 +1784,7 @@ finish(f2)''' loop = parse(ops) looptoken = LoopToken() + looptoken.outermost_jitdriver_sd = FakeJitDriverSD() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) self.cpu.set_future_value_float(0, 1.2) self.cpu.set_future_value_float(1, 2.3) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/support.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/support.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/support.py Fri Jul 2 17:44:46 2010 @@ -62,11 +62,12 @@ warmrunnerdesc = WarmRunnerDesc(t, translate_support_code=True, CPUClass=self.CPUClass, **kwds) - warmrunnerdesc.state.set_param_threshold(3) # for tests - warmrunnerdesc.state.set_param_trace_eagerness(2) # for tests - warmrunnerdesc.state.set_param_trace_limit(trace_limit) - warmrunnerdesc.state.set_param_inlining(inline) - warmrunnerdesc.state.set_param_optimizer(OPTIMIZER_FULL) + for jd in warmrunnerdesc.jitdrivers_sd: + jd.warmstate.set_param_threshold(3) # for tests + jd.warmstate.set_param_trace_eagerness(2) # for tests + jd.warmstate.set_param_trace_limit(trace_limit) + jd.warmstate.set_param_inlining(inline) + jd.warmstate.set_param_optimizer(OPTIMIZER_FULL) mixlevelann = warmrunnerdesc.annhelper entry_point_graph = mixlevelann.getgraph(entry_point, [s_list_of_strings], annmodel.SomeInteger()) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Fri Jul 2 17:44:46 2010 @@ -156,7 +156,6 @@ self.malloc_array_func_addr = 0 self.malloc_str_func_addr = 0 self.malloc_unicode_func_addr = 0 - self.assembler_helper_adr = 0 self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) @@ -192,14 +191,6 @@ ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, ll_new_unicode) - if we_are_translated(): - self.assembler_helper_adr = self.cpu.cast_ptr_to_int( - self.cpu.assembler_helper_ptr) - else: - if getattr(self.cpu, 'assembler_helper_ptr', None): - self.assembler_helper_adr = self.cpu.cast_ptr_to_int( - self.cpu.assembler_helper_ptr) - self.mc = MachineCodeBlockWrapper(self, self.mc_size, self.cpu.profile_agent) self._build_failure_recovery(False) self._build_failure_recovery(True) @@ -1575,7 +1566,10 @@ je_location = self.mc.get_relative_pos() # # Path A: use assembler_helper_adr - self._emit_call(imm(self.assembler_helper_adr), [eax, arglocs[1]], 0, + jd = descr.outermost_jitdriver_sd + assert jd is not None + asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) + self._emit_call(imm(asm_helper_adr), [eax, arglocs[1]], 0, tmp=ecx) if IS_X86_32 and isinstance(result_loc, StackLoc) and result_loc.type == FLOAT: self.mc.FSTP_b(result_loc.value) @@ -1589,9 +1583,9 @@ self.mc.overwrite(je_location - 1, [chr(offset)]) # # Reset the vable token --- XXX really too much special logic here:-( - if self.cpu.index_of_virtualizable >= 0: + if jd.index_of_virtualizable >= 0: from pypy.jit.backend.llsupport.descr import BaseFieldDescr - fielddescr = self.cpu.vable_token_descr + fielddescr = jd.vable_token_descr assert isinstance(fielddescr, BaseFieldDescr) ofs = fielddescr.offset self.mc.MOV(eax, arglocs[1]) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py Fri Jul 2 17:44:46 2010 @@ -650,10 +650,14 @@ self._consider_call(op, guard_op) def consider_call_assembler(self, op, guard_op): - descr = op.descr portal_calldescr = self.assembler.cpu.portal_calldescr size = portal_calldescr.get_result_size(self.translate_support_code) - vable_index = self.assembler.cpu.index_of_virtualizable + # + descr = op.descr + assert isinstance(descr, LoopToken) + jd = descr.outermost_jitdriver_sd + assert jd is not None + vable_index = jd.index_of_virtualizable if vable_index >= 0: self.rm._sync_var(op.args[vable_index]) vable = self.fm.loc(op.args[vable_index]) @@ -793,12 +797,14 @@ arglocs.append(self.loc(op.args[0])) return self._call(op, arglocs) # boehm GC (XXX kill the following code at some point) - scale_of_field, basesize, _ = self._unpack_arraydescr(op.descr) - return self._malloc_varsize(basesize, 0, scale_of_field, op.args[0], - op.result) + scale_of_field, basesize, ofs_length, _ = ( + self._unpack_arraydescr(op.descr)) + return self._malloc_varsize(basesize, ofs_length, scale_of_field, + op.args[0], op.result) def _unpack_arraydescr(self, arraydescr): assert isinstance(arraydescr, BaseArrayDescr) + ofs_length = arraydescr.get_ofs_length(self.translate_support_code) ofs = arraydescr.get_base_size(self.translate_support_code) size = arraydescr.get_item_size(self.translate_support_code) ptr = arraydescr.is_array_of_pointers() @@ -806,7 +812,7 @@ while (1 << scale) < size: scale += 1 assert (1 << scale) == size - return scale, ofs, ptr + return scale, ofs, ofs_length, ptr def _unpack_fielddescr(self, fielddescr): assert isinstance(fielddescr, BaseFieldDescr) @@ -841,7 +847,7 @@ consider_unicodesetitem = consider_strsetitem def consider_setarrayitem_gc(self, op): - scale, ofs, ptr = self._unpack_arraydescr(op.descr) + scale, ofs, _, ptr = self._unpack_arraydescr(op.descr) base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) if scale == 0: need_lower_byte = True @@ -868,7 +874,7 @@ consider_getfield_gc_pure = consider_getfield_gc def consider_getarrayitem_gc(self, op): - scale, ofs, _ = self._unpack_arraydescr(op.descr) + scale, ofs, _, _ = self._unpack_arraydescr(op.descr) base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) ofs_loc = self.rm.make_sure_var_in_reg(op.args[1], op.args) self.rm.possibly_free_vars(op.args) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_zrpy_gc.py Fri Jul 2 17:44:46 2010 @@ -105,6 +105,12 @@ def test_compile_boehm(): myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) + @dont_look_inside + def see(lst, n): + assert len(lst) == 3 + assert lst[0] == n+10 + assert lst[1] == n+20 + assert lst[2] == n+30 def main(n, x): while n > 0: myjitdriver.can_enter_jit(n=n, x=x) @@ -112,6 +118,7 @@ y = X() y.foo = x.foo n -= y.foo + see([n+10, n+20, n+30], n) res = compile_and_run(get_entry(get_g(main)), "boehm", jit=True) assert int(res) >= 16 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/assembler.py Fri Jul 2 17:44:46 2010 @@ -60,37 +60,38 @@ self.code.append(chr(reg.index)) def emit_const(self, const, kind, allow_short=False): - if const not in self.constants_dict: - value = const.value - TYPE = lltype.typeOf(value) - if kind == 'int': - if isinstance(TYPE, lltype.Ptr): - assert TYPE.TO._gckind == 'raw' - self.see_raw_object(value) - value = llmemory.cast_ptr_to_adr(value) - TYPE = llmemory.Address - if TYPE == llmemory.Address: - value = heaptracker.adr2int(value) - elif not isinstance(value, ComputedIntSymbolic): - value = lltype.cast_primitive(lltype.Signed, value) - if allow_short and -128 <= value <= 127: - # emit the constant as a small integer - self.code.append(chr(value & 0xFF)) - return True - constants = self.constants_i - elif kind == 'ref': - value = lltype.cast_opaque_ptr(llmemory.GCREF, value) - constants = self.constants_r - elif kind == 'float': - assert TYPE == lltype.Float - constants = self.constants_f - else: - raise NotImplementedError(const) + value = const.value + TYPE = lltype.typeOf(value) + if kind == 'int': + if isinstance(TYPE, lltype.Ptr): + assert TYPE.TO._gckind == 'raw' + self.see_raw_object(value) + value = llmemory.cast_ptr_to_adr(value) + TYPE = llmemory.Address + if TYPE == llmemory.Address: + value = heaptracker.adr2int(value) + elif not isinstance(value, ComputedIntSymbolic): + value = lltype.cast_primitive(lltype.Signed, value) + if allow_short and -128 <= value <= 127: + # emit the constant as a small integer + self.code.append(chr(value & 0xFF)) + return True + constants = self.constants_i + elif kind == 'ref': + value = lltype.cast_opaque_ptr(llmemory.GCREF, value) + constants = self.constants_r + elif kind == 'float': + assert TYPE == lltype.Float + constants = self.constants_f + else: + raise NotImplementedError(const) + key = (kind, Constant(value)) + if key not in self.constants_dict: constants.append(value) - self.constants_dict[const] = 256 - len(constants) + self.constants_dict[key] = 256 - len(constants) # emit the constant normally, as one byte that is an index in the # list of constants - self.code.append(chr(self.constants_dict[const])) + self.code.append(chr(self.constants_dict[key])) return False def write_insn(self, insn): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/call.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/call.py Fri Jul 2 17:44:46 2010 @@ -16,21 +16,22 @@ class CallControl(object): virtualref_info = None # optionally set from outside - virtualizable_info = None # optionally set from outside - portal_runner_ptr = None # optionally set from outside - def __init__(self, cpu=None, portal_graph=None): + def __init__(self, cpu=None, jitdrivers_sd=[]): + assert isinstance(jitdrivers_sd, list) # debugging self.cpu = cpu - self.portal_graph = portal_graph + self.jitdrivers_sd = jitdrivers_sd self.jitcodes = {} # map {graph: jitcode} self.unfinished_graphs = [] # list of graphs with pending jitcodes - self.jitdriver = None if hasattr(cpu, 'rtyper'): # for tests self.rtyper = cpu.rtyper translator = self.rtyper.annotator.translator self.raise_analyzer = RaiseAnalyzer(translator) self.readwrite_analyzer = ReadWriteAnalyzer(translator) self.virtualizable_analyzer = VirtualizableAnalyzer(translator) + # + for index, jd in enumerate(jitdrivers_sd): + jd.index = index def find_all_graphs(self, policy): try: @@ -41,8 +42,8 @@ def is_candidate(graph): return policy.look_inside_graph(graph) - assert self.portal_graph is not None - todo = [self.portal_graph] + assert len(self.jitdrivers_sd) > 0 + todo = [jd.portal_graph for jd in self.jitdrivers_sd] if hasattr(self, 'rtyper'): for oopspec_name, ll_args, ll_res in support.inline_calls_to: c_func, _ = support.builtin_func_for_spec(self.rtyper, @@ -122,7 +123,7 @@ def guess_call_kind(self, op, is_candidate=None): if op.opname == 'direct_call': funcptr = op.args[0].value - if funcptr is self.portal_runner_ptr: + if self.jitdriver_sd_from_portal_runner_ptr(funcptr) is not None: return 'recursive' funcobj = get_funcobj(funcptr) if getattr(funcobj, 'graph', None) is None: @@ -143,6 +144,11 @@ # used only after find_all_graphs() return graph in self.candidate_graphs + def grab_initial_jitcodes(self): + for jd in self.jitdrivers_sd: + jd.mainjitcode = self.get_jitcode(jd.portal_graph) + jd.mainjitcode.is_portal = True + def enum_pending_graphs(self): while self.unfinished_graphs: graph = self.unfinished_graphs.pop() @@ -241,12 +247,32 @@ return (effectinfo is None or effectinfo.extraeffect >= EffectInfo.EF_CAN_RAISE) - def found_jitdriver(self, jitdriver): - if self.jitdriver is None: - self.jitdriver = jitdriver - else: - assert self.jitdriver is jitdriver + def jitdriver_sd_from_portal_graph(self, graph): + for jd in self.jitdrivers_sd: + if jd.portal_graph is graph: + return jd + return None - def getjitdriver(self): - assert self.jitdriver is not None, "order dependency issue?" - return self.jitdriver + def jitdriver_sd_from_portal_runner_ptr(self, funcptr): + for jd in self.jitdrivers_sd: + if funcptr is jd.portal_runner_ptr: + return jd + return None + + def jitdriver_sd_from_jitdriver(self, jitdriver): + for jd in self.jitdrivers_sd: + if jd.jitdriver is jitdriver: + return jd + return None + + def get_vinfo(self, VTYPEPTR): + seen = set() + for jd in self.jitdrivers_sd: + if jd.virtualizable_info is not None: + if jd.virtualizable_info.is_vtypeptr(VTYPEPTR): + seen.add(jd.virtualizable_info) + if seen: + assert len(seen) == 1 + return seen.pop() + else: + return None Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/codewriter.py Fri Jul 2 17:44:46 2010 @@ -14,29 +14,30 @@ class CodeWriter(object): callcontrol = None # for tests - def __init__(self, cpu=None, maingraph=None): + def __init__(self, cpu=None, jitdrivers_sd=[]): self.cpu = cpu self.assembler = Assembler() - self.portal_graph = maingraph - self.callcontrol = CallControl(cpu, maingraph) + self.callcontrol = CallControl(cpu, jitdrivers_sd) + self._seen_files = set() def transform_func_to_jitcode(self, func, values, type_system='lltype'): """For testing.""" rtyper = support.annotate(func, values, type_system=type_system) graph = rtyper.annotator.translator.graphs[0] jitcode = JitCode("test") - self.transform_graph_to_jitcode(graph, jitcode, True, True) + self.transform_graph_to_jitcode(graph, jitcode, True) return jitcode - def transform_graph_to_jitcode(self, graph, jitcode, portal, verbose): + def transform_graph_to_jitcode(self, graph, jitcode, verbose): """Transform a graph into a JitCode containing the same bytecode in a different format. """ + portal_jd = self.callcontrol.jitdriver_sd_from_portal_graph(graph) graph = copygraph(graph, shallowvars=True) # # step 1: mangle the graph so that it contains the final instructions # that we want in the JitCode, but still as a control flow graph - transform_graph(graph, self.cpu, self.callcontrol, portal) + transform_graph(graph, self.cpu, self.callcontrol, portal_jd) # # step 2: perform register allocation on it regallocs = {} @@ -59,16 +60,14 @@ self.assembler.assemble(ssarepr, jitcode) # # print the resulting assembler - self.print_ssa_repr(ssarepr, portal, verbose) + self.print_ssa_repr(ssarepr, portal_jd, verbose) def make_jitcodes(self, verbose=False): log.info("making JitCodes...") - maingraph = self.portal_graph - self.mainjitcode = self.callcontrol.get_jitcode(maingraph) + self.callcontrol.grab_initial_jitcodes() count = 0 for graph, jitcode in self.callcontrol.enum_pending_graphs(): - self.transform_graph_to_jitcode(graph, jitcode, - graph is maingraph, verbose) + self.transform_graph_to_jitcode(graph, jitcode, verbose) count += 1 if not count % 500: log.info("Produced %d jitcodes" % count) @@ -76,33 +75,35 @@ log.info("there are %d JitCode instances." % count) def setup_vrefinfo(self, vrefinfo): + # must be called at most once + assert self.callcontrol.virtualref_info is None self.callcontrol.virtualref_info = vrefinfo - def setup_virtualizable_info(self, vinfo): - self.callcontrol.virtualizable_info = vinfo - - def setup_portal_runner_ptr(self, portal_runner_ptr): - self.callcontrol.portal_runner_ptr = portal_runner_ptr + def setup_jitdriver(self, jitdriver_sd): + # Must be called once per jitdriver. Usually jitdriver_sd is an + # instance of pypy.jit.metainterp.jitdriver.JitDriverStaticData. + self.callcontrol.jitdrivers_sd.append(jitdriver_sd) def find_all_graphs(self, policy): return self.callcontrol.find_all_graphs(policy) - def print_ssa_repr(self, ssarepr, portal, verbose): + def print_ssa_repr(self, ssarepr, portal_jitdriver, verbose): if verbose: print '%s:' % (ssarepr.name,) print format_assembler(ssarepr) else: dir = udir.ensure("jitcodes", dir=1) - if portal: - name = "00_portal_runner" + if portal_jitdriver: + name = "%02d_portal_runner" % (portal_jitdriver.index,) elif ssarepr.name and ssarepr.name != '?': name = ssarepr.name else: name = 'unnamed' % id(ssarepr) i = 1 extra = '' - while dir.join(name+extra).check(exists=1): + while name+extra in self._seen_files: i += 1 extra = '.%d' % i + self._seen_files.add(name+extra) dir.join(name+extra).write(format_assembler(ssarepr)) log.dot() Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/jitcode.py Fri Jul 2 17:44:46 2010 @@ -13,6 +13,7 @@ self.name = name self.fnaddr = fnaddr self.calldescr = calldescr + self.is_portal = False self._called_from = called_from # debugging self._ssarepr = None # debugging @@ -24,11 +25,11 @@ self.constants_i = constants_i or self._empty_i self.constants_r = constants_r or self._empty_r self.constants_f = constants_f or self._empty_f - # encode the three num_regs into a single integer + # encode the three num_regs into a single char each assert num_regs_i < 256 and num_regs_r < 256 and num_regs_f < 256 - self.num_regs_encoded = ((num_regs_i << 16) | - (num_regs_f << 8) | - (num_regs_r << 0)) + self.c_num_regs_i = chr(num_regs_i) + self.c_num_regs_r = chr(num_regs_r) + self.c_num_regs_f = chr(num_regs_f) self.liveness = make_liveness_cache(liveness) self._startpoints = startpoints # debugging self._alllabels = alllabels # debugging @@ -37,13 +38,13 @@ return heaptracker.adr2int(self.fnaddr) def num_regs_i(self): - return self.num_regs_encoded >> 16 - - def num_regs_f(self): - return (self.num_regs_encoded >> 8) & 0xFF + return ord(self.c_num_regs_i) def num_regs_r(self): - return self.num_regs_encoded & 0xFF + return ord(self.c_num_regs_r) + + def num_regs_f(self): + return ord(self.c_num_regs_f) def has_liveness_info(self, pc): return pc in self.liveness Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/jtransform.py Fri Jul 2 17:44:46 2010 @@ -13,23 +13,23 @@ from pypy.translator.simplify import get_funcobj -def transform_graph(graph, cpu=None, callcontrol=None, portal=True): +def transform_graph(graph, cpu=None, callcontrol=None, portal_jd=None): """Transform a control flow graph to make it suitable for being flattened in a JitCode. """ - t = Transformer(cpu, callcontrol) - t.transform(graph, portal) + t = Transformer(cpu, callcontrol, portal_jd) + t.transform(graph) class Transformer(object): - def __init__(self, cpu=None, callcontrol=None): + def __init__(self, cpu=None, callcontrol=None, portal_jd=None): self.cpu = cpu self.callcontrol = callcontrol + self.portal_jd = portal_jd # non-None only for the portal graph(s) - def transform(self, graph, portal): + def transform(self, graph): self.graph = graph - self.portal = portal for block in list(graph.iterblocks()): self.optimize_block(block) @@ -325,10 +325,13 @@ return op1 def handle_recursive_call(self, op): - ops = self.promote_greens(op.args[1:]) - targetgraph = self.callcontrol.portal_graph - num_green_args = len(self.callcontrol.getjitdriver().greens) - args = (self.make_three_lists(op.args[1:1+num_green_args]) + + jitdriver_sd = self.callcontrol.jitdriver_sd_from_portal_runner_ptr( + op.args[0].value) + assert jitdriver_sd is not None + ops = self.promote_greens(op.args[1:], jitdriver_sd.jitdriver) + num_green_args = len(jitdriver_sd.jitdriver.greens) + args = ([Constant(jitdriver_sd.index, lltype.Signed)] + + self.make_three_lists(op.args[1:1+num_green_args]) + self.make_three_lists(op.args[1+num_green_args:])) kind = getkind(op.result.concretetype)[0] op0 = SpaceOperation('recursive_call_%s' % kind, args, op.result) @@ -400,7 +403,12 @@ log.WARNING('ignoring hint %r at %r' % (hints, self.graph)) def rewrite_op_malloc_varsize(self, op): - assert op.args[1].value == {'flavor': 'gc'} + if op.args[1].value['flavor'] == 'raw': + ARRAY = op.args[0].value + return self._do_builtin_call(op, 'raw_malloc', + [op.args[2]], + extra = (ARRAY,), + extrakey = ARRAY) if op.args[0].value == rstr.STR: return SpaceOperation('newstr', [op.args[2]], op.result) elif op.args[0].value == rstr.UNICODE: @@ -483,14 +491,14 @@ # check for virtualizable try: if self.is_virtualizable_getset(op): - descr = self.get_virtualizable_field_descr(op.args[1].value) + descr = self.get_virtualizable_field_descr(op) kind = getkind(RESULT)[0] return [SpaceOperation('-live-', [], None), SpaceOperation('getfield_vable_%s' % kind, [v_inst, descr], op.result)] - except VirtualizableArrayField: + except VirtualizableArrayField, e: # xxx hack hack hack - vinfo = self.callcontrol.virtualizable_info + vinfo = e.args[1] arrayindex = vinfo.array_field_counter[op.args[1].value] arrayfielddescr = vinfo.array_field_descrs[arrayindex] arraydescr = vinfo.array_descrs[arrayindex] @@ -527,7 +535,7 @@ return # check for virtualizable if self.is_virtualizable_getset(op): - descr = self.get_virtualizable_field_descr(op.args[1].value) + descr = self.get_virtualizable_field_descr(op) kind = getkind(RESULT)[0] return [SpaceOperation('-live-', [], None), SpaceOperation('setfield_vable_%s' % kind, @@ -544,21 +552,23 @@ return (op.args[1].value == 'typeptr' and op.args[0].concretetype.TO._hints.get('typeptr')) + def get_vinfo(self, v_virtualizable): + if self.callcontrol is None: # for tests + return None + return self.callcontrol.get_vinfo(v_virtualizable.concretetype) + def is_virtualizable_getset(self, op): # every access of an object of exactly the type VTYPEPTR is # likely to be a virtualizable access, but we still have to # check it in pyjitpl.py. - try: - vinfo = self.callcontrol.virtualizable_info - except AttributeError: - return False - if vinfo is None or not vinfo.is_vtypeptr(op.args[0].concretetype): + vinfo = self.get_vinfo(op.args[0]) + if vinfo is None: return False res = False if op.args[1].value in vinfo.static_field_to_extra_box: res = True if op.args[1].value in vinfo.array_fields: - res = VirtualizableArrayField(self.graph) + res = VirtualizableArrayField(self.graph, vinfo) if res: flags = self.vable_flags[op.args[0]] @@ -568,8 +578,9 @@ raise res return res - def get_virtualizable_field_descr(self, fieldname): - vinfo = self.callcontrol.virtualizable_info + def get_virtualizable_field_descr(self, op): + fieldname = op.args[1].value + vinfo = self.get_vinfo(op.args[0]) index = vinfo.static_field_to_extra_box[fieldname] return vinfo.static_field_descrs[index] @@ -750,9 +761,10 @@ return Constant(value, lltype.Bool) return op - def promote_greens(self, args): + def promote_greens(self, args, jitdriver): ops = [] - num_green_args = len(self.callcontrol.getjitdriver().greens) + num_green_args = len(jitdriver.greens) + assert len(args) == num_green_args + len(jitdriver.reds) for v in args[:num_green_args]: if isinstance(v, Variable) and v.concretetype is not lltype.Void: kind = getkind(v.concretetype) @@ -762,21 +774,28 @@ return ops def rewrite_op_jit_marker(self, op): - self.callcontrol.found_jitdriver(op.args[1].value) key = op.args[0].value - return getattr(self, 'handle_jit_marker__%s' % key)(op) + jitdriver = op.args[1].value + return getattr(self, 'handle_jit_marker__%s' % key)(op, jitdriver) - def handle_jit_marker__jit_merge_point(self, op): - assert self.portal, "jit_merge_point in non-main graph!" - ops = self.promote_greens(op.args[2:]) - num_green_args = len(self.callcontrol.getjitdriver().greens) - args = (self.make_three_lists(op.args[2:2+num_green_args]) + + def handle_jit_marker__jit_merge_point(self, op, jitdriver): + assert self.portal_jd is not None, ( + "'jit_merge_point' in non-portal graph!") + assert jitdriver is self.portal_jd.jitdriver, ( + "general mix-up of jitdrivers?") + ops = self.promote_greens(op.args[2:], jitdriver) + num_green_args = len(jitdriver.greens) + args = ([Constant(self.portal_jd.index, lltype.Signed)] + + self.make_three_lists(op.args[2:2+num_green_args]) + self.make_three_lists(op.args[2+num_green_args:])) op1 = SpaceOperation('jit_merge_point', args, None) return ops + [op1] - def handle_jit_marker__can_enter_jit(self, op): - return SpaceOperation('can_enter_jit', [], None) + def handle_jit_marker__can_enter_jit(self, op, jitdriver): + jd = self.callcontrol.jitdriver_sd_from_jitdriver(jitdriver) + assert jd is not None + c_index = Constant(jd.index, lltype.Signed) + return SpaceOperation('can_enter_jit', [c_index], None) def rewrite_op_debug_assert(self, op): log.WARNING("found debug_assert in %r; should have be removed" % @@ -974,9 +993,8 @@ def rewrite_op_jit_force_virtualizable(self, op): # this one is for virtualizables - vinfo = self.callcontrol.virtualizable_info + vinfo = self.get_vinfo(op.args[0]) assert vinfo is not None - assert vinfo.is_vtypeptr(op.args[0].concretetype) self.vable_flags[op.args[0]] = op.args[2].value return [] Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/support.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/support.py Fri Jul 2 17:44:46 2010 @@ -282,6 +282,9 @@ # ---------- malloc with del ---------- + def _ll_2_raw_malloc(TP, size): + return lltype.malloc(TP, size, flavor='raw') + def build_ll_0_alloc_with_del(RESULT, vtable): def _ll_0_alloc_with_del(): p = lltype.malloc(RESULT.TO) @@ -289,6 +292,10 @@ return p return _ll_0_alloc_with_del + def build_ll_1_raw_malloc(ARRAY): + def _ll_1_raw_malloc(n): + return lltype.malloc(ARRAY, n, flavor='raw') + return _ll_1_raw_malloc class OOtypeHelpers: Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_assembler.py Fri Jul 2 17:44:46 2010 @@ -127,6 +127,25 @@ assert assembler.insns == {'foobar/IR': 0} assert jitcode.constants_i == [42] +def test_assemble_list_semibug(): + # the semibug is that after forcing 42 into the dict of constants, + # it would be reused for all future 42's, even ones that can be + # encoded directly. + ssarepr = SSARepr("test") + ssarepr.insns = [ + ('foobar', ListOfKind('int', [Constant(42, lltype.Signed)])), + ('foobar', ListOfKind('int', [Constant(42, lltype.Signed)])), + ('baz', Constant(42, lltype.Signed)), + ] + assembler = Assembler() + jitcode = assembler.assemble(ssarepr) + assert jitcode.code == ("\x00\x01\xFF" + "\x00\x01\xFF" + "\x01\x2A") + assert assembler.insns == {'foobar/I': 0, + 'baz/c': 1} + assert jitcode.constants_i == [42] + def test_assemble_descr(): class FooDescr(AbstractDescr): pass Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_call.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_call.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_call.py Fri Jul 2 17:44:46 2010 @@ -52,13 +52,19 @@ # ____________________________________________________________ +class FakeJitDriverSD: + def __init__(self, portal_graph): + self.portal_graph = portal_graph + self.portal_runner_ptr = "???" + def test_find_all_graphs(): def g(x): return x + 2 def f(x): return g(x) + 1 rtyper = support.annotate(f, [7]) - cc = CallControl(portal_graph=rtyper.annotator.translator.graphs[0]) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cc = CallControl(jitdrivers_sd=[jitdriver_sd]) res = cc.find_all_graphs(FakePolicy()) funcs = set([graph.func for graph in res]) assert funcs == set([f, g]) @@ -69,7 +75,8 @@ def f(x): return g(x) + 1 rtyper = support.annotate(f, [7]) - cc = CallControl(portal_graph=rtyper.annotator.translator.graphs[0]) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cc = CallControl(jitdrivers_sd=[jitdriver_sd]) class CustomFakePolicy: def look_inside_graph(self, graph): assert graph.name == 'g' @@ -83,10 +90,11 @@ def test_guess_call_kind_and_calls_from_graphs(): class portal_runner_obj: graph = object() + class FakeJitDriverSD: + portal_runner_ptr = portal_runner_obj g = object() g1 = object() - cc = CallControl() - cc.portal_runner_ptr = portal_runner_obj + cc = CallControl(jitdrivers_sd=[FakeJitDriverSD()]) cc.candidate_graphs = [g, g1] op = SpaceOperation('direct_call', [Constant(portal_runner_obj)], Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_codewriter.py Fri Jul 2 17:44:46 2010 @@ -2,7 +2,7 @@ from pypy.jit.codewriter.codewriter import CodeWriter from pypy.jit.codewriter import support from pypy.jit.metainterp.history import AbstractDescr -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rffi class FakeCallDescr(AbstractDescr): def __init__(self, FUNC, ARGS, RESULT, effectinfo=None): @@ -35,6 +35,12 @@ def look_inside_graph(self, graph): return graph.name != 'dont_look' +class FakeJitDriverSD: + def __init__(self, portal_graph): + self.portal_graph = portal_graph + self.portal_runner_ptr = "???" + self.virtualizable_info = None + def test_loop(): def f(a, b): @@ -70,11 +76,11 @@ def fff(a, b): return ggg(b) - ggg(a) rtyper = support.annotate(fff, [35, 42]) - maingraph = rtyper.annotator.translator.graphs[0] - cw = CodeWriter(FakeCPU(rtyper), maingraph) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cw = CodeWriter(FakeCPU(rtyper), [jitdriver_sd]) cw.find_all_graphs(FakePolicy()) cw.make_jitcodes(verbose=True) - jitcode = cw.mainjitcode + jitcode = jitdriver_sd.mainjitcode print jitcode.dump() [jitcode2] = cw.assembler.descrs print jitcode2.dump() @@ -100,7 +106,7 @@ blackholeinterp.setarg_i(0, 6) blackholeinterp.setarg_i(1, 100) blackholeinterp.run() - assert blackholeinterp.final_result_i() == 100+6+5+4+3 + assert blackholeinterp.get_tmpreg_i() == 100+6+5+4+3 def test_instantiate(): class A1: id = 651 @@ -117,7 +123,7 @@ return x().id + y().id + dont_look(n) rtyper = support.annotate(f, [35]) maingraph = rtyper.annotator.translator.graphs[0] - cw = CodeWriter(FakeCPU(rtyper), maingraph) + cw = CodeWriter(FakeCPU(rtyper), [FakeJitDriverSD(maingraph)]) cw.find_all_graphs(FakePolicy()) cw.make_jitcodes(verbose=True) # @@ -144,10 +150,29 @@ def f(n): return abs(n) rtyper = support.annotate(f, [35]) - maingraph = rtyper.annotator.translator.graphs[0] - cw = CodeWriter(FakeCPU(rtyper), maingraph) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cw = CodeWriter(FakeCPU(rtyper), [jitdriver_sd]) cw.find_all_graphs(FakePolicy()) cw.make_jitcodes(verbose=True) # - s = cw.mainjitcode.dump() + s = jitdriver_sd.mainjitcode.dump() assert "inline_call_ir_i " in s + +def test_raw_malloc_and_access(): + TP = rffi.CArray(lltype.Signed) + + def f(n): + a = lltype.malloc(TP, n, flavor='raw') + #a[0] = n + #res = a[0] + #lltype.free(a, flavor='raw') + #return res + + rtyper = support.annotate(f, [35]) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cw = CodeWriter(FakeCPU(rtyper), [jitdriver_sd]) + cw.find_all_graphs(FakePolicy()) + cw.make_jitcodes(verbose=True) + # + s = jitdriver_sd.mainjitcode.dump() + assert 'residual_call_ir_i $<* fn _ll_1_raw_malloc__Signed>' in s Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_flatten.py Fri Jul 2 17:44:46 2010 @@ -68,11 +68,8 @@ return FakeDescr() def calldescr_canraise(self, calldescr): return calldescr is not self._descr_cannot_raise - def found_jitdriver(self, jitdriver): - assert isinstance(jitdriver, JitDriver) - self.jitdriver = jitdriver - def getjitdriver(self): - return self.jitdriver + def get_vinfo(self, VTYPEPTR): + return None class FakeCallControlWithVRefInfo: class virtualref_info: @@ -118,13 +115,13 @@ return self.rtyper.annotator.translator.graphs def encoding_test(self, func, args, expected, - transform=False, liveness=False, cc=None): + transform=False, liveness=False, cc=None, jd=None): graphs = self.make_graphs(func, args) #graphs[0].show() if transform: from pypy.jit.codewriter.jtransform import transform_graph cc = cc or FakeCallControl() - transform_graph(graphs[0], FakeCPU(self.rtyper), cc) + transform_graph(graphs[0], FakeCPU(self.rtyper), cc, jd) ssarepr = flatten_graph(graphs[0], fake_regallocs(), _include_all_exc_links=not transform) if liveness: @@ -584,13 +581,21 @@ def f(x, y): myjitdriver.jit_merge_point(x=x, y=y) myjitdriver.can_enter_jit(x=y, y=x) + class FakeJitDriverSD: + jitdriver = myjitdriver + index = 27 + jd = FakeJitDriverSD() + class MyFakeCallControl(FakeCallControl): + def jitdriver_sd_from_jitdriver(self, jitdriver): + assert jitdriver == myjitdriver + return jd self.encoding_test(f, [4, 5], """ -live- %i0, %i1 int_guard_value %i0 - jit_merge_point I[%i0], R[], F[], I[%i1], R[], F[] - can_enter_jit + jit_merge_point $27, I[%i0], R[], F[], I[%i1], R[], F[] + can_enter_jit $27 void_return - """, transform=True, liveness=True) + """, transform=True, liveness=True, cc=MyFakeCallControl(), jd=jd) def test_keepalive(self): S = lltype.GcStruct('S') Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/blackhole.py Fri Jul 2 17:44:46 2010 @@ -315,27 +315,12 @@ def get_tmpreg_f(self): return self.tmpreg_f - def final_result_i(self): - assert self._return_type == 'i' - return self.get_tmpreg_i() - - def final_result_r(self): - assert self._return_type == 'r' - return self.get_tmpreg_r() - - def final_result_f(self): - assert self._return_type == 'f' - return self.get_tmpreg_f() - - def final_result_v(self): - assert self._return_type == 'v' - def _final_result_anytype(self): "NOT_RPYTHON" - if self._return_type == 'i': return self.final_result_i() - if self._return_type == 'r': return self.final_result_r() - if self._return_type == 'f': return self.final_result_f() - if self._return_type == 'v': return self.final_result_v() + if self._return_type == 'i': return self.get_tmpreg_i() + if self._return_type == 'r': return self.get_tmpreg_r() + if self._return_type == 'f': return self.get_tmpreg_f() + if self._return_type == 'v': return None raise ValueError(self._return_type) def cleanup_registers(self): @@ -774,12 +759,12 @@ # ---------- # the main hints and recursive calls - @arguments() - def bhimpl_can_enter_jit(): + @arguments("i") + def bhimpl_can_enter_jit(jdindex): pass - @arguments("self", "I", "R", "F", "I", "R", "F") - def bhimpl_jit_merge_point(self, *args): + @arguments("self", "i", "I", "R", "F", "I", "R", "F") + def bhimpl_jit_merge_point(self, jdindex, *args): if self.nextblackholeinterp is None: # we are the last level CRN = self.builder.metainterp_sd.ContinueRunningNormally raise CRN(*args) @@ -793,55 +778,55 @@ # call the interpreter main loop from here, and just return its # result. sd = self.builder.metainterp_sd - if sd.result_type == 'void': - self.bhimpl_recursive_call_v(*args) + result_type = sd.jitdrivers_sd[jdindex].result_type + if result_type == 'v': + self.bhimpl_recursive_call_v(jdindex, *args) self.bhimpl_void_return() - elif sd.result_type == 'int': - x = self.bhimpl_recursive_call_i(*args) + elif result_type == 'i': + x = self.bhimpl_recursive_call_i(jdindex, *args) self.bhimpl_int_return(x) - elif sd.result_type == 'ref': - x = self.bhimpl_recursive_call_r(*args) + elif result_type == 'r': + x = self.bhimpl_recursive_call_r(jdindex, *args) self.bhimpl_ref_return(x) - elif sd.result_type == 'float': - x = self.bhimpl_recursive_call_f(*args) + elif result_type == 'f': + x = self.bhimpl_recursive_call_f(jdindex, *args) self.bhimpl_float_return(x) assert False - def get_portal_runner(self): - metainterp_sd = self.builder.metainterp_sd - fnptr = llmemory.cast_ptr_to_adr(metainterp_sd._portal_runner_ptr) - fnptr = heaptracker.adr2int(fnptr) - calldescr = metainterp_sd.portal_code.calldescr + def get_portal_runner(self, jdindex): + jitdriver_sd = self.builder.metainterp_sd.jitdrivers_sd[jdindex] + fnptr = heaptracker.adr2int(jitdriver_sd.portal_runner_adr) + calldescr = jitdriver_sd.mainjitcode.calldescr return fnptr, calldescr - @arguments("self", "I", "R", "F", "I", "R", "F", returns="i") - def bhimpl_recursive_call_i(self, greens_i, greens_r, greens_f, - reds_i, reds_r, reds_f): - fnptr, calldescr = self.get_portal_runner() + @arguments("self", "i", "I", "R", "F", "I", "R", "F", returns="i") + def bhimpl_recursive_call_i(self, jdindex, greens_i, greens_r, greens_f, + reds_i, reds_r, reds_f): + fnptr, calldescr = self.get_portal_runner(jdindex) return self.cpu.bh_call_i(fnptr, calldescr, greens_i + reds_i, greens_r + reds_r, greens_f + reds_f) - @arguments("self", "I", "R", "F", "I", "R", "F", returns="r") - def bhimpl_recursive_call_r(self, greens_i, greens_r, greens_f, - reds_i, reds_r, reds_f): - fnptr, calldescr = self.get_portal_runner() + @arguments("self", "i", "I", "R", "F", "I", "R", "F", returns="r") + def bhimpl_recursive_call_r(self, jdindex, greens_i, greens_r, greens_f, + reds_i, reds_r, reds_f): + fnptr, calldescr = self.get_portal_runner(jdindex) return self.cpu.bh_call_r(fnptr, calldescr, greens_i + reds_i, greens_r + reds_r, greens_f + reds_f) - @arguments("self", "I", "R", "F", "I", "R", "F", returns="f") - def bhimpl_recursive_call_f(self, greens_i, greens_r, greens_f, - reds_i, reds_r, reds_f): - fnptr, calldescr = self.get_portal_runner() + @arguments("self", "i", "I", "R", "F", "I", "R", "F", returns="f") + def bhimpl_recursive_call_f(self, jdindex, greens_i, greens_r, greens_f, + reds_i, reds_r, reds_f): + fnptr, calldescr = self.get_portal_runner(jdindex) return self.cpu.bh_call_f(fnptr, calldescr, greens_i + reds_i, greens_r + reds_r, greens_f + reds_f) - @arguments("self", "I", "R", "F", "I", "R", "F") - def bhimpl_recursive_call_v(self, greens_i, greens_r, greens_f, - reds_i, reds_r, reds_f): - fnptr, calldescr = self.get_portal_runner() + @arguments("self", "i", "I", "R", "F", "I", "R", "F") + def bhimpl_recursive_call_v(self, jdindex, greens_i, greens_r, greens_f, + reds_i, reds_r, reds_f): + fnptr, calldescr = self.get_portal_runner(jdindex) return self.cpu.bh_call_v(fnptr, calldescr, greens_i + reds_i, greens_r + reds_r, @@ -1178,11 +1163,11 @@ self._done_with_this_frame() kind = self._return_type if kind == 'i': - caller._setup_return_value_i(self.final_result_i()) + caller._setup_return_value_i(self.get_tmpreg_i()) elif kind == 'r': - caller._setup_return_value_r(self.final_result_r()) + caller._setup_return_value_r(self.get_tmpreg_r()) elif kind == 'f': - caller._setup_return_value_f(self.final_result_f()) + caller._setup_return_value_f(self.get_tmpreg_f()) else: assert kind == 'v' return lltype.nullptr(rclass.OBJECTPTR.TO) @@ -1248,15 +1233,15 @@ # rare case: we only get there if the blackhole interps all returned # normally (in general we get a ContinueRunningNormally exception). sd = self.builder.metainterp_sd - if sd.result_type == 'void': - self.final_result_v() + kind = self._return_type + if kind == 'v': raise sd.DoneWithThisFrameVoid() - elif sd.result_type == 'int': - raise sd.DoneWithThisFrameInt(self.final_result_i()) - elif sd.result_type == 'ref': - raise sd.DoneWithThisFrameRef(self.cpu, self.final_result_r()) - elif sd.result_type == 'float': - raise sd.DoneWithThisFrameFloat(self.final_result_f()) + elif kind == 'i': + raise sd.DoneWithThisFrameInt(self.get_tmpreg_i()) + elif kind == 'r': + raise sd.DoneWithThisFrameRef(self.cpu, self.get_tmpreg_r()) + elif kind == 'f': + raise sd.DoneWithThisFrameFloat(self.get_tmpreg_f()) else: assert False @@ -1290,12 +1275,14 @@ blackholeinterp.builder.release_interp(blackholeinterp) blackholeinterp = blackholeinterp.nextblackholeinterp -def resume_in_blackhole(metainterp_sd, resumedescr, all_virtuals=None): +def resume_in_blackhole(metainterp_sd, jitdriver_sd, resumedescr, + all_virtuals=None): from pypy.jit.metainterp.resume import blackhole_from_resumedata debug_start('jit-blackhole') metainterp_sd.profiler.start_blackhole() blackholeinterp = blackhole_from_resumedata( metainterp_sd.blackholeinterpbuilder, + jitdriver_sd, resumedescr, all_virtuals) current_exc = blackholeinterp._prepare_resume_from_failure( Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/compile.py Fri Jul 2 17:44:46 2010 @@ -39,9 +39,10 @@ name = metainterp.staticdata.stats.name_for_new_loop() return TreeLoop(name) -def make_loop_token(nb_args): +def make_loop_token(nb_args, jitdriver_sd): loop_token = LoopToken() loop_token.specnodes = [prebuiltNotSpecNode] * nb_args + loop_token.outermost_jitdriver_sd = jitdriver_sd return loop_token # ____________________________________________________________ @@ -63,11 +64,12 @@ # make a copy, because optimize_loop can mutate the ops and descrs loop.operations = [op.clone() for op in ops] metainterp_sd = metainterp.staticdata - loop_token = make_loop_token(len(loop.inputargs)) + jitdriver_sd = metainterp.jitdriver_sd + loop_token = make_loop_token(len(loop.inputargs), jitdriver_sd) loop.token = loop_token loop.operations[-1].descr = loop_token # patch the target of the JUMP try: - old_loop_token = metainterp_sd.state.optimize_loop( + old_loop_token = jitdriver_sd.warmstate.optimize_loop( metainterp_sd, old_loop_tokens, loop) except InvalidLoop: return None @@ -141,32 +143,32 @@ pass class DoneWithThisFrameDescrVoid(_DoneWithThisFrameDescr): - def handle_fail(self, metainterp_sd): - assert metainterp_sd.result_type == 'void' + def handle_fail(self, metainterp_sd, jitdriver_sd): + assert jitdriver_sd.result_type == history.VOID raise metainterp_sd.DoneWithThisFrameVoid() class DoneWithThisFrameDescrInt(_DoneWithThisFrameDescr): - def handle_fail(self, metainterp_sd): - assert metainterp_sd.result_type == 'int' + def handle_fail(self, metainterp_sd, jitdriver_sd): + assert jitdriver_sd.result_type == history.INT result = metainterp_sd.cpu.get_latest_value_int(0) raise metainterp_sd.DoneWithThisFrameInt(result) class DoneWithThisFrameDescrRef(_DoneWithThisFrameDescr): - def handle_fail(self, metainterp_sd): - assert metainterp_sd.result_type == 'ref' + def handle_fail(self, metainterp_sd, jitdriver_sd): + assert jitdriver_sd.result_type == history.REF cpu = metainterp_sd.cpu result = cpu.get_latest_value_ref(0) cpu.clear_latest_values(1) raise metainterp_sd.DoneWithThisFrameRef(cpu, result) class DoneWithThisFrameDescrFloat(_DoneWithThisFrameDescr): - def handle_fail(self, metainterp_sd): - assert metainterp_sd.result_type == 'float' + def handle_fail(self, metainterp_sd, jitdriver_sd): + assert jitdriver_sd.result_type == history.FLOAT result = metainterp_sd.cpu.get_latest_value_float(0) raise metainterp_sd.DoneWithThisFrameFloat(result) class ExitFrameWithExceptionDescrRef(_DoneWithThisFrameDescr): - def handle_fail(self, metainterp_sd): + def handle_fail(self, metainterp_sd, jitdriver_sd): cpu = metainterp_sd.cpu value = cpu.get_latest_value_ref(0) cpu.clear_latest_values(1) @@ -210,8 +212,6 @@ class ResumeDescr(AbstractFailDescr): def __init__(self, original_greenkey): self.original_greenkey = original_greenkey - def _clone_if_mutable(self): - raise NotImplementedError class ResumeGuardDescr(ResumeDescr): _counter = 0 # if < 0, there is one counter per value; @@ -258,22 +258,27 @@ # a negative value self._counter = cnt | i - def handle_fail(self, metainterp_sd): - if self.must_compile(metainterp_sd): - return self._trace_and_compile_from_bridge(metainterp_sd) + def handle_fail(self, metainterp_sd, jitdriver_sd): + if self.must_compile(metainterp_sd, jitdriver_sd): + return self._trace_and_compile_from_bridge(metainterp_sd, + jitdriver_sd) else: from pypy.jit.metainterp.blackhole import resume_in_blackhole - resume_in_blackhole(metainterp_sd, self) + resume_in_blackhole(metainterp_sd, jitdriver_sd, self) assert 0, "unreachable" - def _trace_and_compile_from_bridge(self, metainterp_sd): + def _trace_and_compile_from_bridge(self, metainterp_sd, jitdriver_sd): + # 'jitdriver_sd' corresponds to the outermost one, i.e. the one + # of the jit_merge_point where we started the loop, even if the + # loop itself may contain temporarily recursion into other + # jitdrivers. from pypy.jit.metainterp.pyjitpl import MetaInterp - metainterp = MetaInterp(metainterp_sd) + metainterp = MetaInterp(metainterp_sd, jitdriver_sd) return metainterp.handle_guard_failure(self) _trace_and_compile_from_bridge._dont_inline_ = True - def must_compile(self, metainterp_sd): - trace_eagerness = metainterp_sd.state.trace_eagerness + def must_compile(self, metainterp_sd, jitdriver_sd): + trace_eagerness = jitdriver_sd.warmstate.trace_eagerness if self._counter >= 0: self._counter += 1 return self._counter >= trace_eagerness @@ -320,8 +325,7 @@ send_bridge_to_backend(metainterp.staticdata, self, inputargs, new_loop.operations) - def _clone_if_mutable(self): - res = self.__class__(self.metainterp_sd, self.original_greenkey) + def copy_all_attrbutes_into(self, res): # XXX a bit ugly to have to list them all here res.rd_snapshot = self.rd_snapshot res.rd_frame_info_list = self.rd_frame_info_list @@ -329,11 +333,19 @@ res.rd_consts = self.rd_consts res.rd_virtuals = self.rd_virtuals res.rd_pendingfields = self.rd_pendingfields + + def _clone_if_mutable(self): + res = ResumeGuardDescr(self.metainterp_sd, self.original_greenkey) + self.copy_all_attrbutes_into(res) return res class ResumeGuardForcedDescr(ResumeGuardDescr): - def handle_fail(self, metainterp_sd): + def __init__(self, metainterp_sd, original_greenkey, jitdriver_sd): + ResumeGuardDescr.__init__(self, metainterp_sd, original_greenkey) + self.jitdriver_sd = jitdriver_sd + + def handle_fail(self, metainterp_sd, jitdriver_sd): # Failures of a GUARD_NOT_FORCED are never compiled, but # always just blackholed. First fish for the data saved when # the virtualrefs and virtualizable have been forced by @@ -343,7 +355,8 @@ all_virtuals = self.fetch_data(token) if all_virtuals is None: all_virtuals = [] - resume_in_blackhole(metainterp_sd, self, all_virtuals) + assert jitdriver_sd is self.jitdriver_sd + resume_in_blackhole(metainterp_sd, jitdriver_sd, self, all_virtuals) assert 0, "unreachable" @staticmethod @@ -358,7 +371,8 @@ def handle_async_forcing(self, force_token): from pypy.jit.metainterp.resume import force_from_resumedata metainterp_sd = self.metainterp_sd - all_virtuals = force_from_resumedata(metainterp_sd, self) + vinfo = self.jitdriver_sd.virtualizable_info + all_virtuals = force_from_resumedata(metainterp_sd, self, vinfo) # The virtualizable data was stored on the real virtualizable above. # Handle all_virtuals: keep them for later blackholing from the # future failure of the GUARD_NOT_FORCED @@ -392,6 +406,13 @@ assert 0, "not found: %r" % (key,) return data + def _clone_if_mutable(self): + res = ResumeGuardForcedDescr(self.metainterp_sd, + self.original_greenkey, + self.jitdriver_sd) + self.copy_all_attrbutes_into(res) + return res + class AbstractResumeGuardCounters(object): # Completely custom algorithm for now: keep 5 pairs (value, counter), @@ -464,19 +485,19 @@ # a loop at all but ends in a jump to the target loop. It starts # with completely unoptimized arguments, as in the interpreter. metainterp_sd = metainterp.staticdata + jitdriver_sd = metainterp.jitdriver_sd metainterp.history.inputargs = self.redkey - new_loop_token = make_loop_token(len(self.redkey)) + new_loop_token = make_loop_token(len(self.redkey), jitdriver_sd) new_loop.greenkey = self.original_greenkey new_loop.inputargs = self.redkey new_loop.token = new_loop_token send_loop_to_backend(metainterp_sd, new_loop, "entry bridge") # send the new_loop to warmspot.py, to be called directly the next time - metainterp_sd.state.attach_unoptimized_bridge_from_interp( + jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp( self.original_greenkey, new_loop_token) # store the new loop in compiled_merge_points too - glob = metainterp_sd.globaldata - old_loop_tokens = glob.get_compiled_merge_points( + old_loop_tokens = metainterp.get_compiled_merge_points( self.original_greenkey) # it always goes at the end of the list, as it is the most # general loop token @@ -500,10 +521,11 @@ # clone ops, as optimize_bridge can mutate the ops new_loop.operations = [op.clone() for op in metainterp.history.operations] metainterp_sd = metainterp.staticdata + state = metainterp.jitdriver_sd.warmstate try: - target_loop_token = metainterp_sd.state.optimize_bridge(metainterp_sd, - old_loop_tokens, - new_loop) + target_loop_token = state.optimize_bridge(metainterp_sd, + old_loop_tokens, + new_loop) except InvalidLoop: # XXX I am fairly convinced that optimize_bridge cannot actually raise # InvalidLoop Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/history.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/history.py Fri Jul 2 17:44:46 2010 @@ -125,9 +125,6 @@ def repr_of_descr(self): return '%r' % (self,) - def _clone_if_mutable(self): - return self - def get_arg_types(self): """ Implement in call descr. Must return a string of INT, REF and FLOAT ('i', 'r', 'f'). @@ -171,10 +168,18 @@ """ raise NotImplementedError + def _clone_if_mutable(self): + return self + def clone_if_mutable(self): + clone = self._clone_if_mutable() + if not we_are_translated(): + assert clone.__class__ is self.__class__ + return clone + class AbstractFailDescr(AbstractDescr): index = -1 - def handle_fail(self, metainterp_sd): + def handle_fail(self, metainterp_sd, jitdriver_sd): raise NotImplementedError def compile_and_attach(self, metainterp, new_loop): raise NotImplementedError @@ -694,6 +699,7 @@ generated assembler. """ terminating = False # see TerminatingLoopToken in compile.py + outermost_jitdriver_sd = None # specnodes = ... # and more data specified by the backend when the loop is compiled number = 0 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py Fri Jul 2 17:44:46 2010 @@ -554,7 +554,7 @@ elif op.can_raise(): self.exception_might_have_happened = True elif op.returns_bool_result(): - self.bool_boxes[op.result] = None + self.bool_boxes[self.getvalue(op.result)] = None self.newoperations.append(op) def store_final_boxes_in_guard(self, op): @@ -568,7 +568,7 @@ descr.store_final_boxes(op, newboxes) # if op.opnum == rop.GUARD_VALUE: - if op.args[0] in self.bool_boxes: + if self.getvalue(op.args[0]) in self.bool_boxes: # Hack: turn guard_value(bool) into guard_true/guard_false. # This is done after the operation is emitted, to let # store_final_boxes_in_guard set the guard_opnum field @@ -746,6 +746,9 @@ self.optimize_default(op) def optimize_INT_IS_TRUE(self, op): + if self.getvalue(op.args[0]) in self.bool_boxes: + self.make_equal_to(op.result, self.getvalue(op.args[0])) + return self._optimize_nullness(op, op.args[0], True) def optimize_INT_IS_ZERO(self, op): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/pyjitpl.py Fri Jul 2 17:44:46 2010 @@ -56,6 +56,7 @@ self.parent_resumedata_snapshot = None self.parent_resumedata_frame_info_list = None + @specialize.arg(3) def copy_constants(self, registers, constants, ConstClass): """Copy jitcode.constants[0] to registers[255], jitcode.constants[1] to registers[254], @@ -68,7 +69,6 @@ assert j >= 0 registers[j] = ConstClass(constants[i]) i -= 1 - copy_constants._annspecialcase_ = 'specialize:arg(3)' def cleanup_registers(self): # To avoid keeping references alive, this cleans up the registers_r. @@ -80,6 +80,7 @@ # ------------------------------ # Decoding of the JitCode + @specialize.arg(4) def prepare_list_of_boxes(self, outvalue, startindex, position, argcode): assert argcode in 'IRF' code = self.bytecode @@ -92,7 +93,6 @@ elif argcode == 'F': reg = self.registers_f[index] else: raise AssertionError(argcode) outvalue[startindex+i] = reg - prepare_list_of_boxes._annspecialcase_ = 'specialize:arg(4)' def get_current_position_info(self): return self.jitcode.get_live_vars_info(self.pc) @@ -516,6 +516,8 @@ def _nonstandard_virtualizable(self, pc, box): # returns True if 'box' is actually not the "standard" virtualizable # that is stored in metainterp.virtualizable_boxes[-1] + if self.metainterp.jitdriver_sd.virtualizable_info is None: + return True # can occur in case of multiple JITs standard_box = self.metainterp.virtualizable_boxes[-1] if standard_box is box: return False @@ -528,17 +530,11 @@ return not isstandard def _get_virtualizable_field_index(self, fielddescr): - vinfo = self.metainterp.staticdata.virtualizable_info + # Get the index of a fielddescr. Must only be called for + # the "standard" virtualizable. + vinfo = self.metainterp.jitdriver_sd.virtualizable_info return vinfo.static_field_by_descrs[fielddescr] - def _get_virtualizable_array_field_descr(self, index): - vinfo = self.metainterp.staticdata.virtualizable_info - return vinfo.array_field_descrs[index] - - def _get_virtualizable_array_descr(self, index): - vinfo = self.metainterp.staticdata.virtualizable_info - return vinfo.array_descrs[index] - @arguments("orgpc", "box", "descr") def _opimpl_getfield_vable(self, pc, box, fielddescr): if self._nonstandard_virtualizable(pc, box): @@ -566,8 +562,11 @@ opimpl_setfield_vable_f = _opimpl_setfield_vable def _get_arrayitem_vable_index(self, pc, arrayfielddescr, indexbox): + # Get the index of an array item: the index'th of the array + # described by arrayfielddescr. Must only be called for + # the "standard" virtualizable. indexbox = self.implement_guard_value(pc, indexbox) - vinfo = self.metainterp.staticdata.virtualizable_info + vinfo = self.metainterp.jitdriver_sd.virtualizable_info virtualizable_box = self.metainterp.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) arrayindex = vinfo.array_field_by_descrs[arrayfielddescr] @@ -616,7 +615,7 @@ arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, fdescr, box) return self.execute_with_descr(rop.ARRAYLEN_GC, adescr, arraybox) - vinfo = self.metainterp.staticdata.virtualizable_info + vinfo = self.metainterp.jitdriver_sd.virtualizable_info virtualizable_box = self.metainterp.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) arrayindex = vinfo.array_field_by_descrs[fdescr] @@ -665,12 +664,12 @@ opimpl_residual_call_irf_f = _opimpl_residual_call3 opimpl_residual_call_irf_v = _opimpl_residual_call3 - @arguments("boxes3", "boxes3") - def _opimpl_recursive_call(self, greenboxes, redboxes): + @arguments("int", "boxes3", "boxes3") + def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes): + targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex] allboxes = greenboxes + redboxes - metainterp_sd = self.metainterp.staticdata - portal_code = metainterp_sd.portal_code - warmrunnerstate = metainterp_sd.state + portal_code = targetjitdriver_sd.mainjitcode + warmrunnerstate = targetjitdriver_sd.warmstate token = None if warmrunnerstate.inlining: if warmrunnerstate.can_inline_callable(greenboxes): @@ -679,12 +678,13 @@ token = warmrunnerstate.get_assembler_token(greenboxes) # verify that we have all green args, needed to make sure # that assembler that we call is still correct - self.verify_green_args(greenboxes) + self.verify_green_args(targetjitdriver_sd, greenboxes) # - k = llmemory.cast_ptr_to_adr(metainterp_sd._portal_runner_ptr) + k = targetjitdriver_sd.portal_runner_adr funcbox = ConstInt(heaptracker.adr2int(k)) return self.do_residual_call(funcbox, portal_code.calldescr, - allboxes, assembler_call_token=token) + allboxes, assembler_call_token=token, + assembler_call_jd=targetjitdriver_sd) opimpl_recursive_call_i = _opimpl_recursive_call opimpl_recursive_call_r = _opimpl_recursive_call @@ -768,24 +768,27 @@ self.generate_guard(rop.GUARD_CLASS, box, [clsbox], resumepc=orgpc) return clsbox - @arguments() - def opimpl_can_enter_jit(self): + @arguments("int") + def opimpl_can_enter_jit(self, jdindex): if self.metainterp.in_recursion: from pypy.jit.metainterp.warmspot import CannotInlineCanEnterJit raise CannotInlineCanEnterJit() + assert jdindex == self.metainterp.jitdriver_sd.index, ( + "found a can_enter_jit that does not match the current jitdriver") self.metainterp.seen_can_enter_jit = True - def verify_green_args(self, varargs): - num_green_args = self.metainterp.staticdata.num_green_args + def verify_green_args(self, jitdriver_sd, varargs): + num_green_args = jitdriver_sd.num_green_args assert len(varargs) == num_green_args for i in range(num_green_args): assert isinstance(varargs[i], Const) - @arguments("orgpc", "boxes3", "boxes3") - def opimpl_jit_merge_point(self, orgpc, greenboxes, redboxes): - self.verify_green_args(greenboxes) + @arguments("orgpc", "int", "boxes3", "boxes3") + def opimpl_jit_merge_point(self, orgpc, jdindex, greenboxes, redboxes): + jitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex] + self.verify_green_args(jitdriver_sd, greenboxes) # xxx we may disable the following line in some context later - self.debug_merge_point(greenboxes) + self.debug_merge_point(jitdriver_sd, greenboxes) if self.metainterp.seen_can_enter_jit: self.metainterp.seen_can_enter_jit = False # Assert that it's impossible to arrive here with in_recursion @@ -793,6 +796,7 @@ # to True by opimpl_can_enter_jit, which should be executed # just before opimpl_jit_merge_point (no recursion inbetween). assert not self.metainterp.in_recursion + assert jitdriver_sd is self.metainterp.jitdriver_sd # Set self.pc to point to jit_merge_point instead of just after: # if reached_can_enter_jit() raises SwitchToBlackhole, then the # pc is still at the jit_merge_point, which is a point that is @@ -802,10 +806,9 @@ self.metainterp.reached_can_enter_jit(greenboxes, redboxes) self.pc = saved_pc - def debug_merge_point(self, greenkey): + def debug_merge_point(self, jitdriver_sd, greenkey): # debugging: produce a DEBUG_MERGE_POINT operation - sd = self.metainterp.staticdata - loc = sd.state.get_location_str(greenkey) + loc = jitdriver_sd.warmstate.get_location_str(greenkey) debug_print(loc) constloc = self.metainterp.cpu.ts.conststr(loc) self.metainterp.history.record(rop.DEBUG_MERGE_POINT, @@ -948,14 +951,15 @@ original_greenkey = metainterp.resumekey.original_greenkey if opnum == rop.GUARD_NOT_FORCED: resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd, - original_greenkey) + original_greenkey, + metainterp.jitdriver_sd) else: resumedescr = compile.ResumeGuardDescr(metainterp_sd, original_greenkey) guard_op = metainterp.history.record(opnum, moreargs, None, descr=resumedescr) virtualizable_boxes = None - if metainterp.staticdata.virtualizable_info is not None: + if metainterp.jitdriver_sd.virtualizable_info is not None: virtualizable_boxes = metainterp.virtualizable_boxes saved_pc = self.pc if resumepc >= 0: @@ -1007,7 +1011,8 @@ return resbox def do_residual_call(self, funcbox, descr, argboxes, - assembler_call_token=None): + assembler_call_token=None, + assembler_call_jd=None): # First build allboxes: it may need some reordering from the # list provided in argboxes, depending on the order in which # the arguments are expected by the function @@ -1052,7 +1057,8 @@ rop.CALL_MAY_FORCE, allboxes, descr=descr) self.metainterp.vrefs_after_residual_call() if assembler_call_token is not None: - self.metainterp.direct_assembler_call(assembler_call_token) + self.metainterp.direct_assembler_call(assembler_call_token, + assembler_call_jd) if resbox is not None: self.make_result_of_lastop(resbox) self.metainterp.vable_after_residual_call() @@ -1137,6 +1143,11 @@ self._addr2name_keys = [key for key, value in list_of_addr2name] self._addr2name_values = [value for key, value in list_of_addr2name] + def setup_jitdrivers_sd(self, optimizer): + if optimizer is not None: + for jd in self.jitdrivers_sd: + jd.warmstate.set_param_optimizer(optimizer) + def finish_setup(self, codewriter, optimizer=None): from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder self.blackholeinterpbuilder = BlackholeInterpBuilder(codewriter, self) @@ -1147,28 +1158,21 @@ self.setup_indirectcalltargets(asm.indirectcalltargets) self.setup_list_of_addr2name(asm.list_of_addr2name) # - self.portal_code = codewriter.mainjitcode - self._portal_runner_ptr = codewriter.callcontrol.portal_runner_ptr + self.jitdrivers_sd = codewriter.callcontrol.jitdrivers_sd self.virtualref_info = codewriter.callcontrol.virtualref_info - self.virtualizable_info = codewriter.callcontrol.virtualizable_info - RESULT = codewriter.portal_graph.getreturnvar().concretetype - self.result_type = history.getkind(RESULT) + self.setup_jitdrivers_sd(optimizer) # # store this information for fastpath of call_assembler - name = self.result_type - tokens = getattr(self, 'loop_tokens_done_with_this_frame_%s' % name) - num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) - setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num) + # (only the paths that can actually be taken) + for jd in self.jitdrivers_sd: + name = {history.INT: 'int', + history.REF: 'ref', + history.FLOAT: 'float', + history.VOID: 'void'}[jd.result_type] + tokens = getattr(self, 'loop_tokens_done_with_this_frame_%s' % name) + num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) + setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num) # - warmrunnerdesc = self.warmrunnerdesc - if warmrunnerdesc is not None: - self.num_green_args = warmrunnerdesc.num_green_args - self.state = warmrunnerdesc.state - if optimizer is not None: - self.state.set_param_optimizer(optimizer) - else: - self.num_green_args = 0 - self.state = None self.globaldata = MetaInterpGlobalData(self) def _setup_once(self): @@ -1189,8 +1193,7 @@ # Build the dictionary at run-time. This is needed # because the keys are function/class addresses, so they # can change from run to run. - k = llmemory.cast_ptr_to_adr(self._portal_runner_ptr) - d = {k: 'recursive call'} + d = {} keys = self._addr2name_keys values = self._addr2name_values for i in range(len(keys)): @@ -1248,47 +1251,31 @@ self.loopnumbering = 0 self.resume_virtuals = {} self.resume_virtuals_not_translated = [] - # - state = staticdata.state - if state is not None: - self.jit_cell_at_key = state.jit_cell_at_key - else: - # for tests only; not RPython - class JitCell: - compiled_merge_points = None - _jitcell_dict = {} - def jit_cell_at_key(greenkey): - greenkey = tuple(greenkey) - return _jitcell_dict.setdefault(greenkey, JitCell()) - self.jit_cell_at_key = jit_cell_at_key - - def get_compiled_merge_points(self, greenkey): - cell = self.jit_cell_at_key(greenkey) - if cell.compiled_merge_points is None: - cell.compiled_merge_points = [] - return cell.compiled_merge_points # ____________________________________________________________ class MetaInterp(object): in_recursion = 0 - def __init__(self, staticdata): + def __init__(self, staticdata, jitdriver_sd): self.staticdata = staticdata self.cpu = staticdata.cpu + self.jitdriver_sd = jitdriver_sd + # Note: self.jitdriver_sd is the JitDriverStaticData that corresponds + # to the current loop -- the outermost one. Be careful, because + # during recursion we can also see other jitdrivers. self.portal_trace_positions = [] self.free_frames_list = [] self.last_exc_value_box = None def perform_call(self, jitcode, boxes, greenkey=None): # causes the metainterp to enter the given subfunction - # with a special case for recursive portal calls f = self.newframe(jitcode, greenkey) f.setup_call(boxes) raise ChangeFrame def newframe(self, jitcode, greenkey=None): - if jitcode is self.staticdata.portal_code: + if jitcode.is_portal: self.in_recursion += 1 if greenkey is not None: self.portal_trace_positions.append( @@ -1303,7 +1290,7 @@ def popframe(self): frame = self.framestack.pop() - if frame.jitcode is self.staticdata.portal_code: + if frame.jitcode.is_portal: self.in_recursion -= 1 if frame.greenkey is not None: self.portal_trace_positions.append( @@ -1327,14 +1314,15 @@ except SwitchToBlackhole, stb: self.aborted_tracing(stb.reason) sd = self.staticdata - if sd.result_type == 'void': + result_type = self.jitdriver_sd.result_type + if result_type == history.VOID: assert resultbox is None raise sd.DoneWithThisFrameVoid() - elif sd.result_type == 'int': + elif result_type == history.INT: raise sd.DoneWithThisFrameInt(resultbox.getint()) - elif sd.result_type == 'ref': + elif result_type == history.REF: raise sd.DoneWithThisFrameRef(self.cpu, resultbox.getref_base()) - elif sd.result_type == 'float': + elif result_type == history.FLOAT: raise sd.DoneWithThisFrameFloat(resultbox.getfloat()) else: assert False @@ -1364,14 +1352,17 @@ in_recursion = -1 for frame in self.framestack: jitcode = frame.jitcode - if jitcode is self.staticdata.portal_code: + assert jitcode.is_portal == len([ + jd for jd in self.staticdata.jitdrivers_sd + if jd.mainjitcode is jitcode]) + if jitcode.is_portal: in_recursion += 1 if in_recursion != self.in_recursion: print "in_recursion problem!!!" print in_recursion, self.in_recursion for frame in self.framestack: jitcode = frame.jitcode - if jitcode is self.staticdata.portal_code: + if jitcode.is_portal: print "P", else: print " ", @@ -1379,7 +1370,6 @@ raise AssertionError def create_empty_history(self): - warmrunnerstate = self.staticdata.state self.history = history.History() self.staticdata.stats.set_history(self.history) @@ -1489,12 +1479,11 @@ self.resumekey.reset_counter_from_failure() def blackhole_if_trace_too_long(self): - warmrunnerstate = self.staticdata.state + warmrunnerstate = self.jitdriver_sd.warmstate if len(self.history.operations) > warmrunnerstate.trace_limit: greenkey_of_huge_function = self.find_biggest_function() self.portal_trace_positions = None if greenkey_of_huge_function is not None: - warmrunnerstate = self.staticdata.state warmrunnerstate.disable_noninlinable_function( greenkey_of_huge_function) raise SwitchToBlackhole(ABORT_TOO_LONG) @@ -1521,21 +1510,27 @@ self.staticdata.log(sys.exc_info()[0].__name__) raise - def compile_and_run_once(self, *args): + @specialize.arg(1) + def compile_and_run_once(self, jitdriver_sd, *args): + # NB. we pass explicity 'jitdriver_sd' around here, even though it + # is also available as 'self.jitdriver_sd', because we need to + # specialize this function and a few other ones for the '*args'. debug_start('jit-tracing') self.staticdata._setup_once() self.staticdata.profiler.start_tracing() + assert jitdriver_sd is self.jitdriver_sd self.create_empty_history() try: - return self._compile_and_run_once(*args) + original_boxes = self.initialize_original_boxes(jitdriver_sd,*args) + return self._compile_and_run_once(original_boxes) finally: self.staticdata.profiler.end_tracing() debug_stop('jit-tracing') - def _compile_and_run_once(self, *args): - original_boxes = self.initialize_state_from_start(*args) + def _compile_and_run_once(self, original_boxes): + self.initialize_state_from_start(original_boxes) self.current_merge_points = [(original_boxes, 0)] - num_green_args = self.staticdata.num_green_args + num_green_args = self.jitdriver_sd.num_green_args original_greenkey = original_boxes[:num_green_args] redkey = original_boxes[num_green_args:] self.resumekey = compile.ResumeFromInterpDescr(original_greenkey, @@ -1602,7 +1597,7 @@ self.remove_consts_and_duplicates(redboxes, len(redboxes), duplicates) live_arg_boxes = greenboxes + redboxes - if self.staticdata.virtualizable_info is not None: + if self.jitdriver_sd.virtualizable_info is not None: # we use pop() to remove the last item, which is the virtualizable # itself self.remove_consts_and_duplicates(self.virtualizable_boxes, @@ -1624,11 +1619,12 @@ # Search in current_merge_points for original_boxes with compatible # green keys, representing the beginning of the same loop as the one # we end now. - + + num_green_args = self.jitdriver_sd.num_green_args for j in range(len(self.current_merge_points)-1, -1, -1): original_boxes, start = self.current_merge_points[j] assert len(original_boxes) == len(live_arg_boxes) or start < 0 - for i in range(self.staticdata.num_green_args): + for i in range(num_green_args): box1 = original_boxes[i] box2 = live_arg_boxes[i] assert isinstance(box1, Const) @@ -1651,7 +1647,7 @@ def designate_target_loop(self, gmp): loop_token = gmp.target_loop_token - num_green_args = self.staticdata.num_green_args + num_green_args = self.jitdriver_sd.num_green_args residual_args = self.get_residual_args(loop_token.specnodes, gmp.argboxes[num_green_args:]) history.set_future_values(self.cpu, residual_args) @@ -1692,12 +1688,17 @@ from pypy.jit.metainterp.resoperation import opname raise NotImplementedError(opname[opnum]) + def get_compiled_merge_points(self, greenkey): + cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey) + if cell.compiled_merge_points is None: + cell.compiled_merge_points = [] + return cell.compiled_merge_points + def compile(self, original_boxes, live_arg_boxes, start): - num_green_args = self.staticdata.num_green_args + num_green_args = self.jitdriver_sd.num_green_args self.history.inputargs = original_boxes[num_green_args:] greenkey = original_boxes[:num_green_args] - glob = self.staticdata.globaldata - old_loop_tokens = glob.get_compiled_merge_points(greenkey) + old_loop_tokens = self.get_compiled_merge_points(greenkey) self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) loop_token = compile.compile_new_loop(self, old_loop_tokens, greenkey, start) @@ -1706,10 +1707,9 @@ self.history.operations.pop() # remove the JUMP def compile_bridge(self, live_arg_boxes): - num_green_args = self.staticdata.num_green_args + num_green_args = self.jitdriver_sd.num_green_args greenkey = live_arg_boxes[:num_green_args] - glob = self.staticdata.globaldata - old_loop_tokens = glob.get_compiled_merge_points(greenkey) + old_loop_tokens = self.get_compiled_merge_points(greenkey) if len(old_loop_tokens) == 0: return self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) @@ -1723,17 +1723,18 @@ self.gen_store_back_in_virtualizable() # temporarily put a JUMP to a pseudo-loop sd = self.staticdata - if sd.result_type == 'void': + result_type = self.jitdriver_sd.result_type + if result_type == history.VOID: assert exitbox is None exits = [] loop_tokens = sd.loop_tokens_done_with_this_frame_void - elif sd.result_type == 'int': + elif result_type == history.INT: exits = [exitbox] loop_tokens = sd.loop_tokens_done_with_this_frame_int - elif sd.result_type == 'ref': + elif result_type == history.REF: exits = [exitbox] loop_tokens = sd.loop_tokens_done_with_this_frame_ref - elif sd.result_type == 'float': + elif result_type == history.FLOAT: exits = [exitbox] loop_tokens = sd.loop_tokens_done_with_this_frame_float else: @@ -1765,26 +1766,32 @@ specnode.extract_runtime_data(self.cpu, args[i], expanded_args) return expanded_args - def _initialize_from_start(self, original_boxes, num_green_args, *args): + @specialize.arg(1) + def initialize_original_boxes(self, jitdriver_sd, *args): + original_boxes = [] + self._fill_original_boxes(jitdriver_sd, original_boxes, + jitdriver_sd.num_green_args, *args) + return original_boxes + + @specialize.arg(1) + def _fill_original_boxes(self, jitdriver_sd, original_boxes, + num_green_args, *args): if args: from pypy.jit.metainterp.warmstate import wrap box = wrap(self.cpu, args[0], num_green_args > 0) original_boxes.append(box) - self._initialize_from_start(original_boxes, num_green_args-1, - *args[1:]) + self._fill_original_boxes(jitdriver_sd, original_boxes, + num_green_args-1, *args[1:]) - def initialize_state_from_start(self, *args): - self.in_recursion = -1 # always one portal around - num_green_args = self.staticdata.num_green_args - original_boxes = [] - self._initialize_from_start(original_boxes, num_green_args, *args) + def initialize_state_from_start(self, original_boxes): # ----- make a new frame ----- + self.in_recursion = -1 # always one portal around self.framestack = [] - f = self.newframe(self.staticdata.portal_code) + f = self.newframe(self.jitdriver_sd.mainjitcode) f.setup_call(original_boxes) + assert self.in_recursion == 0 self.virtualref_boxes = [] self.initialize_virtualizable(original_boxes) - return original_boxes def initialize_state_from_guard_failure(self, resumedescr): # guard failure: rebuild a complete MIFrame stack @@ -1794,9 +1801,11 @@ self.history.inputargs = [box for box in inputargs_and_holes if box] def initialize_virtualizable(self, original_boxes): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info if vinfo is not None: - virtualizable_box = original_boxes[vinfo.index_of_virtualizable] + index = (self.jitdriver_sd.num_green_args + + self.jitdriver_sd.index_of_virtualizable) + virtualizable_box = original_boxes[index] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) # The field 'virtualizable_boxes' is not even present # if 'virtualizable_info' is None. Check for that first. @@ -1807,7 +1816,7 @@ self.initialize_virtualizable_enter() def initialize_virtualizable_enter(self): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.clear_vable_token(virtualizable) @@ -1821,7 +1830,7 @@ # the FORCE_TOKEN is already set at runtime in each vref when # it is created, by optimizeopt.py. # - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) @@ -1847,7 +1856,7 @@ self.stop_tracking_virtualref(i) def vable_after_residual_call(self): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) @@ -1898,7 +1907,7 @@ assert self.last_exc_value_box is None def rebuild_state_after_failure(self, resumedescr): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info self.framestack = [] boxlists = resume.rebuild_from_resumedata(self, resumedescr, vinfo) inputargs_and_holes, virtualizable_boxes, virtualref_boxes = boxlists @@ -1923,24 +1932,19 @@ virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) assert not virtualizable.vable_token - if 0: ## self._already_allocated_resume_virtuals is not None: - # resuming from a ResumeGuardForcedDescr: load the new values - # currently stored on the virtualizable fields - self.load_fields_from_virtualizable() - else: - # normal case: fill the virtualizable with the local boxes - self.synchronize_virtualizable() + # fill the virtualizable with the local boxes + self.synchronize_virtualizable() return inputargs_and_holes def check_synchronized_virtualizable(self): if not we_are_translated(): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.check_boxes(virtualizable, self.virtualizable_boxes) def synchronize_virtualizable(self): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.write_boxes(virtualizable, self.virtualizable_boxes) @@ -1949,7 +1953,7 @@ # Force a reload of the virtualizable fields into the local # boxes (called only in escaping cases). Only call this function # just before SwitchToBlackhole. - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) @@ -1958,7 +1962,7 @@ self.virtualizable_boxes.append(virtualizable_box) def gen_store_back_in_virtualizable(self): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info if vinfo is not None: # xxx only write back the fields really modified vbox = self.virtualizable_boxes[-1] @@ -1979,8 +1983,7 @@ abox, ConstInt(j), itembox) assert i + 1 == len(self.virtualizable_boxes) - def gen_load_from_other_virtualizable(self, vbox): - vinfo = self.staticdata.virtualizable_info + def gen_load_from_other_virtualizable(self, vinfo, vbox): boxes = [] assert vinfo is not None for i in range(vinfo.num_static_extra_boxes): @@ -2004,7 +2007,7 @@ for i in range(len(boxes)): if boxes[i] is oldbox: boxes[i] = newbox - if self.staticdata.virtualizable_info is not None: + if self.jitdriver_sd.virtualizable_info is not None: boxes = self.virtualizable_boxes for i in range(len(boxes)): if boxes[i] is oldbox: @@ -2052,18 +2055,19 @@ op.args = [resbox_as_const] + op.args return resbox - def direct_assembler_call(self, token): + def direct_assembler_call(self, token, targetjitdriver_sd): """ Generate a direct call to assembler for portal entry point, patching the CALL_MAY_FORCE that occurred just now. """ op = self.history.operations.pop() assert op.opnum == rop.CALL_MAY_FORCE - num_green_args = self.staticdata.num_green_args + num_green_args = targetjitdriver_sd.num_green_args args = op.args[num_green_args + 1:] - if self.staticdata.virtualizable_info is not None: - vindex = self.staticdata.virtualizable_info.index_of_virtualizable - vbox = args[vindex - num_green_args] - args = args + self.gen_load_from_other_virtualizable(vbox) + vinfo = targetjitdriver_sd.virtualizable_info + if vinfo is not None: + index = targetjitdriver_sd.index_of_virtualizable + vbox = args[index] + args = args + self.gen_load_from_other_virtualizable(vinfo, vbox) # ^^^ and not "+=", which makes 'args' a resizable list op.opnum = rop.CALL_ASSEMBLER op.args = args @@ -2166,6 +2170,16 @@ position = position3 + 1 + length3 elif argtype == "orgpc": value = orgpc + elif argtype == "int": + argcode = argcodes[next_argcode] + next_argcode = next_argcode + 1 + if argcode == 'i': + value = self.registers_i[ord(code[position])].getint() + elif argcode == 'c': + value = signedord(code[position]) + else: + raise AssertionError("bad argcode") + position += 1 else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/resoperation.py Fri Jul 2 17:44:46 2010 @@ -34,7 +34,7 @@ def clone(self): descr = self.descr if descr is not None: - descr = descr._clone_if_mutable() + descr = descr.clone_if_mutable() op = ResOperation(self.opnum, self.args, self.result, descr) op.fail_args = self.fail_args if not we_are_translated(): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/resume.py Fri Jul 2 17:44:46 2010 @@ -709,11 +709,11 @@ # ---------- when resuming for blackholing, get direct values ---------- -def blackhole_from_resumedata(blackholeinterpbuilder, storage, +def blackhole_from_resumedata(blackholeinterpbuilder, jitdriver_sd, storage, all_virtuals=None): resumereader = ResumeDataDirectReader(blackholeinterpbuilder.cpu, storage, all_virtuals) - vinfo = blackholeinterpbuilder.metainterp_sd.virtualizable_info + vinfo = jitdriver_sd.virtualizable_info vrefinfo = blackholeinterpbuilder.metainterp_sd.virtualref_info resumereader.consume_vref_and_vable(vrefinfo, vinfo) # @@ -745,10 +745,9 @@ resumereader.done() return firstbh -def force_from_resumedata(metainterp_sd, storage): +def force_from_resumedata(metainterp_sd, storage, vinfo=None): resumereader = ResumeDataDirectReader(metainterp_sd.cpu, storage) resumereader.handling_async_forcing() - vinfo = metainterp_sd.virtualizable_info vrefinfo = metainterp_sd.virtualref_info resumereader.consume_vref_and_vable(vrefinfo, vinfo) return resumereader.virtuals Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_basic.py Fri Jul 2 17:44:46 2010 @@ -16,10 +16,18 @@ from pypy.jit.codewriter import support, codewriter from pypy.jit.metainterp import simple_optimize + class FakeJitCell: + compiled_merge_points = None + class FakeWarmRunnerState: def attach_unoptimized_bridge_from_interp(self, greenkey, newloop): pass + def jit_cell_at_key(self, greenkey): + assert greenkey == [] + return self._cell + _cell = FakeJitCell() + # pick the optimizer this way optimize_loop = staticmethod(simple_optimize.optimize_loop) optimize_bridge = staticmethod(simple_optimize.optimize_bridge) @@ -30,14 +38,24 @@ func._jit_unroll_safe_ = True rtyper = support.annotate(func, values, type_system=type_system) graphs = rtyper.annotator.translator.graphs + result_kind = history.getkind(graphs[0].getreturnvar().concretetype)[0] + + class FakeJitDriverSD: + num_green_args = 0 + portal_graph = graphs[0] + virtualizable_info = None + result_type = result_kind + portal_runner_ptr = "???" + stats = history.Stats() cpu = CPUClass(rtyper, stats, None, False) - cw = codewriter.CodeWriter(cpu, graphs[0]) + cw = codewriter.CodeWriter(cpu, [FakeJitDriverSD()]) testself.cw = cw cw.find_all_graphs(JitPolicy()) # testself.warmrunnerstate = FakeWarmRunnerState() testself.warmrunnerstate.cpu = cpu + FakeJitDriverSD.warmstate = testself.warmrunnerstate if hasattr(testself, 'finish_setup_for_interp_operations'): testself.finish_setup_for_interp_operations() # @@ -62,7 +80,8 @@ count_f += 1 else: raise TypeError(T) - blackholeinterp.setposition(cw.mainjitcode, 0) + [jitdriver_sd] = cw.callcontrol.jitdrivers_sd + blackholeinterp.setposition(jitdriver_sd.mainjitcode, 0) blackholeinterp.run() return blackholeinterp._final_result_anytype() @@ -78,16 +97,15 @@ cw = testself.cw opt = history.Options(listops=True) metainterp_sd = pyjitpl.MetaInterpStaticData(cw.cpu, opt) - metainterp_sd.finish_setup(cw, optimizer="bogus") - metainterp_sd.state = testself.warmrunnerstate - metainterp_sd.state.cpu = metainterp_sd.cpu - metainterp = pyjitpl.MetaInterp(metainterp_sd) + metainterp_sd.finish_setup(cw) + [jitdriver_sd] = metainterp_sd.jitdrivers_sd + metainterp = pyjitpl.MetaInterp(metainterp_sd, jitdriver_sd) metainterp_sd.DoneWithThisFrameInt = DoneWithThisFrame metainterp_sd.DoneWithThisFrameRef = DoneWithThisFrameRef metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrame testself.metainterp = metainterp try: - metainterp.compile_and_run_once(*args) + metainterp.compile_and_run_once(jitdriver_sd, *args) except DoneWithThisFrame, e: #if conftest.option.view: # metainterp.stats.view() @@ -864,8 +882,9 @@ translator.config.translation.gc = "boehm" warmrunnerdesc = WarmRunnerDesc(translator, CPUClass=self.CPUClass) - warmrunnerdesc.state.set_param_threshold(3) # for tests - warmrunnerdesc.state.set_param_trace_eagerness(0) # for tests + state = warmrunnerdesc.jitdrivers_sd[0].warmstate + state.set_param_threshold(3) # for tests + state.set_param_trace_eagerness(0) # for tests warmrunnerdesc.finish() for n, k in [(20, 0), (20, 1)]: interp.eval_graph(graph, [n, k]) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_blackhole.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_blackhole.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_blackhole.py Fri Jul 2 17:44:46 2010 @@ -42,7 +42,7 @@ blackholeinterp.setarg_i(0, 40) blackholeinterp.setarg_i(1, 2) blackholeinterp.run() - assert blackholeinterp.final_result_i() == 42 + assert blackholeinterp._final_result_anytype() == 42 def test_simple_const(): jitcode = JitCode("test") @@ -54,7 +54,7 @@ blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(1, 6) blackholeinterp.run() - assert blackholeinterp.final_result_i() == 42 + assert blackholeinterp._final_result_anytype() == 42 def test_simple_bigconst(): jitcode = JitCode("test") @@ -66,7 +66,7 @@ blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(1, 10000) blackholeinterp.run() - assert blackholeinterp.final_result_i() == 42 + assert blackholeinterp._final_result_anytype() == 42 def test_simple_loop(): jitcode = JitCode("test") @@ -85,7 +85,7 @@ blackholeinterp.setarg_i(0x16, 6) # %i0 blackholeinterp.setarg_i(0x17, 100) # %i1 blackholeinterp.run() - assert blackholeinterp.final_result_i() == 100+6+5+4+3 + assert blackholeinterp._final_result_anytype() == 100+6+5+4+3 def test_simple_exception(): jitcode = JitCode("test") @@ -104,12 +104,12 @@ blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(0x9, 100) blackholeinterp.run() - assert blackholeinterp.final_result_i() == 200 + assert blackholeinterp._final_result_anytype() == 200 # blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(0x9, -100) blackholeinterp.run() - assert blackholeinterp.final_result_i() == 42 + assert blackholeinterp._final_result_anytype() == 42 def test_convert_and_run_from_pyjitpl(): class MyMIFrame: Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_compile.py Fri Jul 2 17:44:46 2010 @@ -51,14 +51,14 @@ logger_noopt = FakeLogger() logger_ops = FakeLogger() - state = FakeState() stats = Stats() profiler = jitprof.EmptyProfiler() def log(self, msg, event_kind=None): pass class FakeMetaInterp: - pass + class jitdriver_sd: + warmstate = FakeState() def test_compile_new_loop(): cpu = FakeCPU() Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_list.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_list.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_list.py Fri Jul 2 17:44:46 2010 @@ -143,6 +143,51 @@ assert res == 7 self.check_loops(call=0) + def test_fold_getitem_1(self): + jitdriver = JitDriver(greens = ['pc', 'n', 'l'], reds = ['total']) + def f(n): + l = [100, n, 300, n, 500] + total = 0 + pc = n + while True: + jitdriver.can_enter_jit(l=l, pc=pc, n=n, total=total) + jitdriver.jit_merge_point(l=l, pc=pc, n=n, total=total) + total += l[pc] + if total > 10000: + return total + pc -= 1 + if pc < 0: + pc = n + + res = self.meta_interp(f, [4], listops=True) + assert res == f(4) + self.check_loops(call=0) + + def test_fold_getitem_2(self): + jitdriver = JitDriver(greens = ['pc', 'n', 'l'], reds = ['total', 'x']) + class X: + pass + def f(n): + l = [100, n, 300, n, 500] + total = 0 + x = X() + x.pc = n + while True: + pc = x.pc + jitdriver.can_enter_jit(l=l, pc=pc, n=n, total=total, x=x) + jitdriver.jit_merge_point(l=l, pc=pc, n=n, total=total, x=x) + x.pc = pc + total += l[x.pc] + if total > 10000: + return total + x.pc -= 1 + if x.pc < 0: + x.pc = n + + res = self.meta_interp(f, [4], listops=True) + assert res == f(4) + self.check_loops(call=0, getfield_gc=0) + class TestOOtype(ListTests, OOJitMixin): pass Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py Fri Jul 2 17:44:46 2010 @@ -597,6 +597,25 @@ """ self.optimize_loop(ops, 'Not', ops) + def test_int_is_true_of_bool(self): + ops = """ + [i0, i1] + i2 = int_gt(i0, i1) + i3 = int_is_true(i2) + i4 = int_is_true(i3) + guard_value(i4, 0) [i0, i1] + jump(i0, i1) + """ + expected = """ + [i0, i1] + i2 = int_gt(i0, i1) + guard_false(i2) [i0, i1] + jump(i0, i1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + + def test_p123_simple(self): ops = """ Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_pyjitpl.py Fri Jul 2 17:44:46 2010 @@ -17,9 +17,8 @@ portal.setup(None) class FakeStaticData: cpu = None - portal_code = portal - metainterp = pyjitpl.MetaInterp(FakeStaticData()) + metainterp = pyjitpl.MetaInterp(FakeStaticData(), None) metainterp.framestack = [] class FakeHistory: operations = [] @@ -59,7 +58,7 @@ assert isinstance(box, referencebox.clonebox().__class__) assert box.value == referencebox.value return True - metainterp = pyjitpl.MetaInterp(FakeStaticData()) + metainterp = pyjitpl.MetaInterp(FakeStaticData(), None) metainterp.history = History() b1 = BoxInt(1) b2 = BoxInt(2) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_tl.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_tl.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_tl.py Fri Jul 2 17:44:46 2010 @@ -74,8 +74,7 @@ backendopt=True) assert res == 5040 self.check_loops({'int_mul':1, 'jump':1, - 'int_sub':1, 'int_is_true':1, 'int_le':1, - 'guard_false':1}) + 'int_sub':1, 'int_le':1, 'guard_false':1}) def test_tl_2(self): main = self._get_main() @@ -83,7 +82,7 @@ backendopt=True) assert res == main(1, 10) self.check_loops({'int_sub':1, 'int_le':1, - 'int_is_true':1, 'guard_false':1, 'jump':1}) + 'guard_false':1, 'jump':1}) def test_tl_call(self, listops=True, policy=None): from pypy.jit.tl.tl import interp Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_warmspot.py Fri Jul 2 17:44:46 2010 @@ -128,8 +128,9 @@ from pypy.jit.metainterp import optimize - assert warmrunnerdescr.state.optimize_loop is optimize.optimize_loop - assert warmrunnerdescr.state.optimize_bridge is optimize.optimize_bridge + state = warmrunnerdescr.jitdrivers_sd[0].warmstate + assert state.optimize_loop is optimize.optimize_loop + assert state.optimize_bridge is optimize.optimize_bridge def test_static_debug_level(self, capfd): py.test.skip("debug_level is being deprecated") @@ -294,7 +295,7 @@ def __init__(self, no): self.no = no - def handle_fail(self, metainterp_sd): + def handle_fail(self, metainterp_sd, jitdrivers_sd): if self.no == 0: raise metainterp_sd.warmrunnerdesc.DoneWithThisFrameInt(3) if self.no == 1: @@ -355,12 +356,13 @@ def test_call_helper(self): from pypy.rpython.llinterp import LLException - - assert self.desc.assembler_call_helper(0, 0) == 3 - assert self.desc.assembler_call_helper(1, 0) == 10 - assert self.desc.assembler_call_helper(2, 0) == 10 + + [jd] = self.desc.jitdrivers_sd + assert jd._assembler_call_helper(0, 0) == 3 + assert jd._assembler_call_helper(1, 0) == 10 + assert jd._assembler_call_helper(2, 0) == 10 try: - self.desc.assembler_call_helper(3, 0) + jd._assembler_call_helper(3, 0) except LLException, lle: assert lle[0] == self.exc_vtable else: Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_warmstate.py Fri Jul 2 17:44:46 2010 @@ -4,10 +4,9 @@ from pypy.rpython.annlowlevel import llhelper from pypy.jit.metainterp.warmstate import wrap, unwrap from pypy.jit.metainterp.warmstate import equal_whatever, hash_whatever -from pypy.jit.metainterp.warmstate import WarmEnterState +from pypy.jit.metainterp.warmstate import WarmEnterState, JitCell from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr from pypy.jit.metainterp.history import ConstInt, ConstFloat, ConstPtr -from pypy.rlib.jit import BaseJitCell def test_unwrap(): @@ -58,14 +57,12 @@ def test_make_jitcell_getter_default(): - class FakeWarmRunnerDesc: - green_args_spec = [lltype.Signed, lltype.Float] - class FakeJitCell(BaseJitCell): - pass - state = WarmEnterState(FakeWarmRunnerDesc()) - get_jitcell = state._make_jitcell_getter_default(FakeJitCell) + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + state = WarmEnterState(None, FakeJitDriverSD()) + get_jitcell = state._make_jitcell_getter_default() cell1 = get_jitcell(42, 42.5) - assert isinstance(cell1, FakeJitCell) + assert isinstance(cell1, JitCell) cell2 = get_jitcell(42, 42.5) assert cell1 is cell2 cell3 = get_jitcell(41, 42.5) @@ -73,10 +70,10 @@ assert cell1 is not cell3 is not cell4 is not cell1 def test_make_jitcell_getter(): - class FakeWarmRunnerDesc: - green_args_spec = [lltype.Float] - get_jitcell_at_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc()) + class FakeJitDriverSD: + _green_args_spec = [lltype.Float] + _get_jitcell_at_ptr = None + state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state.make_jitcell_getter() cell1 = get_jitcell(1.75) cell2 = get_jitcell(1.75) @@ -87,8 +84,6 @@ from pypy.rpython.typesystem import LowLevelTypeSystem class FakeRTyper: type_system = LowLevelTypeSystem.instance - class FakeJitCell(BaseJitCell): - pass celldict = {} def getter(x, y): return celldict.get((x, y)) @@ -102,14 +97,14 @@ lltype.Float], lltype.Void)) class FakeWarmRunnerDesc: rtyper = FakeRTyper() - cpu = None - get_jitcell_at_ptr = llhelper(GETTER, getter) - set_jitcell_at_ptr = llhelper(SETTER, setter) + class FakeJitDriverSD: + _get_jitcell_at_ptr = llhelper(GETTER, getter) + _set_jitcell_at_ptr = llhelper(SETTER, setter) # - state = WarmEnterState(FakeWarmRunnerDesc()) - get_jitcell = state._make_jitcell_getter_custom(FakeJitCell) + state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) + get_jitcell = state._make_jitcell_getter_custom() cell1 = get_jitcell(5, 42.5) - assert isinstance(cell1, FakeJitCell) + assert isinstance(cell1, JitCell) assert cell1.x == 5 assert cell1.y == 42.5 cell2 = get_jitcell(5, 42.5) @@ -127,10 +122,11 @@ future_values[j] = "float", value class FakeWarmRunnerDesc: cpu = FakeCPU() - red_args_types = ["int", "float"] + class FakeJitDriverSD: + _red_args_types = ["int", "float"] virtualizable_info = None # - state = WarmEnterState(FakeWarmRunnerDesc()) + state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) set_future_values = state.make_set_future_values() set_future_values(5, 42.5) assert future_values == { @@ -140,19 +136,19 @@ assert set_future_values is state.make_set_future_values() def test_make_unwrap_greenkey(): - class FakeWarmRunnerDesc: - green_args_spec = [lltype.Signed, lltype.Float] - state = WarmEnterState(FakeWarmRunnerDesc()) + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + state = WarmEnterState(None, FakeJitDriverSD()) unwrap_greenkey = state.make_unwrap_greenkey() greenargs = unwrap_greenkey([ConstInt(42), ConstFloat(42.5)]) assert greenargs == (42, 42.5) assert type(greenargs[0]) is int def test_attach_unoptimized_bridge_from_interp(): - class FakeWarmRunnerDesc: - green_args_spec = [lltype.Signed, lltype.Float] - get_jitcell_at_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc()) + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + _get_jitcell_at_ptr = None + state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state.make_jitcell_getter() state.attach_unoptimized_bridge_from_interp([ConstInt(5), ConstFloat(2.25)], @@ -162,14 +158,14 @@ assert cell1.entry_loop_token == "entry loop token" def test_make_jitdriver_callbacks_1(): - class FakeWarmRunnerDesc: - can_inline_ptr = None - get_printable_location_ptr = None - confirm_enter_jit_ptr = None - green_args_spec = [lltype.Signed, lltype.Float] + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + _can_inline_ptr = None + _get_printable_location_ptr = None + _confirm_enter_jit_ptr = None class FakeCell: dont_trace_here = False - state = WarmEnterState(FakeWarmRunnerDesc()) + state = WarmEnterState(None, FakeJitDriverSD()) def jit_getter(*args): return FakeCell() state.jit_getter = jit_getter @@ -190,11 +186,12 @@ dont_trace_here = False class FakeWarmRunnerDesc: rtyper = None - green_args_spec = [lltype.Signed, lltype.Float] - can_inline_ptr = llhelper(CAN_INLINE, can_inline) - get_printable_location_ptr = None - confirm_enter_jit_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc()) + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + _can_inline_ptr = llhelper(CAN_INLINE, can_inline) + _get_printable_location_ptr = None + _confirm_enter_jit_ptr = None + state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) def jit_getter(*args): return FakeCell() state.jit_getter = jit_getter @@ -211,12 +208,13 @@ lltype.Ptr(rstr.STR))) class FakeWarmRunnerDesc: rtyper = None - green_args_spec = [lltype.Signed, lltype.Float] - can_inline_ptr = None - get_printable_location_ptr = llhelper(GET_LOCATION, get_location) - confirm_enter_jit_ptr = None - get_jitcell_at_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc()) + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + _can_inline_ptr = None + _get_printable_location_ptr = llhelper(GET_LOCATION, get_location) + _confirm_enter_jit_ptr = None + _get_jitcell_at_ptr = None + state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) state.make_jitdriver_callbacks() res = state.get_location_str([ConstInt(5), ConstFloat(42.5)]) assert res == "hi there" @@ -231,13 +229,14 @@ lltype.Signed], lltype.Bool)) class FakeWarmRunnerDesc: rtyper = None - green_args_spec = [lltype.Signed, lltype.Float] - can_inline_ptr = None - get_printable_location_ptr = None - confirm_enter_jit_ptr = llhelper(ENTER_JIT, confirm_enter_jit) - get_jitcell_at_ptr = None + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + _can_inline_ptr = None + _get_printable_location_ptr = None + _confirm_enter_jit_ptr = llhelper(ENTER_JIT, confirm_enter_jit) + _get_jitcell_at_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc()) + state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) state.make_jitdriver_callbacks() res = state.confirm_enter_jit(5, 42.5, 3) assert res is True Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_ztranslation.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_ztranslation.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_ztranslation.py Fri Jul 2 17:44:46 2010 @@ -20,6 +20,7 @@ # - profiler # - full optimizer # - jitdriver hooks + # - two JITs class Frame(object): _virtualizable2_ = ['i'] @@ -61,14 +62,28 @@ frame.i -= 2 frame.i -= 1 return total * 10 - res = ll_meta_interp(f, [40], CPUClass=self.CPUClass, + # + myjitdriver2 = JitDriver(greens = ['g'], reds = ['m', 'x'], + can_inline = lambda *args: False) + def f2(g, m, x): + while m > 0: + myjitdriver2.can_enter_jit(g=g, m=m, x=x) + myjitdriver2.jit_merge_point(g=g, m=m, x=x) + m -= 1 + x += 3 + return x + # + def main(i, j): + return f(i) - f2(i+j, i, j) + res = ll_meta_interp(main, [40, 5], CPUClass=self.CPUClass, type_system=self.type_system) - assert res == f(40) - res = rpython_ll_meta_interp(f, [40], loops=2, CPUClass=self.CPUClass, + assert res == main(40, 5) + res = rpython_ll_meta_interp(main, [40, 5], loops=2, + CPUClass=self.CPUClass, type_system=self.type_system, optimizer=OPTIMIZER_FULL, ProfilerClass=Profiler) - assert res == f(40) + assert res == main(40, 5) def test_external_exception_handling_translates(self): jitdriver = JitDriver(greens = [], reds = ['n', 'total']) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/viewnode.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/viewnode.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/viewnode.py Fri Jul 2 17:44:46 2010 @@ -55,7 +55,7 @@ if isinstance(self.knownclsbox.value, int): name += str(self.knownclsbox.value) else: - name += str(self.knownclsbox.value.ptr).rpartition("_vtable")[0].rpartition('.')[2] + name += str(self.knownclsbox.value.adr.ptr).rpartition("_vtable")[0].rpartition('.')[2] elif self.structdescr: name = "Struct " + str(self.structdescr) elif self.arraydescr: Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/virtualizable.py Fri Jul 2 17:44:46 2010 @@ -14,9 +14,8 @@ TOKEN_NONE = 0 # must be 0 -- see also x86.call_assembler TOKEN_TRACING_RESCALL = -1 - def __init__(self, warmrunnerdesc): + def __init__(self, warmrunnerdesc, VTYPEPTR): self.warmrunnerdesc = warmrunnerdesc - jitdriver = warmrunnerdesc.jitdriver cpu = warmrunnerdesc.cpu if cpu.ts.name == 'ootype': import py @@ -24,11 +23,6 @@ self.cpu = cpu self.BoxArray = cpu.ts.BoxRef # - assert len(jitdriver.virtualizables) == 1 # for now - [vname] = jitdriver.virtualizables - index = len(jitdriver.greens) + jitdriver.reds.index(vname) - self.index_of_virtualizable = index - VTYPEPTR = warmrunnerdesc.JIT_ENTER_FUNCTYPE.ARGS[index] while 'virtualizable2_accessor' not in deref(VTYPEPTR)._hints: VTYPEPTR = cpu.ts.get_superclass(VTYPEPTR) self.VTYPEPTR = VTYPEPTR Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmspot.py Fri Jul 2 17:44:46 2010 @@ -21,6 +21,7 @@ from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler from pypy.jit.metainterp.jitexc import JitException +from pypy.jit.metainterp.jitdriver import JitDriverStaticData from pypy.jit.codewriter import support, codewriter from pypy.jit.codewriter.policy import JitPolicy from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE @@ -44,8 +45,9 @@ no_stats = True, ProfilerClass = ProfilerClass, **kwds) - warmrunnerdesc.state.set_param_inlining(inline) - warmrunnerdesc.state.set_param_debug(debug_level) + for jd in warmrunnerdesc.jitdrivers_sd: + jd.warmstate.set_param_inlining(inline) + jd.warmstate.set_param_debug(debug_level) warmrunnerdesc.finish() translator.warmrunnerdesc = warmrunnerdesc # for later debugging @@ -69,11 +71,12 @@ translator.config.translation.gc = "boehm" translator.config.translation.list_comprehension_operations = True warmrunnerdesc = WarmRunnerDesc(translator, backendopt=backendopt, **kwds) - warmrunnerdesc.state.set_param_threshold(3) # for tests - warmrunnerdesc.state.set_param_trace_eagerness(2) # for tests - warmrunnerdesc.state.set_param_trace_limit(trace_limit) - warmrunnerdesc.state.set_param_inlining(inline) - warmrunnerdesc.state.set_param_debug(debug_level) + for jd in warmrunnerdesc.jitdrivers_sd: + jd.warmstate.set_param_threshold(3) # for tests + jd.warmstate.set_param_trace_eagerness(2) # for tests + jd.warmstate.set_param_trace_limit(trace_limit) + jd.warmstate.set_param_inlining(inline) + jd.warmstate.set_param_debug(debug_level) warmrunnerdesc.finish() res = interp.eval_graph(graph, args) if not kwds.get('translate_support_code', False): @@ -110,12 +113,11 @@ raise Exception("no can_enter_jit found!") return results -def find_jit_merge_point(graphs): +def find_jit_merge_points(graphs): results = _find_jit_marker(graphs, 'jit_merge_point') - if len(results) != 1: - raise Exception("found %d jit_merge_points, need exactly one!" % - (len(results),)) - return results[0] + if not results: + raise Exception("no jit_merge_point found!") + return results def find_set_param(graphs): return _find_jit_marker(graphs, 'set_param') @@ -146,8 +148,8 @@ pyjitpl._warmrunnerdesc = self # this is a global for debugging only! self.set_translator(translator) self.build_cpu(CPUClass, **kwds) - self.find_portal() - self.codewriter = codewriter.CodeWriter(self.cpu, self.portal_graph) + self.find_portals() + self.codewriter = codewriter.CodeWriter(self.cpu, self.jitdrivers_sd) if policy is None: policy = JitPolicy() policy.set_supports_floats(self.cpu.supports_floats) @@ -158,35 +160,31 @@ self.prejit_optimizations(policy, graphs) self.build_meta_interp(ProfilerClass) - self.make_args_specification() + self.make_args_specifications() # from pypy.jit.metainterp.virtualref import VirtualRefInfo vrefinfo = VirtualRefInfo(self) self.codewriter.setup_vrefinfo(vrefinfo) - if self.jitdriver.virtualizables: - from pypy.jit.metainterp.virtualizable import VirtualizableInfo - self.virtualizable_info = VirtualizableInfo(self) - self.codewriter.setup_virtualizable_info(self.virtualizable_info) - else: - self.virtualizable_info = None # + self.make_virtualizable_infos() self.make_exception_classes() self.make_driverhook_graphs() - self.make_enter_function() - self.rewrite_jit_merge_point(policy) + self.make_enter_functions() + self.rewrite_jit_merge_points(policy) verbose = not self.cpu.translate_support_code self.codewriter.make_jitcodes(verbose=verbose) - self.rewrite_can_enter_jit() + self.rewrite_can_enter_jits() self.rewrite_set_param() self.rewrite_force_virtual(vrefinfo) self.add_finish() self.metainterp_sd.finish_setup(self.codewriter, optimizer=optimizer) def finish(self): - vinfo = self.virtualizable_info - if vinfo is not None: - vinfo.finish() + vinfos = set([jd.virtualizable_info for jd in self.jitdrivers_sd]) + for vinfo in vinfos: + if vinfo is not None: + vinfo.finish() if self.cpu.translate_support_code: self.annhelper.finish() @@ -198,18 +196,27 @@ self.rtyper = translator.rtyper self.gcdescr = gc.get_description(translator.config) - def find_portal(self): + def find_portals(self): + self.jitdrivers_sd = [] graphs = self.translator.graphs - self.jit_merge_point_pos = find_jit_merge_point(graphs) - graph, block, pos = self.jit_merge_point_pos + for jit_merge_point_pos in find_jit_merge_points(graphs): + self.split_graph_and_record_jitdriver(*jit_merge_point_pos) + # + assert (len(set([jd.jitdriver for jd in self.jitdrivers_sd])) == + len(self.jitdrivers_sd)), \ + "there are multiple jit_merge_points with the same jitdriver" + + def split_graph_and_record_jitdriver(self, graph, block, pos): + jd = JitDriverStaticData() + jd._jit_merge_point_pos = (graph, block, pos) op = block.operations[pos] args = op.args[2:] s_binding = self.translator.annotator.binding - self.portal_args_s = [s_binding(v) for v in args] + jd._portal_args_s = [s_binding(v) for v in args] graph = copygraph(graph) graph.startblock.isstartblock = False - graph.startblock = support.split_before_jit_merge_point( - *find_jit_merge_point([graph])) + [jmpp] = find_jit_merge_points([graph]) + graph.startblock = support.split_before_jit_merge_point(*jmpp) graph.startblock.isstartblock = True # a crash in the following checkgraph() means that you forgot # to list some variable in greens=[] or reds=[] in JitDriver. @@ -218,12 +225,16 @@ assert isinstance(v, Variable) assert len(dict.fromkeys(graph.getargs())) == len(graph.getargs()) self.translator.graphs.append(graph) - self.portal_graph = graph + jd.portal_graph = graph # it's a bit unbelievable to have a portal without func assert hasattr(graph, "func") graph.func._dont_inline_ = True graph.func._jit_unroll_safe_ = True - self.jitdriver = block.operations[pos].args[1].value + jd.jitdriver = block.operations[pos].args[1].value + jd.portal_runner_ptr = "" + jd.result_type = history.getkind(jd.portal_graph.getreturnvar() + .concretetype)[0] + self.jitdrivers_sd.append(jd) def check_access_directly_sanity(self, graphs): from pypy.translator.backendopt.inline import collect_called_graphs @@ -268,6 +279,27 @@ ProfilerClass=ProfilerClass, warmrunnerdesc=self) + def make_virtualizable_infos(self): + vinfos = {} + for jd in self.jitdrivers_sd: + if not jd.jitdriver.virtualizables: + jd.virtualizable_info = None + jd.index_of_virtualizable = -1 + continue + # + jitdriver = jd.jitdriver + assert len(jitdriver.virtualizables) == 1 # for now + [vname] = jitdriver.virtualizables + # XXX skip the Voids here too + jd.index_of_virtualizable = jitdriver.reds.index(vname) + # + index = jd.num_green_args + jd.index_of_virtualizable + VTYPEPTR = jd._JIT_ENTER_FUNCTYPE.ARGS[index] + if VTYPEPTR not in vinfos: + from pypy.jit.metainterp.virtualizable import VirtualizableInfo + vinfos[VTYPEPTR] = VirtualizableInfo(self, VTYPEPTR) + jd.virtualizable_info = vinfos[VTYPEPTR] + def make_exception_classes(self): class DoneWithThisFrameVoid(JitException): @@ -317,6 +349,8 @@ self.green_int, self.green_ref, self.green_float, self.red_int, self.red_ref, self.red_float) + # XXX there is no point any more to not just have the exceptions + # as globals self.DoneWithThisFrameVoid = DoneWithThisFrameVoid self.DoneWithThisFrameInt = DoneWithThisFrameInt self.DoneWithThisFrameRef = DoneWithThisFrameRef @@ -330,11 +364,15 @@ self.metainterp_sd.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef self.metainterp_sd.ContinueRunningNormally = ContinueRunningNormally - def make_enter_function(self): + def make_enter_functions(self): + for jd in self.jitdrivers_sd: + self.make_enter_function(jd) + + def make_enter_function(self, jd): from pypy.jit.metainterp.warmstate import WarmEnterState - state = WarmEnterState(self) + state = WarmEnterState(self, jd) maybe_compile_and_run = state.make_entry_point() - self.state = state + jd.warmstate = state def crash_in_jit(e): if not we_are_translated(): @@ -359,15 +397,16 @@ def maybe_enter_jit(*args): maybe_compile_and_run(*args) maybe_enter_jit._always_inline_ = True - self.maybe_enter_jit_fn = maybe_enter_jit + jd._maybe_enter_jit_fn = maybe_enter_jit - can_inline = self.state.can_inline_greenargs + can_inline = state.can_inline_greenargs + num_green_args = jd.num_green_args def maybe_enter_from_start(*args): - if can_inline is not None and not can_inline(*args[:self.num_green_args]): + if can_inline is not None and not can_inline(*args[:num_green_args]): maybe_compile_and_run(*args) maybe_enter_from_start._always_inline_ = True - self.maybe_enter_from_start_fn = maybe_enter_from_start - + jd._maybe_enter_from_start_fn = maybe_enter_from_start + def make_driverhook_graphs(self): from pypy.rlib.jit import BaseJitCell bk = self.rtyper.annotator.bookkeeper @@ -378,22 +417,23 @@ s_Str = annmodel.SomeString() # annhelper = MixLevelHelperAnnotator(self.translator.rtyper) - self.set_jitcell_at_ptr = self._make_hook_graph( - annhelper, self.jitdriver.set_jitcell_at, annmodel.s_None, - s_BaseJitCell_not_None) - self.get_jitcell_at_ptr = self._make_hook_graph( - annhelper, self.jitdriver.get_jitcell_at, s_BaseJitCell_or_None) - self.can_inline_ptr = self._make_hook_graph( - annhelper, self.jitdriver.can_inline, annmodel.s_Bool) - self.get_printable_location_ptr = self._make_hook_graph( - annhelper, self.jitdriver.get_printable_location, s_Str) - self.confirm_enter_jit_ptr = self._make_hook_graph( - annhelper, self.jitdriver.confirm_enter_jit, annmodel.s_Bool, - onlygreens=False) + for jd in self.jitdrivers_sd: + jd._set_jitcell_at_ptr = self._make_hook_graph(jd, + annhelper, jd.jitdriver.set_jitcell_at, annmodel.s_None, + s_BaseJitCell_not_None) + jd._get_jitcell_at_ptr = self._make_hook_graph(jd, + annhelper, jd.jitdriver.get_jitcell_at, s_BaseJitCell_or_None) + jd._can_inline_ptr = self._make_hook_graph(jd, + annhelper, jd.jitdriver.can_inline, annmodel.s_Bool) + jd._get_printable_location_ptr = self._make_hook_graph(jd, + annhelper, jd.jitdriver.get_printable_location, s_Str) + jd._confirm_enter_jit_ptr = self._make_hook_graph(jd, + annhelper, jd.jitdriver.confirm_enter_jit, annmodel.s_Bool, + onlygreens=False) annhelper.finish() - def _make_hook_graph(self, annhelper, func, s_result, s_first_arg=None, - onlygreens=True): + def _make_hook_graph(self, jitdriver_sd, annhelper, func, + s_result, s_first_arg=None, onlygreens=True): if func is None: return None # @@ -401,38 +441,57 @@ if s_first_arg is not None: extra_args_s.append(s_first_arg) # - args_s = self.portal_args_s + args_s = jitdriver_sd._portal_args_s if onlygreens: - args_s = args_s[:len(self.green_args_spec)] + args_s = args_s[:len(jitdriver_sd._green_args_spec)] graph = annhelper.getgraph(func, extra_args_s + args_s, s_result) funcptr = annhelper.graph2delayed(graph) return funcptr - def make_args_specification(self): - graph, block, index = self.jit_merge_point_pos + def make_args_specifications(self): + for jd in self.jitdrivers_sd: + self.make_args_specification(jd) + + def make_args_specification(self, jd): + graph, block, index = jd._jit_merge_point_pos op = block.operations[index] greens_v, reds_v = support.decode_hp_hint_args(op) ALLARGS = [v.concretetype for v in (greens_v + reds_v)] - self.green_args_spec = [v.concretetype for v in greens_v] - self.red_args_types = [history.getkind(v.concretetype) for v in reds_v] - self.num_green_args = len(self.green_args_spec) + jd._green_args_spec = [v.concretetype for v in greens_v] + jd._red_args_types = [history.getkind(v.concretetype) for v in reds_v] + jd.num_green_args = len(jd._green_args_spec) RESTYPE = graph.getreturnvar().concretetype - (self.JIT_ENTER_FUNCTYPE, - self.PTR_JIT_ENTER_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, lltype.Void) - (self.PORTAL_FUNCTYPE, - self.PTR_PORTAL_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, RESTYPE) - (_, self.PTR_ASSEMBLER_HELPER_FUNCTYPE) = self.cpu.ts.get_FuncType( + (jd._JIT_ENTER_FUNCTYPE, + jd._PTR_JIT_ENTER_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, lltype.Void) + (jd._PORTAL_FUNCTYPE, + jd._PTR_PORTAL_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, RESTYPE) + (_, jd._PTR_ASSEMBLER_HELPER_FUNCTYPE) = self.cpu.ts.get_FuncType( [lltype.Signed, llmemory.GCREF], RESTYPE) - def rewrite_can_enter_jit(self): - FUNC = self.JIT_ENTER_FUNCTYPE - FUNCPTR = self.PTR_JIT_ENTER_FUNCTYPE - jit_enter_fnptr = self.helper_func(FUNCPTR, self.maybe_enter_jit_fn) + def rewrite_can_enter_jits(self): + can_enter_jits = find_can_enter_jit(self.translator.graphs) + sublists = {} + for jd in self.jitdrivers_sd: + sublists[jd.jitdriver] = [] + for graph, block, index in can_enter_jits: + op = block.operations[index] + jitdriver = op.args[1].value + assert jitdriver in sublists, \ + "can_enter_jit with no matching jit_merge_point" + sublists[jitdriver].append((graph, block, index)) + for jd in self.jitdrivers_sd: + sublist = sublists[jd.jitdriver] + assert len(sublist) > 0, \ + "found no can_enter_jit for %r" % (jd.jitdriver,) + self.rewrite_can_enter_jit(jd, sublist) + + def rewrite_can_enter_jit(self, jd, can_enter_jits): + FUNC = jd._JIT_ENTER_FUNCTYPE + FUNCPTR = jd._PTR_JIT_ENTER_FUNCTYPE + jit_enter_fnptr = self.helper_func(FUNCPTR, jd._maybe_enter_jit_fn) - graphs = self.translator.graphs - can_enter_jits = find_can_enter_jit(graphs) for graph, block, index in can_enter_jits: - if graph is self.jit_merge_point_pos[0]: + if graph is jd._jit_merge_point_pos[0]: continue op = block.operations[index] @@ -455,7 +514,11 @@ graph = self.annhelper.getgraph(func, args_s, s_result) return self.annhelper.graph2delayed(graph, FUNC) - def rewrite_jit_merge_point(self, policy): + def rewrite_jit_merge_points(self, policy): + for jd in self.jitdrivers_sd: + self.rewrite_jit_merge_point(jd, policy) + + def rewrite_jit_merge_point(self, jd, policy): # # Mutate the original portal graph from this: # @@ -486,9 +549,9 @@ # while 1: # more stuff # - origportalgraph = self.jit_merge_point_pos[0] - portalgraph = self.portal_graph - PORTALFUNC = self.PORTAL_FUNCTYPE + origportalgraph = jd._jit_merge_point_pos[0] + portalgraph = jd.portal_graph + PORTALFUNC = jd._PORTAL_FUNCTYPE # ____________________________________________________________ # Prepare the portal_runner() helper @@ -496,12 +559,12 @@ from pypy.jit.metainterp.warmstate import specialize_value portal_ptr = self.cpu.ts.functionptr(PORTALFUNC, 'portal', graph = portalgraph) - self.portal_ptr = portal_ptr + jd._portal_ptr = portal_ptr # portalfunc_ARGS = [] nums = {} for i, ARG in enumerate(PORTALFUNC.ARGS): - if i < len(self.jitdriver.greens): + if i < len(jd.jitdriver.greens): color = 'green' else: color = 'red' @@ -519,7 +582,7 @@ def ll_portal_runner(*args): while 1: try: - self.maybe_enter_from_start_fn(*args) + jd._maybe_enter_from_start_fn(*args) return support.maybe_on_top_of_llinterp(rtyper, portal_ptr)(*args) except self.ContinueRunningNormally, e: @@ -548,27 +611,26 @@ value = cast_base_ptr_to_instance(Exception, value) raise Exception, value - self.ll_portal_runner = ll_portal_runner # for debugging - self.portal_runner_ptr = self.helper_func(self.PTR_PORTAL_FUNCTYPE, - ll_portal_runner) + jd._ll_portal_runner = ll_portal_runner # for debugging + jd.portal_runner_ptr = self.helper_func(jd._PTR_PORTAL_FUNCTYPE, + ll_portal_runner) + jd.portal_runner_adr = llmemory.cast_ptr_to_adr(jd.portal_runner_ptr) self.cpu.portal_calldescr = self.cpu.calldescrof( - self.PTR_PORTAL_FUNCTYPE.TO, - self.PTR_PORTAL_FUNCTYPE.TO.ARGS, - self.PTR_PORTAL_FUNCTYPE.TO.RESULT) - self.codewriter.setup_portal_runner_ptr(self.portal_runner_ptr) + jd._PTR_PORTAL_FUNCTYPE.TO, + jd._PTR_PORTAL_FUNCTYPE.TO.ARGS, + jd._PTR_PORTAL_FUNCTYPE.TO.RESULT) - vinfo = self.virtualizable_info + vinfo = jd.virtualizable_info def assembler_call_helper(failindex, virtualizableref): fail_descr = self.cpu.get_fail_descr_from_number(failindex) while True: + if vinfo is not None: + virtualizable = lltype.cast_opaque_ptr( + vinfo.VTYPEPTR, virtualizableref) + vinfo.reset_vable_token(virtualizable) try: - if vinfo is not None: - virtualizable = lltype.cast_opaque_ptr( - vinfo.VTYPEPTR, virtualizableref) - vinfo.reset_vable_token(virtualizable) - loop_token = fail_descr.handle_fail(self.metainterp_sd) - fail_descr = self.cpu.execute_token(loop_token) + loop_token = fail_descr.handle_fail(self.metainterp_sd, jd) except self.ContinueRunningNormally, e: args = () for ARGTYPE, attrname, count in portalfunc_ARGS: @@ -595,29 +657,26 @@ else: value = cast_base_ptr_to_instance(Exception, value) raise Exception, value + fail_descr = self.cpu.execute_token(loop_token) - self.assembler_call_helper = assembler_call_helper # for debugging - self.cpu.assembler_helper_ptr = self.helper_func( - self.PTR_ASSEMBLER_HELPER_FUNCTYPE, + jd._assembler_call_helper = assembler_call_helper # for debugging + jd._assembler_helper_ptr = self.helper_func( + jd._PTR_ASSEMBLER_HELPER_FUNCTYPE, assembler_call_helper) - # XXX a bit ugly sticking + jd.assembler_helper_adr = llmemory.cast_ptr_to_adr( + jd._assembler_helper_ptr) if vinfo is not None: - self.cpu.index_of_virtualizable = (vinfo.index_of_virtualizable - - self.num_green_args) - self.cpu.vable_token_descr = vinfo.vable_token_descr - else: - self.cpu.index_of_virtualizable = -1 - self.cpu.vable_token_descr = None + jd.vable_token_descr = vinfo.vable_token_descr # ____________________________________________________________ # Now mutate origportalgraph to end with a call to portal_runner_ptr # - _, origblock, origindex = self.jit_merge_point_pos + _, origblock, origindex = jd._jit_merge_point_pos op = origblock.operations[origindex] assert op.opname == 'jit_marker' assert op.args[0].value == 'jit_merge_point' greens_v, reds_v = support.decode_hp_hint_args(op) - vlist = [Constant(self.portal_runner_ptr, self.PTR_PORTAL_FUNCTYPE)] + vlist = [Constant(jd.portal_runner_ptr, jd._PTR_PORTAL_FUNCTYPE)] vlist += greens_v vlist += reds_v v_result = Variable() @@ -627,6 +686,18 @@ origblock.operations.append(newop) origblock.exitswitch = None origblock.recloseblock(Link([v_result], origportalgraph.returnblock)) + # + # Also kill any can_enter_jit left behind (example: see + # test_jitdriver.test_simple, which has a can_enter_jit in + # loop1's origportalgraph) + can_enter_jits = _find_jit_marker([origportalgraph], 'can_enter_jit') + for _, block, i in can_enter_jits: + op = block.operations[i] + assert op.opname == 'jit_marker' + block.operations[i] = SpaceOperation('same_as', + [Constant(None, lltype.Void)], + op.result) + # checkgraph(origportalgraph) def add_finish(self): @@ -644,8 +715,8 @@ graphs = self.translator.graphs _, PTR_SET_PARAM_FUNCTYPE = self.cpu.ts.get_FuncType([lltype.Signed], lltype.Void) - def make_closure(fullfuncname): - state = self.state + def make_closure(jd, fullfuncname): + state = jd.warmstate def closure(i): getattr(state, fullfuncname)(i) funcptr = self.helper_func(PTR_SET_PARAM_FUNCTYPE, closure) @@ -653,12 +724,17 @@ # for graph, block, i in find_set_param(graphs): op = block.operations[i] - assert op.args[1].value == self.jitdriver + for jd in self.jitdrivers_sd: + if jd.jitdriver is op.args[1].value: + break + else: + assert 0, "jitdriver of set_param() not found" funcname = op.args[2].value - if funcname not in closures: - closures[funcname] = make_closure('set_param_' + funcname) + key = jd, funcname + if key not in closures: + closures[key] = make_closure(jd, 'set_param_' + funcname) op.opname = 'direct_call' - op.args[:3] = [closures[funcname]] + op.args[:3] = [closures[key]] def rewrite_force_virtual(self, vrefinfo): if self.cpu.ts.name != 'lltype': Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmstate.py Fri Jul 2 17:44:46 2010 @@ -1,7 +1,7 @@ import sys from pypy.rpython.lltypesystem import lltype, llmemory, rstr from pypy.rpython.ootypesystem import ootype -from pypy.rpython.annlowlevel import hlstr, cast_base_ptr_to_instance +from pypy.rpython.annlowlevel import hlstr, llstr, cast_base_ptr_to_instance from pypy.rpython.annlowlevel import cast_object_to_ptr from pypy.rlib.objectmodel import specialize, we_are_translated, r_dict from pypy.rlib.rarithmetic import intmask @@ -120,6 +120,16 @@ else: assert False +class JitCell(BaseJitCell): + # the counter can mean the following things: + # counter >= 0: not yet traced, wait till threshold is reached + # counter == -1: there is an entry bridge for this cell + # counter == -2: tracing is currently going on for this cell + counter = 0 + compiled_merge_points = None + dont_trace_here = False + entry_loop_token = None + # ____________________________________________________________ @@ -127,9 +137,10 @@ THRESHOLD_LIMIT = sys.maxint // 2 default_jitcell_dict = None - def __init__(self, warmrunnerdesc): + def __init__(self, warmrunnerdesc, jitdriver_sd): "NOT_RPYTHON" self.warmrunnerdesc = warmrunnerdesc + self.jitdriver_sd = jitdriver_sd try: self.profiler = warmrunnerdesc.metainterp_sd.profiler except AttributeError: # for tests @@ -176,8 +187,7 @@ cell = self.jit_cell_at_key(greenkey) cell.dont_trace_here = True debug_start("jit-disableinlining") - sd = self.warmrunnerdesc.metainterp_sd - loc = sd.state.get_location_str(greenkey) + loc = self.get_location_str(greenkey) debug_print("disabled inlining", loc) debug_stop("jit-disableinlining") @@ -195,8 +205,10 @@ return self.maybe_compile_and_run metainterp_sd = self.warmrunnerdesc.metainterp_sd - vinfo = self.warmrunnerdesc.virtualizable_info - num_green_args = self.warmrunnerdesc.num_green_args + jitdriver_sd = self.jitdriver_sd + vinfo = jitdriver_sd.virtualizable_info + index_of_virtualizable = jitdriver_sd.index_of_virtualizable + num_green_args = jitdriver_sd.num_green_args get_jitcell = self.make_jitcell_getter() set_future_values = self.make_set_future_values() self.make_jitdriver_callbacks() @@ -206,7 +218,6 @@ """Entry point to the JIT. Called at the point with the can_enter_jit() hint. """ - globaldata = metainterp_sd.globaldata if NonConstant(False): # make sure we always see the saner optimizer from an # annotation point of view, otherwise we get lots of @@ -214,7 +225,7 @@ self.set_param_optimizer(OPTIMIZER_FULL) if vinfo is not None: - virtualizable = args[vinfo.index_of_virtualizable] + virtualizable = args[num_green_args + index_of_virtualizable] virtualizable = vinfo.cast_to_vtype(virtualizable) else: virtualizable = None @@ -234,11 +245,12 @@ return # bound reached; start tracing from pypy.jit.metainterp.pyjitpl import MetaInterp - metainterp = MetaInterp(metainterp_sd) + metainterp = MetaInterp(metainterp_sd, jitdriver_sd) # set counter to -2, to mean "tracing in effect" cell.counter = -2 try: - loop_token = metainterp.compile_and_run_once(*args) + loop_token = metainterp.compile_and_run_once(jitdriver_sd, + *args) finally: if cell.counter == -2: cell.counter = 0 @@ -264,7 +276,8 @@ metainterp_sd.profiler.end_running() if vinfo is not None: vinfo.reset_vable_token(virtualizable) - loop_token = fail_descr.handle_fail(metainterp_sd) + loop_token = fail_descr.handle_fail(metainterp_sd, + jitdriver_sd) maybe_compile_and_run._dont_inline_ = True self.maybe_compile_and_run = maybe_compile_and_run @@ -277,8 +290,8 @@ if hasattr(self, 'unwrap_greenkey'): return self.unwrap_greenkey # - warmrunnerdesc = self.warmrunnerdesc - green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec) + jitdriver_sd = self.jitdriver_sd + green_args_spec = unrolling_iterable(jitdriver_sd._green_args_spec) # def unwrap_greenkey(greenkey): greenargs = () @@ -302,20 +315,10 @@ if hasattr(self, 'jit_getter'): return self.jit_getter # - class JitCell(BaseJitCell): - # the counter can mean the following things: - # counter >= 0: not yet traced, wait till threshold is reached - # counter == -1: there is an entry bridge for this cell - # counter == -2: tracing is currently going on for this cell - counter = 0 - compiled_merge_points = None - dont_trace_here = False - entry_loop_token = None - # - if self.warmrunnerdesc.get_jitcell_at_ptr is None: - jit_getter = self._make_jitcell_getter_default(JitCell) + if self.jitdriver_sd._get_jitcell_at_ptr is None: + jit_getter = self._make_jitcell_getter_default() else: - jit_getter = self._make_jitcell_getter_custom(JitCell) + jit_getter = self._make_jitcell_getter_custom() # unwrap_greenkey = self.make_unwrap_greenkey() # @@ -327,10 +330,10 @@ # return jit_getter - def _make_jitcell_getter_default(self, JitCell): + def _make_jitcell_getter_default(self): "NOT_RPYTHON" - warmrunnerdesc = self.warmrunnerdesc - green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec) + jitdriver_sd = self.jitdriver_sd + green_args_spec = unrolling_iterable(jitdriver_sd._green_args_spec) # def comparekey(greenargs1, greenargs2): i = 0 @@ -361,11 +364,11 @@ return cell return get_jitcell - def _make_jitcell_getter_custom(self, JitCell): + def _make_jitcell_getter_custom(self): "NOT_RPYTHON" rtyper = self.warmrunnerdesc.rtyper - get_jitcell_at_ptr = self.warmrunnerdesc.get_jitcell_at_ptr - set_jitcell_at_ptr = self.warmrunnerdesc.set_jitcell_at_ptr + get_jitcell_at_ptr = self.jitdriver_sd._get_jitcell_at_ptr + set_jitcell_at_ptr = self.jitdriver_sd._set_jitcell_at_ptr lltohlhack = {} # def get_jitcell(*greenargs): @@ -415,9 +418,10 @@ return self.set_future_values warmrunnerdesc = self.warmrunnerdesc + jitdriver_sd = self.jitdriver_sd cpu = warmrunnerdesc.cpu - vinfo = warmrunnerdesc.virtualizable_info - red_args_types = unrolling_iterable(warmrunnerdesc.red_args_types) + vinfo = jitdriver_sd.virtualizable_info + red_args_types = unrolling_iterable(jitdriver_sd._red_args_types) # def set_future_values(*redargs): i = 0 @@ -428,8 +432,9 @@ set_future_values_from_vinfo(*redargs) # if vinfo is not None: - i0 = len(warmrunnerdesc.red_args_types) - num_green_args = warmrunnerdesc.num_green_args + i0 = len(jitdriver_sd._red_args_types) + num_green_args = jitdriver_sd.num_green_args + index_of_virtualizable = jitdriver_sd.index_of_virtualizable vable_static_fields = unrolling_iterable( zip(vinfo.static_extra_types, vinfo.static_fields)) vable_array_fields = unrolling_iterable( @@ -439,8 +444,7 @@ # def set_future_values_from_vinfo(*redargs): i = i0 - virtualizable = redargs[vinfo.index_of_virtualizable - - num_green_args] + virtualizable = redargs[index_of_virtualizable] virtualizable = vinfo.cast_to_vtype(virtualizable) for typecode, fieldname in vable_static_fields: x = getattr(virtualizable, fieldname) @@ -464,8 +468,9 @@ if hasattr(self, 'get_location_str'): return # - can_inline_ptr = self.warmrunnerdesc.can_inline_ptr + can_inline_ptr = self.jitdriver_sd._can_inline_ptr unwrap_greenkey = self.make_unwrap_greenkey() + jit_getter = self.make_jitcell_getter() if can_inline_ptr is None: def can_inline_callable(*greenargs): # XXX shouldn't it be False by default? @@ -477,7 +482,7 @@ fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) return fn(*greenargs) def can_inline(*greenargs): - cell = self.jit_getter(*greenargs) + cell = jit_getter(*greenargs) if cell.dont_trace_here: return False return can_inline_callable(*greenargs) @@ -486,21 +491,25 @@ greenargs = unwrap_greenkey(greenkey) return can_inline(*greenargs) self.can_inline_callable = can_inline_greenkey - - get_jitcell = self.make_jitcell_getter() + def get_assembler_token(greenkey): greenargs = unwrap_greenkey(greenkey) - cell = get_jitcell(*greenargs) + cell = jit_getter(*greenargs) if cell.counter >= 0: return None return cell.entry_loop_token self.get_assembler_token = get_assembler_token # - get_location_ptr = self.warmrunnerdesc.get_printable_location_ptr + get_location_ptr = self.jitdriver_sd._get_printable_location_ptr if get_location_ptr is None: + missing = '(no jitdriver.get_printable_location!)' + missingll = llstr(missing) def get_location_str(greenkey): - return '(no jitdriver.get_printable_location!)' + if we_are_translated(): + return missingll + else: + return missing else: rtyper = self.warmrunnerdesc.rtyper unwrap_greenkey = self.make_unwrap_greenkey() @@ -514,7 +523,7 @@ return res self.get_location_str = get_location_str # - confirm_enter_jit_ptr = self.warmrunnerdesc.confirm_enter_jit_ptr + confirm_enter_jit_ptr = self.jitdriver_sd._confirm_enter_jit_ptr if confirm_enter_jit_ptr is None: def confirm_enter_jit(*args): return True Modified: pypy/branch/x86-64-jit-backend/pypy/jit/tl/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/tl/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/tl/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/tl/spli/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/tl/spli/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/tl/spli/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/tl/spli/targetspli.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/tl/spli/targetspli.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/tl/spli/targetspli.py Fri Jul 2 17:44:46 2010 @@ -31,7 +31,7 @@ def jitpolicy(driver): """Returns the JIT policy to use when translating.""" - from pypy.jit.metainterp.policy import JitPolicy + from pypy.jit.codewriter.policy import JitPolicy return JitPolicy() if __name__ == '__main__': Modified: pypy/branch/x86-64-jit-backend/pypy/jit/tl/targettlc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/tl/targettlc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/tl/targettlc.py Fri Jul 2 17:44:46 2010 @@ -2,7 +2,7 @@ import py py.path.local(__file__) from pypy.jit.tl.tlc import interp, interp_nonjit, ConstantPool -from pypy.jit.metainterp.policy import JitPolicy +from pypy.jit.codewriter.policy import JitPolicy from pypy.jit.backend.hlinfo import highleveljitinfo Modified: pypy/branch/x86-64-jit-backend/pypy/jit/tl/targettlr.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/tl/targettlr.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/tl/targettlr.py Fri Jul 2 17:44:46 2010 @@ -33,7 +33,7 @@ # ____________________________________________________________ -from pypy.jit.metainterp.policy import JitPolicy +from pypy.jit.codewriter.policy import JitPolicy def jitpolicy(driver): return JitPolicy() Modified: pypy/branch/x86-64-jit-backend/pypy/jit/tl/tinyframe/targettinyframe.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/tl/tinyframe/targettinyframe.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/tl/tinyframe/targettinyframe.py Fri Jul 2 17:44:46 2010 @@ -1,6 +1,6 @@ from pypy.jit.tl.tinyframe.tinyframe import main -from pypy.jit.metainterp.policy import JitPolicy +from pypy.jit.codewriter.policy import JitPolicy def jitpolicy(driver): return JitPolicy() Modified: pypy/branch/x86-64-jit-backend/pypy/jit/tool/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/tool/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/tool/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/functional.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/functional.py Fri Jul 2 17:44:46 2010 @@ -176,10 +176,75 @@ if not collections_w: msg = "map() requires at least two arguments" raise OperationError(space.w_TypeError, space.wrap(msg)) - num_collections = len(collections_w) none_func = space.is_w(w_func, space.w_None) - if none_func and num_collections == 1: - return space.call_function(space.w_list, collections_w[0]) + if len(collections_w) == 1: + w_collection = collections_w[0] + if none_func: + result_w = space.unpackiterable(w_collection) + else: + result_w = map_single_collection(space, w_func, w_collection) + else: + result_w = map_multiple_collections(space, w_func, collections_w, + none_func) + return space.newlist(result_w) +map.unwrap_spec = [ObjSpace, W_Root, "args_w"] + +def map_single_collection(space, w_func, w_collection): + """Special case for 'map(func, coll)', where 'func' is not None and there + is only one 'coll' argument.""" + w_iter = space.iter(w_collection) + # xxx special hacks for speed + from pypy.interpreter import function, pycode + if isinstance(w_func, function.Function): + # xxx compatibility issue: what if func_code is modified in the + # middle of running map()?? That's far too obscure for me to care... + code = w_func.getcode() + fast_natural_arity = code.fast_natural_arity + if fast_natural_arity == (1|pycode.PyCode.FLATPYCALL): + assert isinstance(code, pycode.PyCode) + return map_single_user_function(code, w_func, w_iter) + # /xxx end of special hacks + return map_single_other_callable(space, w_func, w_iter) + +def map_single_other_callable(space, w_func, w_iter): + result_w = [] + while True: + try: + w_item = space.next(w_iter) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break + result_w.append(space.call_function(w_func, w_item)) + return result_w +map_single_other_callable._dont_inline_ = True + +from pypy.rlib.jit import JitDriver +mapjitdriver = JitDriver(greens = ['code'], + reds = ['w_func', 'w_iter', 'result_w'], + can_inline = lambda *args: False) +def map_single_user_function(code, w_func, w_iter): + result_w = [] + while True: + mapjitdriver.can_enter_jit(code=code, w_func=w_func, + w_iter=w_iter, result_w=result_w) + mapjitdriver.jit_merge_point(code=code, w_func=w_func, + w_iter=w_iter, result_w=result_w) + space = w_func.space + try: + w_item = space.next(w_iter) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break + new_frame = space.createframe(code, w_func.w_func_globals, + w_func.closure) + new_frame.fastlocals_w[0] = w_item + w_res = new_frame.run() + result_w.append(w_res) + return result_w + +def map_multiple_collections(space, w_func, collections_w, none_func): result_w = [] iterators_w = [space.iter(w_seq) for w_seq in collections_w] num_iterators = len(iterators_w) @@ -196,16 +261,15 @@ iterators_w[i] = None else: cont = True - if cont: - w_args = space.newtuple(args_w) - if none_func: - result_w.append(w_args) - else: - w_res = space.call(w_func, w_args) - result_w.append(w_res) + if not cont: + break + w_args = space.newtuple(args_w) + if none_func: + w_res = w_args else: - return space.newlist(result_w) -map.unwrap_spec = [ObjSpace, W_Root, "args_w"] + w_res = space.call(w_func, w_args) + result_w.append(w_res) + return result_w def sum(space, w_sequence, w_start=None): if space.is_w(w_start, space.w_None): Modified: pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/test/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/test/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/test/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/test/test_functional.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/test/test_functional.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/test/test_functional.py Fri Jul 2 17:44:46 2010 @@ -6,6 +6,9 @@ def test_trivial_map_one_seq(self): assert map(lambda x: x+2, [1, 2, 3, 4]) == [3, 4, 5, 6] + def test_trivial_map_one_seq_2(self): + assert map(str, [1, 2, 3, 4]) == ['1', '2', '3', '4'] + def test_trivial_map_two_seq(self): assert map(lambda x,y: x+y, [1, 2, 3, 4],[1, 2, 3, 4]) == ( Modified: pypy/branch/x86-64-jit-backend/pypy/module/_codecs/__init__.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_codecs/__init__.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_codecs/__init__.py Fri Jul 2 17:44:46 2010 @@ -3,22 +3,43 @@ from pypy.module._codecs import interp_codecs class Module(MixedModule): - appleveldefs = { - '__doc__' : 'app_codecs.__doc__', - '__name__' : 'app_codecs.__name__', - 'charmap_encode' : 'app_codecs.charmap_encode', - 'escape_decode' : 'app_codecs.escape_decode', - 'escape_encode' : 'app_codecs.escape_encode', - 'raw_unicode_escape_decode' : 'app_codecs.raw_unicode_escape_decode', - 'raw_unicode_escape_encode' : 'app_codecs.raw_unicode_escape_encode', - 'unicode_escape_decode' : 'app_codecs.unicode_escape_decode', - 'unicode_escape_encode' : 'app_codecs.unicode_escape_encode', - 'unicode_internal_decode' : 'app_codecs.unicode_internal_decode', - 'unicode_internal_encode' : 'app_codecs.unicode_internal_encode', - 'utf_7_decode' : 'app_codecs.utf_7_decode', - 'utf_7_encode' : 'app_codecs.utf_7_encode', - 'charmap_build' : 'app_codecs.charmap_build' - } + """ + _codecs -- Provides access to the codec registry and the builtin + codecs. + + This module should never be imported directly. The standard library + module "codecs" wraps this builtin module for use within Python. + + The codec registry is accessible via: + + register(search_function) -> None + + lookup(encoding) -> (encoder, decoder, stream_reader, stream_writer) + + The builtin Unicode codecs use the following interface: + + _encode(Unicode_object[,errors='strict']) -> + (string object, bytes consumed) + + _decode(char_buffer_obj[,errors='strict']) -> + (Unicode object, bytes consumed) + + _encode() interfaces also accept non-Unicode object as + input. The objects are then converted to Unicode using + PyUnicode_FromObject() prior to applying the conversion. + + These s are available: utf_8, unicode_escape, + raw_unicode_escape, unicode_internal, latin_1, ascii (7-bit), + mbcs (on win32). + + +Written by Marc-Andre Lemburg (mal at lemburg.com). + +Copyright (c) Corporation for National Research Initiatives. +""" + + appleveldefs = {} + interpleveldefs = { 'encode': 'interp_codecs.encode', 'decode': 'interp_codecs.decode', @@ -26,12 +47,15 @@ 'lookup_error': 'interp_codecs.lookup_error', 'register': 'interp_codecs.register_codec', 'register_error': 'interp_codecs.register_error', + 'charmap_build' : 'interp_codecs.charmap_build', # encoders and decoders 'ascii_decode' : 'interp_codecs.ascii_decode', 'ascii_encode' : 'interp_codecs.ascii_encode', 'latin_1_decode' : 'interp_codecs.latin_1_decode', 'latin_1_encode' : 'interp_codecs.latin_1_encode', + 'utf_7_decode' : 'interp_codecs.utf_7_decode', + 'utf_7_encode' : 'interp_codecs.utf_7_encode', 'utf_8_decode' : 'interp_codecs.utf_8_decode', 'utf_8_encode' : 'interp_codecs.utf_8_encode', 'utf_16_be_decode' : 'interp_codecs.utf_16_be_decode', @@ -44,6 +68,15 @@ 'charbuffer_encode': 'interp_codecs.buffer_encode', 'readbuffer_encode': 'interp_codecs.buffer_encode', 'charmap_decode' : 'interp_codecs.charmap_decode', + 'charmap_encode' : 'interp_codecs.charmap_encode', + 'escape_encode' : 'interp_codecs.escape_encode', + 'escape_decode' : 'interp_codecs.escape_decode', + 'unicode_escape_decode' : 'interp_codecs.unicode_escape_decode', + 'unicode_escape_encode' : 'interp_codecs.unicode_escape_encode', + 'raw_unicode_escape_decode' : 'interp_codecs.raw_unicode_escape_decode', + 'raw_unicode_escape_encode' : 'interp_codecs.raw_unicode_escape_encode', + 'unicode_internal_decode' : 'interp_codecs.unicode_internal_decode', + 'unicode_internal_encode' : 'interp_codecs.unicode_internal_encode', } def __init__(self, space, *args): Modified: pypy/branch/x86-64-jit-backend/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_codecs/interp_codecs.py Fri Jul 2 17:44:46 2010 @@ -1,5 +1,6 @@ from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import ObjSpace, NoneNotWrapped, applevel +from pypy.interpreter.gateway import ObjSpace, NoneNotWrapped, interp2app +from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.baseobjspace import W_Root from pypy.rlib.rstring import StringBuilder, UnicodeBuilder from pypy.rlib.objectmodel import we_are_translated @@ -13,6 +14,8 @@ self.decode_error_handler = self.make_errorhandler(space, True) self.encode_error_handler = self.make_errorhandler(space, False) + self.unicodedata_handler = None + def make_errorhandler(self, space, decode): def unicode_call_errorhandler(errors, encoding, reason, input, startpos, endpos): @@ -53,6 +56,21 @@ return replace, newpos return unicode_call_errorhandler + def get_unicodedata_handler(self, space): + if self.unicodedata_handler: + return self.unicodedata_handler + try: + w_builtin = space.getbuiltinmodule('__builtin__') + w_import = space.getattr(w_builtin, space.wrap("__import__")) + w_unicodedata = space.call_function(w_import, + space.wrap("unicodedata")) + w_getcode = space.getattr(w_unicodedata, space.wrap("_get_code")) + except OperationError: + return None + else: + self.unicodedata_handler = UnicodeData_Handler(space, w_getcode) + return self.unicodedata_handler + def _freeze_(self): assert not self.codec_search_path return False @@ -114,78 +132,125 @@ "unknown encoding: %s", encoding) lookup_codec.unwrap_spec = [ObjSpace, str] -app_errors = applevel(""" -def check_exception(exc): +# ____________________________________________________________ +# Register standard error handlers + +def check_exception(space, w_exc): try: - delta = exc.end - exc.start - if delta < 0 or not isinstance(exc.object, (unicode, str)): - raise TypeError("wrong exception") - except AttributeError: - raise TypeError("wrong exception") - -def strict_errors(exc): - if isinstance(exc, Exception): - raise exc - else: - raise TypeError("codec must pass exception instance") - -def ignore_errors(exc): - check_exception(exc) - if isinstance(exc, UnicodeEncodeError): - return u'', exc.end - elif isinstance(exc, (UnicodeDecodeError, UnicodeTranslateError)): - return u'', exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%exc) - -Py_UNICODE_REPLACEMENT_CHARACTER = u"\ufffd" - -def replace_errors(exc): - check_exception(exc) - if isinstance(exc, UnicodeEncodeError): - return u'?'*(exc.end-exc.start), exc.end - elif isinstance(exc, (UnicodeTranslateError, UnicodeDecodeError)): - return Py_UNICODE_REPLACEMENT_CHARACTER*(exc.end-exc.start), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%exc) - -def xmlcharrefreplace_errors(exc): - if isinstance(exc, UnicodeEncodeError): - res = [] - for ch in exc.object[exc.start:exc.end]: - res += '&#' - res += str(ord(ch)) - res += ';' - return u''.join(res), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) - -def backslashreplace_errors(exc): - if isinstance(exc, UnicodeEncodeError): - p = [] - for c in exc.object[exc.start:exc.end]: - p += '\\\\' - oc = ord(c) - if (oc >= 0x00010000): - p += 'U' - p += "%.8x" % ord(c) + w_start = space.getattr(w_exc, space.wrap('start')) + w_end = space.getattr(w_exc, space.wrap('end')) + w_obj = space.getattr(w_exc, space.wrap('object')) + except OperationError, e: + if not e.match(space, space.w_AttributeError): + raise + raise OperationError(space.w_TypeError, space.wrap( + "wrong exception")) + + delta = space.int_w(w_end) - space.int_w(w_start) + if delta < 0 or not (space.isinstance_w(w_obj, space.w_str) or + space.isinstance_w(w_obj, space.w_unicode)): + raise OperationError(space.w_TypeError, space.wrap( + "wrong exception")) + +def strict_errors(space, w_exc): + check_exception(space, w_exc) + if space.isinstance_w(w_exc, space.w_BaseException): + raise OperationError(space.type(w_exc), w_exc) + else: + raise OperationError(space.w_TypeError, space.wrap( + "codec must pass exception instance")) + +def ignore_errors(space, w_exc): + check_exception(space, w_exc) + w_end = space.getattr(w_exc, space.wrap('end')) + if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): + return space.newtuple([space.wrap(''), w_end]) + elif (space.isinstance_w(w_exc, space.w_UnicodeDecodeError) or + space.isinstance_w(w_exc, space.w_UnicodeTranslateError)): + return space.newtuple([space.wrap(u''), w_end]) + else: + typename = space.type(w_exc).getname(space, '?') + raise operationerrfmt(space.w_TypeError, + "don't know how to handle %s in error callback", typename) + +def replace_errors(space, w_exc): + check_exception(space, w_exc) + w_start = space.getattr(w_exc, space.wrap('start')) + w_end = space.getattr(w_exc, space.wrap('end')) + size = space.int_w(w_end) - space.int_w(w_start) + if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): + text = '?' * size + return space.newtuple([space.wrap(text), w_end]) + elif (space.isinstance_w(w_exc, space.w_UnicodeDecodeError) or + space.isinstance_w(w_exc, space.w_UnicodeTranslateError)): + text = u'\ufffd' * size + return space.newtuple([space.wrap(text), w_end]) + else: + typename = space.type(w_exc).getname(space, '?') + raise operationerrfmt(space.w_TypeError, + "don't know how to handle %s in error callback", typename) + +def xmlcharrefreplace_errors(space, w_exc): + check_exception(space, w_exc) + if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): + obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object'))) + start = space.int_w(space.getattr(w_exc, space.wrap('start'))) + w_end = space.getattr(w_exc, space.wrap('end')) + end = space.int_w(w_end) + builder = UnicodeBuilder() + pos = start + while pos < end: + ch = obj[pos] + builder.append(u"&#") + builder.append(unicode(str(ord(ch)))) + builder.append(u";") + pos += 1 + return space.newtuple([space.wrap(builder.build()), w_end]) + else: + typename = space.type(w_exc).getname(space, '?') + raise operationerrfmt(space.w_TypeError, + "don't know how to handle %s in error callback", typename) + +def backslashreplace_errors(space, w_exc): + check_exception(space, w_exc) + if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): + obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object'))) + start = space.int_w(space.getattr(w_exc, space.wrap('start'))) + w_end = space.getattr(w_exc, space.wrap('end')) + end = space.int_w(w_end) + builder = UnicodeBuilder() + pos = start + while pos < end: + oc = ord(obj[pos]) + num = hex(oc) + if (oc >= 0x10000): + builder.append(u"\\U") + zeros = 8 elif (oc >= 0x100): - p += 'u' - p += "%.4x" % ord(c) + builder.append(u"\\u") + zeros = 4 else: - p += 'x' - p += "%.2x" % ord(c) - return u''.join(p), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) -""") + builder.append(u"\\x") + zeros = 2 + lnum = len(num) + nb = zeros + 2 - lnum # num starts with '0x' + if nb > 0: + builder.append_multiple_char(u'0', nb) + builder.append_slice(unicode(num), 2, lnum) + pos += 1 + return space.newtuple([space.wrap(builder.build()), w_end]) + else: + typename = space.type(w_exc).getname(space, '?') + raise operationerrfmt(space.w_TypeError, + "don't know how to handle %s in error callback", typename) def register_builtin_error_handlers(space): + "NOT_RPYTHON" state = space.fromcache(CodecState) for error in ("strict", "ignore", "replace", "xmlcharrefreplace", "backslashreplace"): name = error + "_errors" - state.codec_error_registry[error] = app_errors.wget(space, name) + state.codec_error_registry[error] = space.wrap(interp2app(globals()[name])) def lookup_error(space, errors): @@ -279,6 +344,38 @@ from pypy.rlib import runicode +def make_raw_encoder(name): + rname = "unicode_encode_%s" % (name.replace("_encode", ""), ) + assert hasattr(runicode, rname) + def raw_encoder(space, uni): + state = space.fromcache(CodecState) + func = getattr(runicode, rname) + errors = "strict" + return func(uni, len(uni), errors, state.encode_error_handler) + raw_encoder.func_name = rname + return raw_encoder + +def make_raw_decoder(name): + rname = "str_decode_%s" % (name.replace("_decode", ""), ) + assert hasattr(runicode, rname) + def raw_decoder(space, string): + final = True + errors = "strict" + state = space.fromcache(CodecState) + func = getattr(runicode, rname) + kwargs = {} + if name == 'unicode_escape': + unicodedata_handler = state.get_unicodedata_handler(space) + result, consumed = func(string, len(string), errors, + final, state.decode_error_handler, + unicodedata_handler=unicodedata_handler) + else: + result, consumed = func(string, len(string), errors, + final, state.decode_error_handler) + return result + raw_decoder.func_name = rname + return raw_decoder + def make_encoder_wrapper(name): rname = "unicode_encode_%s" % (name.replace("_encode", ""), ) assert hasattr(runicode, rname) @@ -308,20 +405,26 @@ for encoders in [ "ascii_encode", "latin_1_encode", + "utf_7_encode", "utf_8_encode", "utf_16_encode", "utf_16_be_encode", "utf_16_le_encode", + "unicode_escape_encode", + "raw_unicode_escape_encode", + "unicode_internal_encode", ]: make_encoder_wrapper(encoders) for decoders in [ "ascii_decode", "latin_1_decode", + "utf_7_decode", "utf_8_decode", "utf_16_decode", "utf_16_be_decode", "utf_16_le_decode", + "raw_unicode_escape_decode", ]: make_decoder_wrapper(decoders) @@ -330,8 +433,6 @@ make_decoder_wrapper('mbcs_decode') def utf_16_ex_decode(space, data, errors='strict', byteorder=0, w_final=False): - """None - """ final = space.is_true(w_final) state = space.fromcache(CodecState) if byteorder == 0: @@ -349,77 +450,213 @@ space.wrap(byteorder)]) utf_16_ex_decode.unwrap_spec = [ObjSpace, str, str, int, W_Root] -def _extract_from_mapping(space, mapping_w, w_mapping, ch): - if mapping_w is not None: +# ____________________________________________________________ +# Charmap + +class Charmap_Decode: + def __init__(self, space, w_mapping): + self.space = space + self.w_mapping = w_mapping + + # fast path for all the stuff in the encodings module + if space.is_true(space.isinstance(w_mapping, space.w_tuple)): + self.mapping_w = space.fixedview(w_mapping) + else: + self.mapping_w = None + + def get(self, ch, errorchar): + space = self.space + + # get the character from the mapping + if self.mapping_w is not None: + w_ch = self.mapping_w[ord(ch)] + else: + try: + w_ch = space.getitem(self.w_mapping, space.newint(ord(ch))) + except OperationError, e: + if not e.match(space, space.w_LookupError): + raise + return errorchar + + # Charmap may return a unicode string try: - return mapping_w[ord(ch)] - except IndexError: - pass - else: + x = space.unicode_w(w_ch) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + else: + return x + + # Charmap may return a number + try: + x = space.int_w(w_ch) + except OperationError: + if not e.match(space, space.w_TypeError): + raise + else: + if 0 <= x < 65536: # Even on wide unicode builds... + return unichr(x) + else: + raise OperationError(space.w_TypeError, space.wrap( + "character mapping must be in range(65536)")) + + # Charmap may return None + if space.is_w(w_ch, space.w_None): + return errorchar + + raise OperationError(space.w_TypeError, space.wrap("invalid mapping")) + +class Charmap_Encode: + def __init__(self, space, w_mapping): + self.space = space + self.w_mapping = w_mapping + + def get(self, ch, errorchar): + space = self.space + + # get the character from the mapping try: - return space.getitem(w_mapping, space.newint(ord(ch))) + w_ch = space.getitem(self.w_mapping, space.newint(ord(ch))) except OperationError, e: - if (not e.match(space, space.w_KeyError) and - not e.match(space, space.w_IndexError)): + if not e.match(space, space.w_LookupError): raise - pass + return errorchar -def _append_unicode(space, builder, w_x): - try: - x = space.unicode_w(w_x) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - else: - if x != u"\ufffe": - builder.append(x) - return True - return False - try: - x = space.int_w(w_x) - except OperationError: - if not e.match(space, space.w_TypeError): - raise - else: - if x < 65536: - builder.append(unichr(x)) + # Charmap may return a string + try: + x = space.realstr_w(w_ch) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise else: - raise OperationError(space.w_TypeError, space.wrap("character mapping must be in range(65536)")) - return True - if not space.is_true(w_x): - return False + return x + + # Charmap may return a number + try: + x = space.int_w(w_ch) + except OperationError: + if not e.match(space, space.w_TypeError): + raise + else: + if 0 <= x < 256: + return chr(x) + else: + raise OperationError(space.w_TypeError, space.wrap( + "character mapping must be in range(256)")) + + # Charmap may return None + if space.is_w(w_ch, space.w_None): + return errorchar + + raise OperationError(space.w_TypeError, space.wrap("invalid mapping")) + + + at unwrap_spec(ObjSpace, str, str, W_Root) +def charmap_decode(space, string, errors="strict", w_mapping=None): + if len(string) == 0: + return space.newtuple([space.wrap(u''), space.wrap(0)]) + + if space.is_w(w_mapping, space.w_None): + mapping = None else: - raise OperationError(space.w_TypeError, space.w_None) + mapping = Charmap_Decode(space, w_mapping) + + final = True + state = space.fromcache(CodecState) + result, consumed = runicode.str_decode_charmap( + string, len(string), errors, + final, state.decode_error_handler, mapping) + return space.newtuple([space.wrap(result), space.wrap(consumed)]) + + at unwrap_spec(ObjSpace, unicode, str, W_Root) +def charmap_encode(space, uni, errors="strict", w_mapping=None): + if space.is_w(w_mapping, space.w_None): + mapping = None + else: + mapping = Charmap_Encode(space, w_mapping) + + state = space.fromcache(CodecState) + result = runicode.unicode_encode_charmap( + uni, len(uni), errors, + state.encode_error_handler, mapping) + return space.newtuple([space.wrap(result), space.wrap(len(uni))]) -def charmap_decode(space, s, errors="strict", w_mapping=None): - size = len(s) - # Default to Latin-1 - if space.is_true(space.is_(w_mapping, space.w_None)): - return latin_1_decode(space, s, errors, space.w_False) + at unwrap_spec(ObjSpace, unicode) +def charmap_build(space, chars): + # XXX CPython sometimes uses a three-level trie + w_charmap = space.newdict() + for num in range(len(chars)): + elem = chars[num] + space.setitem(w_charmap, space.newint(ord(elem)), space.newint(num)) + return w_charmap - if (size == 0): +# ____________________________________________________________ +# Unicode escape + +class UnicodeData_Handler: + def __init__(self, space, w_getcode): + self.space = space + self.w_getcode = w_getcode + + def call(self, name): + space = self.space + try: + w_code = space.call_function(self.w_getcode, space.wrap(name)) + except OperationError, e: + if not e.match(space, space.w_KeyError): + raise + return -1 + return space.int_w(w_code) + + at unwrap_spec(ObjSpace, 'bufferstr', str, W_Root) +def unicode_escape_decode(space, string, errors="strict", w_final=False): + final = space.is_true(w_final) + state = space.fromcache(CodecState) + errorhandler=state.decode_error_handler + + unicode_name_handler = state.get_unicodedata_handler(space) + + result, consumed = runicode.str_decode_unicode_escape( + string, len(string), errors, + final, state.decode_error_handler, + unicode_name_handler) + + return space.newtuple([space.wrap(result), space.wrap(consumed)]) + +# ____________________________________________________________ +# Unicode-internal + + at unwrap_spec(ObjSpace, W_Root, str) +def unicode_internal_decode(space, w_string, errors="strict"): + # special case for this codec: unicodes are returned as is + if space.isinstance_w(w_string, space.w_unicode): + return space.newtuple([w_string, space.len(w_string)]) + + string = space.str_w(w_string) + + if len(string) == 0: return space.newtuple([space.wrap(u''), space.wrap(0)]) - - # fast path for all the stuff in the encodings module - if space.is_true(space.isinstance(w_mapping, space.w_tuple)): - mapping_w = space.fixedview(w_mapping) - else: - mapping_w = None - - builder = UnicodeBuilder(size) - inpos = 0 - while (inpos < len(s)): - #/* Get mapping_w (char ordinal -> integer, Unicode char or None) */ - ch = s[inpos] - w_x = _extract_from_mapping(space, mapping_w, w_mapping, ch) - if w_x is not None and _append_unicode(space, builder, w_x): - inpos += 1 - continue - state = space.fromcache(CodecState) - next, inpos = state.decode_error_handler(errors, "charmap", - "character maps to ", s, inpos, inpos+1) - builder.append(next) - res = builder.build() - return space.newtuple([space.wrap(res), space.wrap(size)]) -charmap_decode.unwrap_spec = [ObjSpace, str, str, W_Root] + + final = True + state = space.fromcache(CodecState) + result, consumed = runicode.str_decode_unicode_internal( + string, len(string), errors, + final, state.decode_error_handler) + return space.newtuple([space.wrap(result), space.wrap(consumed)]) + +# ____________________________________________________________ +# support for the "string escape" codec +# This is a bytes-to bytes transformation + + at unwrap_spec(ObjSpace, W_Root, str) +def escape_encode(space, w_string, errors='strict'): + w_repr = space.repr(w_string) + w_result = space.getslice(w_repr, space.wrap(1), space.wrap(-1)) + return space.newtuple([w_result, space.len(w_string)]) + + at unwrap_spec(ObjSpace, str, str) +def escape_decode(space, data, errors='strict'): + from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape + result = PyString_DecodeEscape(space, data, None) + return space.newtuple([space.wrap(result), space.wrap(len(data))]) Modified: pypy/branch/x86-64-jit-backend/pypy/module/_codecs/test/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_codecs/test/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_codecs/test/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_codecs/test/test_codecs.py Fri Jul 2 17:44:46 2010 @@ -1,7 +1,5 @@ import autopath from pypy.conftest import gettestobjspace -from pypy.module._codecs.app_codecs import unicode_escape_encode,\ - charmap_encode, unicode_escape_decode class AppTestCodecs: @@ -14,26 +12,16 @@ raises(TypeError, _codecs.register, 1) def test_bigU_codecs(self): - import sys - oldmaxunicode = sys.maxunicode - if sys.maxunicode <= 0xffff: - return # this test cannot run on UCS2 builds u = u'\U00010001\U00020002\U00030003\U00040004\U00050005' for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', 'raw_unicode_escape', 'unicode_escape', 'unicode_internal'): assert unicode(u.encode(encoding),encoding) == u - sys.maxunicode = oldmaxunicode def test_ucs4(self): - import sys - oldmaxunicode = sys.maxunicode - if sys.maxunicode <= 0xffff: - sys.maxunicode = 0xffffffff x = u'\U00100000' y = x.encode("raw-unicode-escape").decode("raw-unicode-escape") assert x == y - sys.maxunicode = oldmaxunicode def test_named_unicode(self): assert unicode('\\N{SPACE}','unicode-escape') == u" " @@ -118,12 +106,20 @@ def test_charmap_decode(self): from _codecs import charmap_decode + import sys assert charmap_decode('', 'strict', 'blablabla') == ('', 0) assert charmap_decode('xxx') == ('xxx', 3) assert charmap_decode('xxx', 'strict', {ord('x'): u'XX'}) == ('XXXXXX', 3) map = tuple([unichr(i) for i in range(256)]) assert charmap_decode('xxx\xff', 'strict', map) == (u'xxx\xff', 4) + raises(TypeError, charmap_decode, '\xff', "replace", {0xff: 0x10001}) + + def test_unicode_escape(self): + from _codecs import unicode_escape_encode, unicode_escape_decode + assert unicode_escape_encode(u'abc') == (u'abc'.encode('unicode_escape'), 3) + assert unicode_escape_decode('abc') == (u'abc'.decode('unicode_escape'), 3) + assert unicode_escape_decode('\\x61\\x62\\x63') == (u'abc', 12) class AppTestPartialEvaluation: @@ -275,7 +271,6 @@ assert u"\u0663".encode("raw-unicode-escape") == "\u0663" def test_escape_decode(self): - test = 'a\n\\b\x00c\td\u2045'.encode('string_escape') assert test.decode('string_escape') =='a\n\\b\x00c\td\u2045' assert '\\077'.decode('string_escape') == '?' @@ -283,6 +278,14 @@ assert '\\253'.decode('string_escape') == chr(0253) assert '\\312'.decode('string_escape') == chr(0312) + def test_escape_decode_wrap_around(self): + assert '\\400'.decode('string_escape') == chr(0) + + def test_escape_decode_ignore_invalid(self): + assert '\\9'.decode('string_escape') == '\\9' + assert '\\01'.decode('string_escape') == chr(01) + assert '\\0f'.decode('string_escape') == chr(0) + 'f' + assert '\\08'.decode('string_escape') == chr(0) + '8' def test_decode_utf8_different_case(self): constant = u"a" @@ -377,6 +380,9 @@ def test_charmap_decode_1(self): import codecs + assert codecs.charmap_encode(u'xxx') == ('xxx', 3) + assert codecs.charmap_encode(u'xxx', 'strict', {ord('x'): 'XX'}) == ('XXXXXX', 3) + res = codecs.charmap_decode("\x00\x01\x02", "replace", u"ab") assert res == (u"ab\ufffd", 3) res = codecs.charmap_decode("\x00\x01\x02", "replace", u"ab\ufffe") @@ -464,6 +470,9 @@ assert '\xff'.decode('utf-7', 'ignore') == '' assert '\x00'.decode('unicode-internal', 'ignore') == '' + def test_backslahreplace(self): + assert u'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == 'a\\xac\u1234\u20ac\u8000' + def test_badhandler(self): import codecs results = ( 42, u"foo", (1,2,3), (u"foo", 1, 3), (u"foo", None), (u"foo",), ("foo", 1, 3), ("foo", None), ("foo",) ) @@ -527,9 +536,26 @@ def test_charmap_encode(self): assert 'xxx'.encode('charmap') == 'xxx' + import codecs + raises(TypeError, codecs.charmap_encode, u'\xff', "replace", {0xff: 300}) + raises(UnicodeError, codecs.charmap_encode, u"\xff", "replace", {0xff: None}) + + def test_charmap_encode_replace(self): + charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"]) + charmap[ord("?")] = "XYZ" + import codecs + sin = u"abcDEF" + sout = codecs.charmap_encode(sin, "replace", charmap)[0] + assert sout == "AABBCCXYZXYZXYZ" + def test_charmap_decode_2(self): assert 'foo'.decode('charmap') == 'foo' + def test_charmap_build(self): + import codecs + assert codecs.charmap_build(u'123456') == {49: 0, 50: 1, 51: 2, + 52: 3, 53: 4, 54: 5} + def test_utf7_start_end_in_exception(self): try: '+IC'.decode('utf-7') @@ -537,6 +563,9 @@ assert exc.start == 0 assert exc.end == 3 + def test_utf7_surrogate(self): + raises(UnicodeDecodeError, '+3ADYAA-'.decode, 'utf-7') + def test_utf_16_encode_decode(self): import codecs x = u'123abc' @@ -546,6 +575,8 @@ def test_unicode_escape(self): assert u'\\'.encode('unicode-escape') == '\\\\' assert '\\\\'.decode('unicode-escape') == u'\\' + assert u'\ud801'.encode('unicode-escape') == '\\ud801' + assert u'\u0013'.encode('unicode-escape') == '\\x13' def test_mbcs(self): import sys @@ -555,14 +586,3 @@ assert u'caf\xe9'.encode('mbcs') == 'caf\xe9' assert u'\u040a'.encode('mbcs') == '?' # some cyrillic letter assert 'cafx\e9'.decode('mbcs') == u'cafx\e9' - - -class TestDirect: - def test_charmap_encode(self): - assert charmap_encode(u'xxx') == ('xxx', 3) - assert charmap_encode(u'xxx', 'strict', {ord('x'): 'XX'}) == ('XXXXXX', 6) - - def test_unicode_escape(self): - assert unicode_escape_encode(u'abc') == (u'abc'.encode('unicode_escape'), 3) - assert unicode_escape_decode('abc') == (u'abc'.decode('unicode_escape'), 3) - assert unicode_escape_decode('\\x61\\x62\\x63') == (u'abc', 12) Modified: pypy/branch/x86-64-jit-backend/pypy/module/_sre/test/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_sre/test/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_sre/test/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_stackless/interp_coroutine.py Fri Jul 2 17:44:46 2010 @@ -24,7 +24,9 @@ from pypy.interpreter.function import StaticMethod from pypy.module._stackless.stackless_flags import StacklessFlags -from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState, AbstractThunk +from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState, AbstractThunk, CoroutineExit + +from pypy.module.exceptions.interp_exceptions import W_SystemExit, _new_exception from pypy.rlib import rstack # for resume points from pypy.tool import stdlib_opcode as pythonopcode @@ -48,6 +50,13 @@ rstack.resume_point("appthunk", costate, returns=w_result) costate.w_tempval = w_result +W_CoroutineExit = _new_exception('CoroutineExit', W_SystemExit, + """Coroutine killed manually.""") + +# Should be moved to interp_stackless.py if it's ever implemented... Currently +# used by pypy/lib/stackless.py. +W_TaskletExit = _new_exception('TaskletExit', W_SystemExit, + """Tasklet killed manually.""") class AppCoroutine(Coroutine): # XXX, StacklessFlags): @@ -92,6 +101,13 @@ w_ret, state.w_tempval = state.w_tempval, space.w_None return w_ret + def switch(self): + space = self.space + try: + Coroutine.switch(self) + except CoroutineExit: + raise OperationError(self.costate.w_CoroutineExit, space.w_None) + def w_finished(self, w_excinfo): pass @@ -102,6 +118,9 @@ w_excvalue = operror.get_w_value(space) w_exctraceback = operror.application_traceback w_excinfo = space.newtuple([w_exctype, w_excvalue, w_exctraceback]) + + if w_exctype is self.costate.w_CoroutineExit: + self.coroutine_exit = True else: w_N = space.w_None w_excinfo = space.newtuple([w_N, w_N, w_N]) @@ -118,6 +137,23 @@ def w_kill(self): self.kill() + + def w_throw(self, w_type, w_value=None, w_traceback=None): + space = self.space + + operror = OperationError(w_type, w_value) + operror.normalize_exception(space) + + if not space.is_w(w_traceback, space.w_None): + from pypy.interpreter import pytraceback + tb = space.interpclass_w(w_traceback) + if tb is None or not space.is_true(space.isinstance(tb, + space.gettypeobject(pytraceback.PyTraceback.typedef))): + raise OperationError(space.w_TypeError, + space.wrap("throw: arg 3 must be a traceback or None")) + operror.application_traceback = tb + + self._kill(operror) def _userdel(self): if self.get_is_zombie(): @@ -309,6 +345,7 @@ unwrap_spec=['self', W_Root, Arguments]), switch = interp2app(AppCoroutine.w_switch), kill = interp2app(AppCoroutine.w_kill), + throw = interp2app(AppCoroutine.w_throw), finished = interp2app(AppCoroutine.w_finished), is_alive = GetSetProperty(AppCoroutine.w_get_is_alive), is_zombie = GetSetProperty(AppCoroutine.w_get_is_zombie, @@ -330,6 +367,26 @@ BaseCoState.__init__(self) self.w_tempval = space.w_None self.space = space + + # Exporting new exception to space + self.w_CoroutineExit = space.gettypefor(W_CoroutineExit) + space.setitem( + space.exceptions_module.w_dict, + space.new_interned_str('CoroutineExit'), + self.w_CoroutineExit) + space.setitem(space.builtin.w_dict, + space.new_interned_str('CoroutineExit'), + self.w_CoroutineExit) + + # Should be moved to interp_stackless.py if it's ever implemented... + self.w_TaskletExit = space.gettypefor(W_TaskletExit) + space.setitem( + space.exceptions_module.w_dict, + space.new_interned_str('TaskletExit'), + self.w_TaskletExit) + space.setitem(space.builtin.w_dict, + space.new_interned_str('TaskletExit'), + self.w_TaskletExit) def post_install(self): self.current = self.main = AppCoroutine(self.space, state=self) Modified: pypy/branch/x86-64-jit-backend/pypy/module/_stackless/rcoroutine.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_stackless/rcoroutine.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_stackless/rcoroutine.py Fri Jul 2 17:44:46 2010 @@ -7,3 +7,4 @@ BaseCoState = d['BaseCoState'] AbstractThunk = d['AbstractThunk'] syncstate = d['syncstate'] +CoroutineExit = d['CoroutineExit'] Modified: pypy/branch/x86-64-jit-backend/pypy/module/_stackless/test/test_coroutine.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_stackless/test/test_coroutine.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_stackless/test/test_coroutine.py Fri Jul 2 17:44:46 2010 @@ -84,8 +84,7 @@ assert not co.is_alive def test_kill_running(self): - skip("kill is not really working (there is only CoroutineExit, " - "which is not an app-level exception)") + coroutineexit = [] import _stackless as stackless main = stackless.coroutine.getcurrent() result = [] @@ -96,6 +95,9 @@ result.append(1) main.switch() x = 3 + except CoroutineExit: + coroutineexit.append(True) + raise finally: result.append(x) result.append(4) @@ -107,6 +109,7 @@ co.kill() assert not co.is_alive assert result == [1, 2] + assert coroutineexit == [True] def test_bogus_bind(self): import _stackless as stackless Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/api.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/api.py Fri Jul 2 17:44:46 2010 @@ -45,11 +45,9 @@ pypydir = py.path.local(autopath.pypydir) include_dir = pypydir / 'module' / 'cpyext' / 'include' source_dir = pypydir / 'module' / 'cpyext' / 'src' -interfaces_dir = pypydir / "_interfaces" include_dirs = [ include_dir, udir, - interfaces_dir, ] class CConfig: @@ -100,9 +98,16 @@ udir.join('pypy_macros.h').write("/* Will be filled later */") globals().update(rffi_platform.configure(CConfig_constants)) -def copy_header_files(): +def copy_header_files(dstdir): + assert dstdir.check(dir=True) + headers = include_dir.listdir('*.h') + include_dir.listdir('*.inl') for name in ("pypy_decl.h", "pypy_macros.h"): - udir.join(name).copy(interfaces_dir / name) + headers.append(udir.join(name)) + for header in headers: + header.copy(dstdir) + target = dstdir.join(header.basename) + target.chmod(0444) # make the file read-only, to make sure that nobody + # edits it by mistake _NOT_SPECIFIED = object() CANNOT_FAIL = object() @@ -809,7 +814,6 @@ kwds["includes"] = ['Python.h'] # this is our Python.h # Generate definitions for global structures - struct_file = udir.join('pypy_structs.c') structs = ["#include "] for name, (typ, expr) in GLOBALS.iteritems(): if name.endswith('#'): @@ -819,7 +823,7 @@ structs.append('PyObject* %s = (PyObject*)&_%s;' % (name, name)) elif typ == 'PyDateTime_CAPI*': structs.append('%s %s = NULL;' % (typ, name)) - struct_file.write('\n'.join(structs)) + struct_source = '\n'.join(structs) eci = ExternalCompilationInfo( include_dirs=include_dirs, @@ -833,9 +837,8 @@ source_dir / "bufferobject.c", source_dir / "object.c", source_dir / "cobject.c", - struct_file, ], - separate_module_sources = [code], + separate_module_sources = [code, struct_source], export_symbols=export_symbols_eci, compile_extra=compile_extra, **kwds @@ -883,7 +886,8 @@ deco(func.get_wrapper(space)) setup_init_functions(eci) - copy_header_files() + trunk_include = pypydir.dirpath() / 'include' + copy_header_files(trunk_include) initfunctype = lltype.Ptr(lltype.FuncType([], lltype.Void)) @unwrap_spec(ObjSpace, str, str) Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_api.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_api.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_api.py Fri Jul 2 17:44:46 2010 @@ -1,3 +1,4 @@ +import py from pypy.conftest import gettestobjspace from pypy.rpython.lltypesystem import rffi, lltype from pypy.interpreter.baseobjspace import W_Root @@ -68,3 +69,13 @@ api.PyPy_GetWrapped(space.w_None) api.PyPy_GetReference(space.w_None) + +def test_copy_header_files(tmpdir): + api.copy_header_files(tmpdir) + def check(name): + f = tmpdir.join(name) + assert f.check(file=True) + py.test.raises(py.error.EACCES, "f.open('w')") # check that it's not writable + check('Python.h') + check('modsupport.inl') + check('pypy_decl.h') Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_cpyext.py Fri Jul 2 17:44:46 2010 @@ -15,7 +15,7 @@ from pypy.module.cpyext.pyobject import RefcountState from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException from pypy.translator.goal import autopath -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict @api.cpython_api([], api.PyObject) def PyPy_Crash1(space): @@ -160,7 +160,8 @@ kwds["compile_extra"] = ["/we4013"] else: kwds["link_files"] = [str(api_library + '.so')] - kwds["compile_extra"] = ["-Werror=implicit-function-declaration"] + if sys.platform == 'linux2': + kwds["compile_extra"]=["-Werror=implicit-function-declaration"] return compile_module(name, **kwds) Modified: pypy/branch/x86-64-jit-backend/pypy/module/operator/test/test_operator.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/operator/test/test_operator.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/operator/test/test_operator.py Fri Jul 2 17:44:46 2010 @@ -154,7 +154,6 @@ def test_irepeat(self): import operator - import py class X(object): def __index__(self): Modified: pypy/branch/x86-64-jit-backend/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/sys/__init__.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/sys/__init__.py Fri Jul 2 17:44:46 2010 @@ -30,9 +30,8 @@ 'stderr' : 'state.getio(space).w_stderr', '__stderr__' : 'state.getio(space).w_stderr', 'pypy_objspaceclass' : 'space.wrap(repr(space))', - #'pypy_prefix': added by pypy_initial_path() when it succeeds, pointing - # to the trunk of a checkout or to the dir /usr/share/pypy-1.1 . - + #'prefix' : # added by pypy_initial_path() when it + #'exec_prefix' : # succeeds, pointing to trunk or /usr 'path' : 'state.get(space).w_path', 'modules' : 'state.get(space).w_modules', 'argv' : 'state.get(space).w_argv', @@ -96,11 +95,8 @@ if space.config.translating and not we_are_translated(): # don't get the filesystemencoding at translation time assert self.filesystemencoding is None - else: - from pypy.module.sys.interp_encoding import _getfilesystemencoding - self.filesystemencoding = _getfilesystemencoding(space) - def getmodule(self, name): + def getmodule(self, name): space = self.space w_modules = self.get('modules') try: Modified: pypy/branch/x86-64-jit-backend/pypy/module/sys/interp_encoding.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/sys/interp_encoding.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/sys/interp_encoding.py Fri Jul 2 17:44:46 2010 @@ -55,4 +55,6 @@ """Return the encoding used to convert Unicode filenames in operating system filenames. """ + if space.sys.filesystemencoding is None: + space.sys.filesystemencoding = _getfilesystemencoding(space) return space.wrap(space.sys.filesystemencoding) Modified: pypy/branch/x86-64-jit-backend/pypy/module/sys/state.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/sys/state.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/sys/state.py Fri Jul 2 17:44:46 2010 @@ -32,26 +32,23 @@ if not stat.S_ISDIR(st[0]): raise OSError(errno.ENOTDIR, path) -def getinitialpath(srcdir): - # build the initial path from the srcdir, which is the path of - # the "dist" directory of a PyPy checkout. - from pypy.module.sys.version import CPYTHON_VERSION +def getinitialpath(prefix): + from pypy.module.sys.version import CPYTHON_VERSION dirname = '%d.%d.%d' % (CPYTHON_VERSION[0], CPYTHON_VERSION[1], CPYTHON_VERSION[2]) - lib_python = os.path.join(srcdir, 'lib-python') - + lib_python = os.path.join(prefix, 'lib-python') python_std_lib = os.path.join(lib_python, dirname) checkdir(python_std_lib) python_std_lib_modified = os.path.join(lib_python, 'modified-' + dirname) checkdir(python_std_lib_modified) - pypydir = os.path.join(srcdir, 'pypy') - pypy_lib = os.path.join(pypydir, 'lib') - checkdir(pypy_lib) + + lib_pypy = os.path.join(prefix, 'lib_pypy') + checkdir(lib_pypy) importlist = [] - importlist.append(pypy_lib) + importlist.append(lib_pypy) importlist.append(python_std_lib_modified) importlist.append(python_std_lib) return importlist @@ -62,7 +59,9 @@ except OSError: return space.w_None else: - space.setitem(space.sys.w_dict, space.wrap('pypy_prefix'), + space.setitem(space.sys.w_dict, space.wrap('prefix'), + space.wrap(srcdir)) + space.setitem(space.sys.w_dict, space.wrap('exec_prefix'), space.wrap(srcdir)) return space.newlist([space.wrap(p) for p in path]) Modified: pypy/branch/x86-64-jit-backend/pypy/module/sys/test/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/sys/test/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/sys/test/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/module/sys/version.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/sys/version.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/sys/version.py Fri Jul 2 17:44:46 2010 @@ -7,7 +7,7 @@ CPYTHON_VERSION = (2, 5, 2, "beta", 42) CPYTHON_API_VERSION = 1012 -PYPY_VERSION = (1, 2, 0, "beta", '?') +PYPY_VERSION = (1, 3, 0, "beta", '?') # the last item is replaced by the svn revision ^^^ TRIM_URL_UP_TO = 'svn/pypy/' Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/flow/model.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/flow/model.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/flow/model.py Fri Jul 2 17:44:46 2010 @@ -7,7 +7,7 @@ from pypy.tool.uid import uid, Hashable from pypy.tool.descriptor import roproperty from pypy.tool.sourcetools import PY_IDENTIFIER, nice_repr_for_func -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict """ memory size before and after introduction of __slots__ Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/floatobject.py Fri Jul 2 17:44:46 2010 @@ -77,23 +77,34 @@ def float_w__Float(space, w_float): return w_float.floatval -def should_not_look_like_an_int(s): - for c in s: - if c in '.eE': - break +def float2string(space, w_float, format): + x = w_float.floatval + # we special-case explicitly inf and nan here + if isinf(x): + if x > 0.0: + s = "inf" + else: + s = "-inf" + elif isnan(x): + s = "nan" else: - s += '.0' - return s + s = formatd(format, x) + # We want float numbers to be recognizable as such, + # i.e., they should contain a decimal point or an exponent. + # However, %g may print the number as an integer; + # in such cases, we append ".0" to the string. + for c in s: + if c in '.eE': + break + else: + s += '.0' + return space.wrap(s) def repr__Float(space, w_float): - x = w_float.floatval - s = formatd("%.17g", x) - return space.wrap(should_not_look_like_an_int(s)) + return float2string(space, w_float, "%.17g") def str__Float(space, w_float): - x = w_float.floatval - s = formatd("%.12g", x) - return space.wrap(should_not_look_like_an_int(s)) + return float2string(space, w_float, "%.12g") # ____________________________________________________________ # A mess to handle all cases of float comparison without relying Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/marshal_impl.py Fri Jul 2 17:44:46 2010 @@ -447,11 +447,11 @@ register(TYPE_CODE, unmarshal_pycode) def marshal_w__Unicode(space, w_unicode, m): - s = space.str_w(unicodehelper.PyUnicode_EncodeUTF8(space, w_unicode)) + s = unicodehelper.PyUnicode_EncodeUTF8(space, space.unicode_w(w_unicode)) m.atom_str(TYPE_UNICODE, s) def unmarshal_Unicode(space, u, tc): - return unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(u.get_str())) + return space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, u.get_str())) register(TYPE_UNICODE, unmarshal_Unicode) app = gateway.applevel(r''' Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_floatobject.py Fri Jul 2 17:44:46 2010 @@ -95,7 +95,14 @@ inf = 1e200*1e200 assert float("inf") == inf assert float("-INf") == -inf - assert 'nan' in str(float("-nAn")).lower() + assert str(inf) == "inf" + assert str(-inf) == "-inf" + assert str(float("nan")) == "nan" + assert str(float("-nAn")) == "nan" + assert repr(inf) == "inf" + assert repr(-inf) == "-inf" + assert repr(float("nan")) == "nan" + assert repr(float("-nAn")) == "nan" def test_float_unicode(self): # u00A0 and u2000 are some kind of spaces Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_shadowtracking.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_shadowtracking.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_shadowtracking.py Fri Jul 2 17:44:46 2010 @@ -209,6 +209,7 @@ a = A() names = [name for name in A.__dict__.keys() if not name.startswith('_')] + names.sort() names_repeated = names * 10 result = [] __pypy__.reset_method_cache_counter() Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/unicodeobject.py Fri Jul 2 17:44:46 2010 @@ -12,6 +12,7 @@ from pypy.rlib.rarithmetic import intmask, ovfcheck from pypy.rlib.objectmodel import compute_hash from pypy.rlib.rstring import string_repeat +from pypy.rlib.runicode import unicode_encode_unicode_escape from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb from pypy.tool.sourcetools import func_with_new_name @@ -892,101 +893,11 @@ space.wrap("character mapping must return integer, None or unicode")) return W_UnicodeObject(u''.join(result)) -# Move this into the _codecs module as 'unicodeescape_string (Remember to cater for quotes)' def repr__Unicode(space, w_unicode): - hexdigits = "0123456789abcdef" chars = w_unicode._value size = len(chars) - - singlequote = doublequote = False - for c in chars: - if c == u'\'': - singlequote = True - elif c == u'"': - doublequote = True - if singlequote and not doublequote: - quote = '"' - else: - quote = '\'' - result = ['u', quote] - j = 0 - while j= 0x10000: - # Resize if needed - result.extend(['\\', "U", - hexdigits[(code >> 28) & 0xf], - hexdigits[(code >> 24) & 0xf], - hexdigits[(code >> 20) & 0xf], - hexdigits[(code >> 16) & 0xf], - hexdigits[(code >> 12) & 0xf], - hexdigits[(code >> 8) & 0xf], - hexdigits[(code >> 4) & 0xf], - hexdigits[(code >> 0) & 0xf], - ]) - j += 1 - continue - if code >= 0xD800 and code < 0xDC00: - if j < size - 1: - ch2 = chars[j+1] - code2 = ord(ch2) - if code2 >= 0xDC00 and code2 <= 0xDFFF: - code = (((code & 0x03FF) << 10) | (code2 & 0x03FF)) + 0x00010000 - result.extend(['\\', "U", - hexdigits[(code >> 28) & 0xf], - hexdigits[(code >> 24) & 0xf], - hexdigits[(code >> 20) & 0xf], - hexdigits[(code >> 16) & 0xf], - hexdigits[(code >> 12) & 0xf], - hexdigits[(code >> 8) & 0xf], - hexdigits[(code >> 4) & 0xf], - hexdigits[(code >> 0) & 0xf], - ]) - j += 2 - continue - - if code >= 0x100: - result.extend(['\\', "u", - hexdigits[(code >> 12) & 0xf], - hexdigits[(code >> 8) & 0xf], - hexdigits[(code >> 4) & 0xf], - hexdigits[(code >> 0) & 0xf], - ]) - j += 1 - continue - if code == ord('\\') or code == ord(quote): - result.append('\\') - result.append(chr(code)) - j += 1 - continue - if code == ord('\t'): - result.append('\\') - result.append('t') - j += 1 - continue - if code == ord('\r'): - result.append('\\') - result.append('r') - j += 1 - continue - if code == ord('\n'): - result.append('\\') - result.append('n') - j += 1 - continue - if code < ord(' ') or code >= 0x7f: - result.extend(['\\', "x", - hexdigits[(code >> 4) & 0xf], - hexdigits[(code >> 0) & 0xf], - ]) - j += 1 - continue - result.append(chr(code)) - j += 1 - result.append(quote) - return space.wrap(''.join(result)) - + s = unicode_encode_unicode_escape(chars, size, "strict", quotes=True) + return space.wrap(s) def mod__Unicode_ANY(space, w_format, w_values): return mod_format(space, w_format, w_values, do_unicode=True) Modified: pypy/branch/x86-64-jit-backend/pypy/pytest-A.cfg ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/pytest-A.cfg (original) +++ pypy/branch/x86-64-jit-backend/pypy/pytest-A.cfg Fri Jul 2 17:44:46 2010 @@ -1,4 +1,4 @@ -cherrypick = ['interpreter', 'objspace/test', 'objspace/std', 'module', 'lib'] +cherrypick = ['interpreter', 'objspace/test', 'objspace/std', 'module'] interp = ['translator/goal/pypy-c'] test_driver = ['test_all.py', '-A'] Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/debug.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/debug.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/debug.py Fri Jul 2 17:44:46 2010 @@ -250,3 +250,16 @@ def specialize_call(self, hop): hop.exception_cannot_occur() return hop.inputarg(hop.args_r[0], arg=0) + + +class IntegerCanBeNegative(Exception): + pass + +def _check_nonneg(ann, bk): + from pypy.annotation.model import SomeInteger + s_nonneg = SomeInteger(nonneg=True) + if not s_nonneg.contains(ann): + raise IntegerCanBeNegative + +def check_nonneg(x): + check_annotation(x, _check_nonneg) Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/parsing/test/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/parsing/test/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/parsing/test/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rcoroutine.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rcoroutine.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rcoroutine.py Fri Jul 2 17:44:46 2010 @@ -32,6 +32,8 @@ from pypy.rlib.rstack import yield_current_frame_to_caller, resume_point from pypy.rlib.objectmodel import we_are_translated +from pypy.interpreter.error import OperationError + try: from greenlet import greenlet main_greenlet = greenlet.getcurrent() @@ -158,6 +160,7 @@ self.costate = state self.parent = None self.thunk = None + self.coroutine_exit = False def __repr__(self): 'NOT_RPYTHON' @@ -236,11 +239,12 @@ self = state.current self.finish(exc) except CoroutineExit: - # ignore a shutdown exception pass except Exception, e: - # redirect all unhandled exceptions to the parent - syncstate.push_exception(e) + if self.coroutine_exit is False: + # redirect all unhandled exceptions to the parent + syncstate.push_exception(e) + while self.parent is not None and self.parent.frame is None: # greenlet behavior is fine self.parent = self.parent.parent @@ -257,10 +261,13 @@ syncstate.switched(incoming_frame) def kill(self): + self._kill(CoroutineExit()) + + def _kill(self, exc): if self.frame is None: return state = self.costate - syncstate.push_exception(CoroutineExit()) + syncstate.push_exception(exc) # careful here - if setting self.parent to state.current would # create a loop, break it. The assumption is that 'self' # will die, so that state.current's chain of parents can be Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rmd5.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rmd5.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rmd5.py Fri Jul 2 17:44:46 2010 @@ -2,7 +2,7 @@ """ RPython implementation of MD5 checksums. -See also the pure Python implementation in pypy/lib/md5.py, which might +See also the pure Python implementation in lib_pypy/md5.py, which might or might not be faster than this one on top of CPython. This is an implementation of the MD5 hash function, Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rsdl/test/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rsdl/test/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rsdl/test/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rsha.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rsha.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rsha.py Fri Jul 2 17:44:46 2010 @@ -2,7 +2,7 @@ """A sample implementation of SHA-1 in RPython. - See also the pure Python implementation in pypy/lib/sha.py, which might + See also the pure Python implementation in lib_pypy/sha.py, which might or might not be faster than this one on top of CPython. Framework adapted from Dinu Gherman's MD5 implementation by Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rsre/test/targetrsre.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rsre/test/targetrsre.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rsre/test/targetrsre.py Fri Jul 2 17:44:46 2010 @@ -1,3 +1,4 @@ +#!/usr/bin/env python from pypy.rlib.rarithmetic import intmask from pypy.rlib.rsre import rsre import os, time Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rstring.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rstring.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rstring.py Fri Jul 2 17:44:46 2010 @@ -56,6 +56,7 @@ self.l.append(s) def append_slice(self, s, start, end): + assert 0 <= start <= end <= len(s) self.l.append(s[start:end]) def append_multiple_char(self, c, times): Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/runicode.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/runicode.py Fri Jul 2 17:44:46 2010 @@ -1,7 +1,9 @@ import sys from pypy.rlib.bitmanipulation import splitter from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize +from pypy.rlib.rstring import StringBuilder, UnicodeBuilder +from pypy.rlib.rarithmetic import r_uint if rffi.sizeof(lltype.UniChar) == 4: MAXUNICODE = 0x10ffff @@ -42,8 +44,6 @@ UNICHR = unichr ORD = ord -# XXX review the functions below and think about using stringbuilders for them - def raise_unicode_exception_decode(errors, encoding, msg, s, startingpos, endingpos): @@ -55,8 +55,8 @@ assert isinstance(u, unicode) raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg) -# ____________________________________________________________ -# unicode decoding +# ____________________________________________________________ +# utf-8 utf8_code_length = [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -81,9 +81,10 @@ errorhandler=None): if errorhandler is None: errorhandler = raise_unicode_exception_decode - if (size == 0): + if size == 0: return u'', 0 - result = [] + + result = UnicodeBuilder(size) pos = 0 while pos < size: ch = s[pos] @@ -94,14 +95,14 @@ continue n = utf8_code_length[ordch1] - if (pos + n > size): + if pos + n > size: if not final: break else: r, pos = errorhandler(errors, "utf-8", "unexpected end of data", s, pos, size) result.append(r) - if (pos + n > size): + if pos + n > size: break if n == 0: r, pos = errorhandler(errors, "utf-8", "unexpected code byte", @@ -116,7 +117,7 @@ z, two = splitter[6, 2](ordch2) y, six = splitter[5, 3](ordch1) assert six == 6 - if (two != 2): + if two != 2: r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 2) result.append(r) @@ -137,7 +138,7 @@ y, two2 = splitter[6, 2](ordch2) x, fourteen = splitter[4, 4](ordch1) assert fourteen == 14 - if (two1 != 2 or two2 != 2): + if two1 != 2 or two2 != 2: r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 3) result.append(r) @@ -166,7 +167,7 @@ x, two3 = splitter[6, 2](ordch2) w, thirty = splitter[3, 5](ordch1) assert thirty == 30 - if (two1 != 2 or two2 != 2 or two3 != 2): + if two1 != 2 or two2 != 2 or two3 != 2: r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 4) result.append(r) @@ -174,7 +175,7 @@ c = (w << 18) + (x << 12) + (y << 6) + z # minimum value allowed for 4 byte encoding # maximum value allowed for UTF-16 - if ((c < 0x10000) or (c > 0x10ffff)): + if c < 0x10000 or c > 0x10ffff: r, pos = errorhandler(errors, "utf-8", "illegal encoding", s, pos, pos + 4) result.append(r) @@ -197,8 +198,53 @@ s, pos, pos + n) result.append(r) - return u"".join(result), pos + return result.build(), pos +def _encodeUCS4(result, ch): + # Encode UCS4 Unicode ordinals + result.append((chr((0xf0 | (ch >> 18))))) + result.append((chr((0x80 | ((ch >> 12) & 0x3f))))) + result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) + result.append((chr((0x80 | (ch & 0x3f))))) + +def unicode_encode_utf_8(s, size, errors, errorhandler=None): + assert(size >= 0) + result = StringBuilder(size) + i = 0 + while i < size: + ch = ord(s[i]) + i += 1 + if ch < 0x80: + # Encode ASCII + result.append(chr(ch)) + elif ch < 0x0800: + # Encode Latin-1 + result.append(chr((0xc0 | (ch >> 6)))) + result.append(chr((0x80 | (ch & 0x3f)))) + else: + # Encode UCS2 Unicode ordinals + if ch < 0x10000: + # Special case: check for high surrogate + if 0xD800 <= ch <= 0xDBFF and i != size: + ch2 = ord(s[i]) + # Check for low surrogate and combine the two to + # form a UCS4 value + if 0xDC00 <= ch2 <= 0xDFFF: + ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000 + i += 1 + _encodeUCS4(result, ch3) + continue + # Fall through: handles isolated high surrogates + result.append((chr((0xe0 | (ch >> 12))))) + result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) + result.append((chr((0x80 | (ch & 0x3f))))) + continue + else: + _encodeUCS4(result, ch) + return result.build() + +# ____________________________________________________________ +# utf-16 def str_decode_utf_16(s, size, errors, final=True, errorhandler=None): @@ -238,12 +284,11 @@ # mark is skipped, in all other modes, it is copied to the output # stream as-is (giving a ZWNBSP character). pos = 0 - result = [] if byteorder == 'native': - if (size >= 2): + if size >= 2: bom = (ord(s[ihi]) << 8) | ord(s[ilo]) if BYTEORDER == 'little': - if (bom == 0xFEFF): + if bom == 0xFEFF: pos += 2 bo = -1 elif bom == 0xFFFE: @@ -260,20 +305,22 @@ bo = -1 else: bo = 1 - if (size == 0): + if size == 0: return u'', 0, bo - if (bo == -1): + if bo == -1: # force little endian ihi = 1 ilo = 0 - elif (bo == 1): + elif bo == 1: # force big endian ihi = 0 ilo = 1 + result = UnicodeBuilder(size // 2) + #XXX I think the errors are not correctly handled here - while (pos < len(s)): + while pos < size: # remaining bytes at the end? (size should be even) if len(s) - pos < 2: if not final: @@ -285,7 +332,7 @@ break ch = (ord(s[pos + ihi]) << 8) | ord(s[pos + ilo]) pos += 2 - if (ch < 0xD800 or ch > 0xDFFF): + if ch < 0xD800 or ch > 0xDFFF: result.append(unichr(ch)) continue # UTF-16 code pair: @@ -297,10 +344,10 @@ result.append(r) if len(s) - pos < 2: break - elif (0xD800 <= ch and ch <= 0xDBFF): + elif 0xD800 <= ch <= 0xDBFF: ch2 = (ord(s[pos+ihi]) << 8) | ord(s[pos+ilo]) pos += 2 - if (0xDC00 <= ch2 and ch2 <= 0xDFFF): + if 0xDC00 <= ch2 <= 0xDFFF: if MAXUNICODE < 65536: result.append(unichr(ch)) result.append(unichr(ch2)) @@ -318,17 +365,305 @@ "illegal encoding", s, pos - 2, pos) result.append(r) - return u"".join(result), pos, bo + return result.build(), pos, bo + +def _STORECHAR(result, CH, byteorder): + hi = chr(((CH) >> 8) & 0xff) + lo = chr((CH) & 0xff) + if byteorder == 'little': + result.append(lo) + result.append(hi) + else: + result.append(hi) + result.append(lo) + +def unicode_encode_utf_16_helper(s, size, errors, + errorhandler=None, + byteorder='little'): + if size == 0: + return "" + + result = StringBuilder(size * 2 + 2) + if byteorder == 'native': + _STORECHAR(result, 0xFEFF, BYTEORDER) + byteorder = BYTEORDER + + i = 0 + while i < size: + ch = ord(s[i]) + i += 1 + ch2 = 0 + if ch >= 0x10000: + ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) + ch = 0xD800 | ((ch-0x10000) >> 10) + + _STORECHAR(result, ch, byteorder) + if ch2: + _STORECHAR(result, ch2, byteorder) + + return result.build() + +def unicode_encode_utf_16(s, size, errors, + errorhandler=None): + return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "native") + + +def unicode_encode_utf_16_be(s, size, errors, + errorhandler=None): + return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "big") + + +def unicode_encode_utf_16_le(s, size, errors, + errorhandler=None): + return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "little") + + +# ____________________________________________________________ +# utf-7 + +## indicate whether a UTF-7 character is special i.e. cannot be directly +## encoded: +## 0 - not special +## 1 - special +## 2 - whitespace (optional) +## 3 - RFC2152 Set O (optional) + +_utf7_special = [ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, + 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3, + 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1, +] + +def _utf7_SPECIAL(oc, encodeO=False, encodeWS=False): + return (oc > 127 or _utf7_special[oc] == 1 or + (encodeWS and _utf7_special[oc] == 2) or + (encodeO and _utf7_special[oc] == 3)) + +def _utf7_B64CHAR(oc): + if oc > 127: + return False + c = chr(oc) + return c.isalnum() or c == '+' or c == '/' +def _utf7_TO_BASE64(n): + "Returns the base-64 character of the bottom 6 bits of n" + return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[n & 0x3f] +def _utf7_FROM_BASE64(c): + "Retuns the base-64 value of a base-64 character" + if c == '+': + return 62 + elif c == '/': + return 63 + elif c >= 'a': + return ord(c) - 71 + elif c >= 'A': + return ord(c) - 65 + else: + return ord(c) + 4 + +def _utf7_ENCODE(result, ch, bits): + while bits >= 6: + result.append(_utf7_TO_BASE64(ch >> (bits - 6))) + bits -= 6 + return bits + +def _utf7_DECODE(s, result, errorhandler, errors, + pos, charsleft, bitsleft, surrogate): + while bitsleft >= 16: + outCh = (charsleft >> (bitsleft-16)) & 0xffff + bitsleft -= 16 + + if surrogate: + ## We have already generated an error for the high + ## surrogate so let's not bother seeing if the low + ## surrogate is correct or not + surrogate = False + elif 0xDC00 <= outCh <= 0xDFFF: + ## This is a surrogate pair. Unfortunately we can't + ## represent it in a 16-bit character + surrogate = True + msg = "code pairs are not supported" + res, pos = errorhandler(errors, 'utf-7', + msg, s, pos-1, pos) + result.append(res) + bitsleft = 0 + break + else: + result.append(unichr(outCh)) + return pos, charsleft, bitsleft, surrogate + + +def str_decode_utf_7(s, size, errors, final=False, + errorhandler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode + if size == 0: + return u'', 0 + + inShift = False + bitsleft = 0 + startinpos = 0 + charsleft = 0 + surrogate = False + + result = UnicodeBuilder(size) + pos = 0 + while pos < size: + ch = s[pos] + oc = ord(ch) + + if inShift: + if ch == '-' or not _utf7_B64CHAR(oc): + inShift = 0 + pos += 1 + + pos, charsleft, bitsleft, surrogate = _utf7_DECODE( + s, result, errorhandler, errors, + pos, charsleft, bitsleft, surrogate) + if bitsleft >= 6: + ## The shift sequence has a partial character in it. If + ## bitsleft < 6 then we could just classify it as padding + ## but that is not the case here + msg = "partial character in shift sequence" + res, pos = errorhandler(errors, 'utf-7', + msg, s, pos-1, pos) + result.append(res) + ## According to RFC2152 the remaining bits should be + ## zero. We choose to signal an error/insert a replacement + ## character here so indicate the potential of a + ## misencoded character. + if ch == '-': + if pos < size and s[pos] == '-': + result.append(u'-') + inShift = True + + elif _utf7_SPECIAL(oc): + msg = "unexpected special character" + res, pos = errorhandler(errors, 'utf-7', + msg, s, pos-1, pos) + result.append(res) + else: + result.append(unichr(ord(ch))) + else: + charsleft = (charsleft << 6) | _utf7_FROM_BASE64(ch) + bitsleft += 6 + pos += 1 + + pos, charsleft, bitsleft, surrogate = _utf7_DECODE( + s, result, errorhandler, errors, + pos, charsleft, bitsleft, surrogate) + elif ch == '+': + startinpos = pos + pos += 1 + if pos < size and s[pos] == '-': + pos += 1 + result.append(u'+') + else: + inShift = 1 + bitsleft = 0 + + elif _utf7_SPECIAL(oc): + pos += 1 + msg = "unexpected special character" + res, pos = errorhandler(errors, 'utf-7', msg, s, pos-1, pos) + result.append(res) + else: + result.append(unichr(oc)) + pos += 1 + + if inShift: + endinpos = size + msg = "unterminated shift sequence" + res, pos = errorhandler(errors, 'utf-7', msg, s, startinpos, pos) + result.append(res) + + return result.build(), pos + +def unicode_encode_utf_7(s, size, errors, errorhandler=None): + if size == 0: + return '' + result = StringBuilder(size) + + encodeSetO = encodeWhiteSpace = False + + inShift = False + bitsleft = 0 + charsleft = 0 + + pos = 0 + while pos < size: + ch = s[pos] + oc = ord(ch) + if not inShift: + if ch == u'+': + result.append('+-') + elif _utf7_SPECIAL(oc, encodeSetO, encodeWhiteSpace): + charsleft = oc + bitsleft = 16 + result.append('+') + bitsleft = _utf7_ENCODE(result, charsleft, bitsleft) + inShift = bitsleft > 0 + else: + result.append(chr(oc)) + else: + if not _utf7_SPECIAL(oc, encodeSetO, encodeWhiteSpace): + result.append(_utf7_TO_BASE64(charsleft << (6-bitsleft))) + charsleft = 0 + bitsleft = 0 + ## Characters not in the BASE64 set implicitly unshift the + ## sequence so no '-' is required, except if the character is + ## itself a '-' + if _utf7_B64CHAR(oc) or ch == u'-': + result.append('-') + inShift = False + result.append(chr(oc)) + else: + bitsleft += 16 + charsleft = (charsleft << 16) | oc + bitsleft = _utf7_ENCODE(result, charsleft, bitsleft) + ## If the next character is special then we dont' need to + ## terminate the shift sequence. If the next character is not + ## a BASE64 character or '-' then the shift sequence will be + ## terminated implicitly and we don't have to insert a '-'. + if bitsleft == 0: + if pos + 1 < size: + ch2 = s[pos + 1] + oc2 = ord(ch2) + + if _utf7_SPECIAL(oc2, encodeSetO, encodeWhiteSpace): + pass + elif _utf7_B64CHAR(oc2) or ch2 == u'-': + result.append('-') + inShift = False + else: + inShift = False + else: + result.append('-') + inShift = False + pos += 1 + + if bitsleft: + result.append(_utf7_TO_BASE64(charsleft << (6 - bitsleft))) + result.append('-') + + return result.build() + +# ____________________________________________________________ +# ascii and latin-1 def str_decode_latin_1(s, size, errors, final=False, errorhandler=None): # latin1 is equivalent to the first 256 ordinals in Unicode. pos = 0 - result = [] - while (pos < size): + result = UnicodeBuilder(size) + while pos < size: result.append(unichr(ord(s[pos]))) pos += 1 - return u"".join(result), pos + return result.build(), pos def str_decode_ascii(s, size, errors, final=False, @@ -336,9 +671,9 @@ if errorhandler is None: errorhandler = raise_unicode_exception_decode # ASCII is equivalent to the first 128 ordinals in Unicode. - result = [] + result = UnicodeBuilder(size) pos = 0 - while pos < len(s): + while pos < size: c = s[pos] if ord(c) < 128: result.append(unichr(ord(c))) @@ -347,55 +682,7 @@ r, pos = errorhandler(errors, "ascii", "ordinal not in range(128)", s, pos, pos + 1) result.append(r) - return u"".join(result), pos - - -# ____________________________________________________________ -# unicode encoding - - -def unicode_encode_utf_8(s, size, errors, errorhandler=None): - assert(size >= 0) - result = [] - i = 0 - while i < size: - ch = ord(s[i]) - i += 1 - if (ch < 0x80): - # Encode ASCII - result.append(chr(ch)) - elif (ch < 0x0800) : - # Encode Latin-1 - result.append(chr((0xc0 | (ch >> 6)))) - result.append(chr((0x80 | (ch & 0x3f)))) - else: - # Encode UCS2 Unicode ordinals - if (ch < 0x10000): - # Special case: check for high surrogate - if (0xD800 <= ch and ch <= 0xDBFF and i != size) : - ch2 = ord(s[i]) - # Check for low surrogate and combine the two to - # form a UCS4 value - if (0xDC00 <= ch2 and ch2 <= 0xDFFF) : - ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000 - i += 1 - _encodeUCS4(result, ch3) - continue - # Fall through: handles isolated high surrogates - result.append((chr((0xe0 | (ch >> 12))))) - result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) - result.append((chr((0x80 | (ch & 0x3f))))) - continue - else: - _encodeUCS4(result, ch) - return "".join(result) - -def _encodeUCS4(result, ch): - # Encode UCS4 Unicode ordinals - result.append((chr((0xf0 | (ch >> 18))))) - result.append((chr((0x80 | ((ch >> 12) & 0x3f))))) - result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) - result.append((chr((0x80 | (ch & 0x3f))))) + return result.build(), pos def unicode_encode_ucs1_helper(p, size, errors, @@ -408,12 +695,12 @@ else: reason = "ordinal not in range(128)" encoding = "ascii" - - if (size == 0): + + if size == 0: return '' - result = [] + result = StringBuilder(size) pos = 0 - while pos < len(p): + while pos < size: ch = p[pos] if ord(ch) < limit: @@ -427,9 +714,9 @@ collend += 1 r, pos = errorhandler(errors, encoding, reason, p, collstart, collend) - result += r # extend 'result' as a list of characters + result.append(r) - return "".join(result) + return result.build() def unicode_encode_latin_1(p, size, errors, errorhandler=None): res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 256) @@ -439,57 +726,479 @@ res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 128) return res +# ____________________________________________________________ +# Charmap -def _STORECHAR(result, CH, byteorder): - hi = chr(((CH) >> 8) & 0xff) - lo = chr((CH) & 0xff) - if byteorder == 'little': - result.append(lo) - result.append(hi) +ERROR_CHAR = u'\ufffe' + + at specialize.argtype(5) +def str_decode_charmap(s, size, errors, final=False, + errorhandler=None, mapping=None): + "mapping can be a rpython dictionary, or a dict-like object." + + # Default to Latin-1 + if mapping is None: + return str_decode_latin_1(s, size, errors, final=final, + errorhandler=errorhandler) + if errorhandler is None: + errorhandler = raise_unicode_exception_decode + if size == 0: + return u'', 0 + + pos = 0 + result = UnicodeBuilder(size) + while pos < size: + ch = s[pos] + + c = mapping.get(ch, ERROR_CHAR) + if c == ERROR_CHAR: + r, pos = errorhandler(errors, "charmap", + "character maps to ", + s, pos, pos + 1) + result.append(r) + continue + result.append(c) + pos += 1 + return result.build(), pos + +def unicode_encode_charmap(s, size, errors, errorhandler=None, + mapping=None): + if mapping is None: + return unicode_encode_latin_1(s, size, errors, + errorhandler=errorhandler) + + if errorhandler is None: + errorhandler = raise_unicode_exception_encode + + if size == 0: + return '' + result = StringBuilder(size) + pos = 0 + while pos < size: + ch = s[pos] + + c = mapping.get(ch, '') + if len(c) == 0: + res, pos = errorhandler(errors, "charmap", + "character maps to ", + s, pos, pos + 1) + for ch2 in res: + c2 = mapping.get(unichr(ord(ch2)), '') + if len(c2) == 0: + errorhandler( + "strict", "charmap", + "character maps to ", + s, pos, pos + 1) + result.append(c2) + continue + result.append(c) + pos += 1 + return result.build() + +# ____________________________________________________________ +# Unicode escape + +hexdigits = "0123456789ABCDEFabcdef" + +def hexescape(builder, s, pos, digits, + encoding, errorhandler, message, errors): + import sys + chr = 0 + if pos + digits > len(s): + message = "end of string in escape sequence" + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-2, len(s)) + builder.append(res) else: - result.append(hi) - result.append(lo) + try: + chr = r_uint(int(s[pos:pos+digits], 16)) + except ValueError: + endinpos = pos + while s[endinpos] in hexdigits: + endinpos += 1 + res, pos = errorhandler(errors, encoding, + message, s, pos-2, endinpos+1) + builder.append(res) + else: + # when we get here, chr is a 32-bit unicode character + if chr <= MAXUNICODE: + builder.append(UNICHR(chr)) + pos += digits + + elif chr <= 0x10ffff: + chr -= 0x10000L + builder.append(unichr(0xD800 + (chr >> 10))) + builder.append(unichr(0xDC00 + (chr & 0x03FF))) + pos += digits + else: + message = "illegal Unicode character" + res, pos = errorhandler(errors, encoding, + message, s, pos-2, pos+digits) + builder.append(res) + return pos + +def str_decode_unicode_escape(s, size, errors, final=False, + errorhandler=False, + unicodedata_handler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode -def unicode_encode_utf_16_helper(s, size, errors, - errorhandler=None, - byteorder='little'): - result = [] - if (byteorder == 'native'): - _STORECHAR(result, 0xFEFF, BYTEORDER) - byteorder = BYTEORDER - if size == 0: - return "" + return u'', 0 - i = 0 - while i < size: - ch = ord(s[i]) - i += 1 - ch2 = 0 - if (ch >= 0x10000) : - ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) - ch = 0xD800 | ((ch-0x10000) >> 10) + builder = UnicodeBuilder(size) + pos = 0 + while pos < size: + ch = s[pos] - _STORECHAR(result, ch, byteorder) - if ch2: - _STORECHAR(result, ch2, byteorder) + # Non-escape characters are interpreted as Unicode ordinals + if ch != '\\': + builder.append(unichr(ord(ch))) + pos += 1 + continue - return "".join(result) + # - Escapes + pos += 1 + if pos >= size: + message = "\\ at end of string" + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, size) + builder.append(res) + continue -def unicode_encode_utf_16(s, size, errors, - errorhandler=None): - return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "native") + ch = s[pos] + pos += 1 + # \x escapes + if ch == '\n': pass + elif ch == '\\': builder.append(u'\\') + elif ch == '\'': builder.append(u'\'') + elif ch == '\"': builder.append(u'\"') + elif ch == 'b' : builder.append(u'\b') + elif ch == 'f' : builder.append(u'\f') + elif ch == 't' : builder.append(u'\t') + elif ch == 'n' : builder.append(u'\n') + elif ch == 'r' : builder.append(u'\r') + elif ch == 'v' : builder.append(u'\v') + elif ch == 'a' : builder.append(u'\a') + elif '0' <= ch <= '7': + x = ord(ch) - ord('0') + if pos < size: + ch = s[pos] + if '0' <= ch <= '7': + pos += 1 + x = (x<<3) + ord(ch) - ord('0') + if pos < size: + ch = s[pos] + if '0' <= ch <= '7': + pos += 1 + x = (x<<3) + ord(ch) - ord('0') + builder.append(unichr(x)) + # hex escapes + # \xXX + elif ch == 'x': + digits = 2 + message = "truncated \\xXX escape" + pos = hexescape(builder, s, pos, digits, + "unicodeescape", errorhandler, message, errors) + + # \uXXXX + elif ch == 'u': + digits = 4 + message = "truncated \\uXXXX escape" + pos = hexescape(builder, s, pos, digits, + "unicodeescape", errorhandler, message, errors) + + # \UXXXXXXXX + elif ch == 'U': + digits = 8 + message = "truncated \\UXXXXXXXX escape" + pos = hexescape(builder, s, pos, digits, + "unicodeescape", errorhandler, message, errors) + + # \N{name} + elif ch == 'N': + message = "malformed \\N character escape" + look = pos + if unicodedata_handler is None: + message = ("\\N escapes not supported " + "(can't load unicodedata module)") + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, size) + builder.append(res) + continue + + if look < size and s[look] == '{': + # look for the closing brace + while look < size and s[look] != '}': + look += 1 + if look < size and s[look] == '}': + # found a name. look it up in the unicode database + message = "unknown Unicode character name" + name = s[pos+1:look] + code = unicodedata_handler.call(name) + if code < 0: + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, look+1) + builder.append(res) + continue + pos = look + 1 + if code <= MAXUNICODE: + builder.append(UNICHR(code)) + else: + code -= 0x10000L + builder.append(unichr(0xD800 + (code >> 10))) + builder.append(unichr(0xDC00 + (code & 0x03FF))) + else: + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, look+1) + builder.append(res) + else: + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, look+1) + builder.append(res) + else: + builder.append(u'\\') + builder.append(unichr(ord(ch))) + return builder.build(), pos -def unicode_encode_utf_16_be(s, size, errors, - errorhandler=None): - return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "big") +def unicode_encode_unicode_escape(s, size, errors, errorhandler=None, quotes=False): + # errorhandler is not used: this function cannot cause Unicode errors + result = StringBuilder(size) + + if quotes: + if s.find(u'\'') != -1 and s.find(u'\"') == -1: + quote = ord('\"') + result.append('u"') + else: + quote = ord('\'') + result.append('u\'') + else: + quote = 0 + if size == 0: + return '' -def unicode_encode_utf_16_le(s, size, errors, - errorhandler=None): - return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "little") + pos = 0 + while pos < size: + ch = s[pos] + oc = ord(ch) + + # Escape quotes + if quotes and (oc == quote or ch == '\\'): + result.append('\\') + result.append(chr(oc)) + pos += 1 + continue + + if 0xD800 <= oc < 0xDC00 and pos + 1 < size: + # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes + pos += 1 + oc2 = ord(s[pos]) + + if 0xDC00 <= oc2 <= 0xDFFF: + ucs = (((oc & 0x03FF) << 10) | (oc2 & 0x03FF)) + 0x00010000 + raw_unicode_escape_helper(result, ucs) + pos += 1 + continue + # Fall through: isolated surrogates are copied as-is + pos -= 1 + + # Map special whitespace to '\t', \n', '\r' + if ch == '\t': + result.append('\\t') + elif ch == '\n': + result.append('\\n') + elif ch == '\r': + result.append('\\r') + elif ch == '\\': + result.append('\\\\') + + # Map non-printable or non-ascii to '\xhh' or '\uhhhh' + elif oc < 32 or oc >= 0x7F: + raw_unicode_escape_helper(result, oc) + + # Copy everything else as-is + else: + result.append(chr(oc)) + pos += 1 + + if quotes: + result.append(chr(quote)) + return result.build() + +# ____________________________________________________________ +# Raw unicode escape + +def str_decode_raw_unicode_escape(s, size, errors, final=False, + errorhandler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode + if size == 0: + return u'', 0 + + result = UnicodeBuilder(size) + pos = 0 + while pos < size: + ch = s[pos] + + # Non-escape characters are interpreted as Unicode ordinals + if ch != '\\': + result.append(unichr(ord(ch))) + pos += 1 + continue + + startinpos = pos + # \u-escapes are only interpreted iff the number of leading + # backslashes is odd + bs = pos + while pos < size: + pos += 1 + if pos == size or s[pos] != '\\': + break + result.append(u'\\') + + # we have a backslash at the end of the string, stop here + if pos >= size: + result.append(u'\\') + break + + if ((pos - bs) & 1 == 0 or + pos >= size or + (s[pos] != 'u' and s[pos] != 'U')): + result.append(u'\\') + result.append(unichr(ord(s[pos]))) + pos += 1 + continue + + if s[pos] == 'u': + digits = 4 + message = "truncated \\uXXXX escape" + else: + digits = 8 + message = "truncated \\UXXXXXXXX escape" + pos += 1 + pos = hexescape(result, s, pos, digits, + "rawunicodeescape", errorhandler, message, errors) + + return result.build(), pos + +def raw_unicode_escape_helper(result, char): + num = hex(char) + if char >= 0x10000: + result.append("\\U") + zeros = 8 + elif char >= 0x100: + result.append("\\u") + zeros = 4 + else: + result.append("\\x") + zeros = 2 + lnum = len(num) + nb = zeros + 2 - lnum # num starts with '0x' + if nb > 0: + result.append_multiple_char('0', nb) + result.append_slice(num, 2, lnum) + +def unicode_encode_raw_unicode_escape(s, size, errors, errorhandler=None): + # errorhandler is not used: this function cannot cause Unicode errors + if size == 0: + return '' + result = StringBuilder(size) + pos = 0 + while pos < size: + oc = ord(s[pos]) + if oc < 0x100: + result.append(chr(oc)) + else: + raw_unicode_escape_helper(result, oc) + pos += 1 + + return result.build() + +# ____________________________________________________________ +# unicode-internal + +def str_decode_unicode_internal(s, size, errors, final=False, + errorhandler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode + if size == 0: + return u'', 0 + + if MAXUNICODE < 65536: + unicode_bytes = 2 + else: + unicode_bytes = 4 + if BYTEORDER == "little": + start = 0 + stop = unicode_bytes + step = 1 + else: + start = unicode_bytes - 1 + stop = -1 + step = -1 + + result = UnicodeBuilder(size // unicode_bytes) + pos = 0 + while pos < size: + if pos > size - unicode_bytes: + res, pos = errorhandler(errors, "unicode_internal", + "truncated input", + s, pos, size) + result.append(res) + if pos > size - unicode_bytes: + break + continue + t = r_uint(0) + h = 0 + for j in range(start, stop, step): + t += r_uint(ord(s[pos + j])) << (h*8) + h += 1 + if t > MAXUNICODE: + res, pos = errorhandler(errors, "unicode_internal", + "unichr(%d) not in range" % (t,), + s, pos, pos + unicode_bytes) + result.append(res) + continue + result.append(unichr(t)) + pos += unicode_bytes + return result.build(), pos + +def unicode_encode_unicode_internal(s, size, errors, errorhandler=None): + if size == 0: + return '' + + if MAXUNICODE < 65536: + unicode_bytes = 2 + else: + unicode_bytes = 4 + + result = StringBuilder(size * unicode_bytes) + pos = 0 + while pos < size: + oc = ord(s[pos]) + if MAXUNICODE < 65536: + if BYTEORDER == "little": + result.append(chr(oc & 0xFF)) + result.append(chr(oc >> 8 & 0xFF)) + else: + result.append(chr(oc >> 8 & 0xFF)) + result.append(chr(oc & 0xFF)) + else: + if BYTEORDER == "little": + result.append(chr(oc & 0xFF)) + result.append(chr(oc >> 8 & 0xFF)) + result.append(chr(oc >> 16 & 0xFF)) + result.append(chr(oc >> 24 & 0xFF)) + else: + result.append(chr(oc >> 24 & 0xFF)) + result.append(chr(oc >> 16 & 0xFF)) + result.append(chr(oc >> 8 & 0xFF)) + result.append(chr(oc & 0xFF)) + pos += 1 + return result.build() # ____________________________________________________________ # MBCS codecs for Windows Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rzipfile.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rzipfile.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rzipfile.py Fri Jul 2 17:44:46 2010 @@ -12,7 +12,8 @@ rzlib = None # XXX hack to get crc32 to work -from pypy.lib.binascii import crc_32_tab +from pypy.tool.lib_pypy import import_from_lib_pypy +crc_32_tab = import_from_lib_pypy('binascii').crc_32_tab rcrc_32_tab = [r_uint(i) for i in crc_32_tab] Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_debug.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_debug.py Fri Jul 2 17:44:46 2010 @@ -3,6 +3,7 @@ from pypy.rlib.debug import check_annotation, make_sure_not_resized from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rlib.debug import have_debug_prints +from pypy.rlib.debug import check_nonneg, IntegerCanBeNegative from pypy.rlib import debug from pypy.rpython.test.test_llinterp import interpret @@ -30,6 +31,16 @@ py.test.raises(Error, "interpret(g, [3])") +def test_check_nonneg(): + def f(x): + assert x >= 5 + check_nonneg(x) + interpret(f, [9]) + + def g(x): + check_nonneg(x-1) + py.test.raises(IntegerCanBeNegative, interpret, g, [9]) + def test_make_sure_not_resized(): from pypy.annotation.listdef import TooLateForChange def f(): Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_runicode.py Fri Jul 2 17:44:46 2010 @@ -5,11 +5,14 @@ def test_unichr(): a = runicode.UNICHR(0xffff) assert a == u'\uffff' - a = runicode.UNICHR(0x10000) - if sys.maxunicode < 0x10000: - assert len(a) == 2 # surrogates + if runicode.MAXUNICODE > 0xffff: + a = runicode.UNICHR(0x10000) + if sys.maxunicode < 0x10000: + assert len(a) == 2 # surrogates + else: + assert len(a) == 1 else: - assert len(a) == 1 + py.test.raises(ValueError, runicode.UNICHR, 0x10000) class UnicodeTests(object): @@ -228,7 +231,7 @@ class TestTranslation(object): def setup_class(cls): - if len(runicode.UNICHR(0x10000)) == 2: + if runicode.MAXUNICODE != sys.maxunicode: py.test.skip("these tests cannot run on the llinterp") def test_utf8(self): Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/extfunc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/extfunc.py Fri Jul 2 17:44:46 2010 @@ -94,6 +94,7 @@ class BaseLazyRegistering(object): __metaclass__ = LazyRegisteringMeta + compilation_info = None def configure(self, CConfig): classes_seen = self.__dict__.setdefault('__classes_seen', {}) @@ -101,7 +102,11 @@ return from pypy.rpython.tool import rffi_platform as platform # copy some stuff - self.compilation_info = CConfig._compilation_info_ + if self.compilation_info is None: + self.compilation_info = CConfig._compilation_info_ + else: + self.compilation_info = self.compilation_info.merge( + CConfig._compilation_info_) self.__dict__.update(platform.configure(CConfig)) classes_seen[CConfig] = True Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/lltype.py Fri Jul 2 17:44:46 2010 @@ -9,7 +9,7 @@ from pypy.rlib.objectmodel import Symbolic from pypy.tool.uid import Hashable from pypy.tool.tls import tlsobject -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict from types import NoneType from sys import maxint import weakref Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rclass.py Fri Jul 2 17:44:46 2010 @@ -21,7 +21,7 @@ from pypy.annotation import model as annmodel from pypy.rlib.rarithmetic import intmask from pypy.rlib import objectmodel -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict from pypy.rpython.lltypesystem.lloperation import llop # Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rffi.py Fri Jul 2 17:44:46 2010 @@ -642,16 +642,8 @@ allows for the process to be performed without an extra copy. Make sure to call keep_buffer_alive_until_here on the returned values. """ - str_chars_offset = (offsetof(STRTYPE, 'chars') + \ - itemoffsetof(STRTYPE.chars, 0)) - gc_buf = rgc.malloc_nonmovable(STRTYPE, count) - if gc_buf: - realbuf = cast_ptr_to_adr(gc_buf) + str_chars_offset - raw_buf = cast(TYPEP, realbuf) - return raw_buf, gc_buf - else: - raw_buf = lltype.malloc(TYPEP.TO, count, flavor='raw') - return raw_buf, lltype.nullptr(STRTYPE) + raw_buf = lltype.malloc(TYPEP.TO, count, flavor='raw') + return raw_buf, lltype.nullptr(STRTYPE) alloc_buffer._always_inline_ = True # to get rid of the returned tuple # (char*, str, int, int) -> None Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/test/test_lltype.py Fri Jul 2 17:44:46 2010 @@ -1,7 +1,7 @@ import py from pypy.rpython.lltypesystem.lltype import * from pypy.rpython.lltypesystem import lltype, rffi -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict def isweak(p, T): try: Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/memory/gctypelayout.py Fri Jul 2 17:44:46 2010 @@ -3,7 +3,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import ll_assert -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict class GCData(object): Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/microbench/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/microbench/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/microbench/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/microbench/microbench.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/microbench/microbench.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/microbench/microbench.py Fri Jul 2 17:44:46 2010 @@ -3,7 +3,7 @@ import sys import autopath from time import clock -from py.compat import subprocess +import subprocess from pypy.translator.interactive import Translation LOOPS = 10000000 Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_time.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_time.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_time.py Fri Jul 2 17:44:46 2010 @@ -17,24 +17,28 @@ STRUCT_TIMEB = 'struct __timeb64' includes = ['winsock2.h', 'windows.h', TIME_H, 'sys/types.h', 'sys/timeb.h'] + need_rusage = False else: TIME_H = 'sys/time.h' FTIME = 'ftime' STRUCT_TIMEB = 'struct timeb' includes = [TIME_H, 'time.h', 'errno.h', 'sys/select.h', - 'sys/types.h', 'unistd.h', 'sys/timeb.h'] + 'sys/types.h', 'unistd.h', 'sys/timeb.h', + 'sys/time.h', 'sys/resource.h'] + need_rusage = True class CConfig: _compilation_info_ = ExternalCompilationInfo( includes=includes ) - CLOCK_T = platform.SimpleType('clock_t', rffi.INT) TIMEVAL = platform.Struct('struct timeval', [('tv_sec', rffi.INT), ('tv_usec', rffi.INT)]) HAVE_GETTIMEOFDAY = platform.Has('gettimeofday') HAVE_FTIME = platform.Has(FTIME) - + if need_rusage: + RUSAGE = platform.Struct('struct rusage', [('ru_utime', TIMEVAL), + ('ru_stime', TIMEVAL)]) if sys.platform == 'freebsd7': libraries = ['compat'] @@ -49,21 +53,20 @@ TIMEB = platform.Struct(STRUCT_TIMEB, [('time', rffi.INT), ('millitm', rffi.INT)]) -constant_names = ['CLOCKS_PER_SEC', 'CLK_TCK', 'EINTR'] +constant_names = ['RUSAGE_SELF', 'EINTR'] for const in constant_names: setattr(CConfig, const, platform.DefinedConstantInteger(const)) defs_names = ['GETTIMEOFDAY_NO_TZ'] for const in defs_names: setattr(CConfig, const, platform.Defined(const)) +def decode_timeval(t): + return (float(rffi.cast(lltype.Signed, t.c_tv_sec)) + + float(rffi.cast(lltype.Signed, t.c_tv_usec)) * 0.000001) + class RegisterTime(BaseLazyRegistering): def __init__(self): self.configure(CConfig) - if self.CLOCKS_PER_SEC is None: - if self.CLK_TCK is None: - self.CLOCKS_PER_SEC = 1000000 - else: - self.CLOCKS_PER_SEC = self.CLK_TCK self.TIMEVALP = lltype.Ptr(self.TIMEVAL) @registering(time.time) @@ -109,10 +112,7 @@ errcode = c_gettimeofday(t, void) if rffi.cast(rffi.LONG, errcode) == 0: - result = float(rffi.cast(lltype.Signed, t.c_tv_sec)) \ - + float(rffi.cast(lltype.Signed, t.c_tv_usec)) \ - * 0.000001 - + result = decode_timeval(t) lltype.free(t, flavor='raw') if result != -1: return result @@ -129,8 +129,6 @@ @registering(time.clock) def register_time_clock(self): - c_clock = self.llexternal('clock', [], self.CLOCK_T, - threadsafe=False) if sys.platform == 'win32': # hacking to avoid LARGE_INTEGER which is a union... A = lltype.FixedSizeArray(lltype.SignedLongLong, 1) @@ -157,9 +155,19 @@ lltype.free(a, flavor='raw') return float(diff) / state.divisor else: + RUSAGE = self.RUSAGE + RUSAGE_SELF = self.RUSAGE_SELF or 0 + c_getrusage = self.llexternal('getrusage', + [rffi.INT, lltype.Ptr(RUSAGE)], + lltype.Void, + threadsafe=False) def time_clock_llimpl(): - result = c_clock() - return float(result) / self.CLOCKS_PER_SEC + a = lltype.malloc(RUSAGE, flavor='raw') + c_getrusage(RUSAGE_SELF, a) + result = (decode_timeval(a.c_ru_utime) + + decode_timeval(a.c_ru_stime)) + lltype.free(a, flavor='raw') + return result return extdef([], float, llimpl=time_clock_llimpl, export_name='ll_time.ll_time_clock') Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/ootypesystem/rclass.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/ootypesystem/rclass.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/ootypesystem/rclass.py Fri Jul 2 17:44:46 2010 @@ -10,7 +10,7 @@ from pypy.rpython.exceptiondata import standardexceptions from pypy.tool.pairtype import pairtype from pypy.tool.sourcetools import func_with_new_name -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict OBJECT = ootype.ROOT META = ootype.Instance("Meta", ootype.ROOT, Modified: pypy/branch/x86-64-jit-backend/pypy/tool/algo/graphlib.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/algo/graphlib.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/algo/graphlib.py Fri Jul 2 17:44:46 2010 @@ -6,7 +6,7 @@ 'edges' is a dict mapping vertices to a list of edges with its source. Note that we can usually use 'edges' as the set of 'vertices' too. """ -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict class Edge: Modified: pypy/branch/x86-64-jit-backend/pypy/tool/algo/test/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/algo/test/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/algo/test/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/tool/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/tool/compat.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/compat.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/compat.py Fri Jul 2 17:44:46 2010 @@ -6,5 +6,8 @@ try: from md5 import md5 except ImportError: - # no _md5 module on this platform - from pypy.lib.md5 import md5 + # no _md5 module on this platform. Try hard to find a pure-python one + # by fishing it from lib_pypy + from pypy.tool.lib_pypy import import_from_lib_pypy + md5 = import_from_lib_pypy('md5') + del import_from_lib_pypy Modified: pypy/branch/x86-64-jit-backend/pypy/tool/pytest/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/pytest/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/pytest/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/tool/pytest/confpath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/pytest/confpath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/pytest/confpath.py Fri Jul 2 17:44:46 2010 @@ -1,11 +1,12 @@ import autopath import py import pypy +from pypy.tool import lib_pypy pypydir = py.path.local(pypy.__file__).dirpath() distdir = pypydir.dirpath() testresultdir = distdir.join('testresult') assert pypydir.check(dir=1) -libpythondir = distdir.join('lib-python') -regrtestdir = libpythondir.join('2.5.2', 'test') -modregrtestdir = libpythondir.join('modified-2.5.2', 'test') +libpythondir = lib_pypy.LIB_PYTHON +regrtestdir = lib_pypy.LIB_PYTHON_VANILLA.join('test') +modregrtestdir = lib_pypy.LIB_PYTHON_MODIFIED.join('test') Modified: pypy/branch/x86-64-jit-backend/pypy/tool/release/force-builds.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/release/force-builds.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/release/force-builds.py Fri Jul 2 17:44:46 2010 @@ -19,13 +19,13 @@ BUILDERS = [ 'own-linux-x86-32', 'own-linux-x86-64', - 'own-macosx-x86-32', +# 'own-macosx-x86-32', 'pypy-c-app-level-linux-x86-32', 'pypy-c-app-level-linux-64', 'pypy-c-stackless-app-level-linux-x86-32', 'pypy-c-app-level-win-32', 'pypy-c-jit-linux-x86-32', - 'pypy-c-jit-macosx-x86-32', +# 'pypy-c-jit-macosx-x86-32', 'pypy-c-jit-win-x86-32', ] Modified: pypy/branch/x86-64-jit-backend/pypy/tool/release/make_release.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/release/make_release.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/release/make_release.py Fri Jul 2 17:44:46 2010 @@ -16,6 +16,8 @@ from pypy.tool.udir import udir from pypy.tool.release.package import package import tarfile +import os +import shutil BASEURL = 'http://buildbot.pypy.org/nightly/' @@ -47,28 +49,39 @@ to_download = browse_nightly(branch) tmpdir = udir.join('download') tmpdir.ensure(dir=True) - for (kind, platform), (rev, name) in to_download.iteritems(): - if platform == 'win32': - print 'Ignoring %s, windows unsupported' % name - else: - print "Downloading %s at rev %d" % (name, rev) - url = BASEURL + branch + "/" + name - data = urllib2.urlopen(url).read() - tmpdir.join(name).write(data, mode="wb") - t = tarfile.open(str(tmpdir.join(name))) - data = t.extractfile('pypy-c').read() - pypy_c = tmpdir.join('pypy-c') - pypy_c.write(data, mode="wb") - if kind == 'jit': - kind = '' + alltars = [] + try: + os.chdir(str(tmpdir)) + for (kind, platform), (rev, name) in to_download.iteritems(): + if platform == 'win32': + print 'Ignoring %s, windows unsupported' % name else: - kind = '-' + kind - name = 'pypy-%s%s-%s' % (release, kind, platform) - builddir = package(py.path.local(autopath.pypydir).join('..'), - name=name, - override_pypy_c=pypy_c) - print "Build %s/%s.tar.bz2" % (builddir, name) - print "\nLook into %s for packages" % builddir + print "Downloading %s at rev %d" % (name, rev) + url = BASEURL + branch + "/" + name + data = urllib2.urlopen(url).read() + tmpdir.join(name).write(data, mode="wb") + t = tarfile.open(str(tmpdir.join(name))) + dirname = t.getmembers()[0].name + t.extractall(path=str(tmpdir)) + os.system('mv %s %s' % (str(tmpdir.join(dirname)), + str(tmpdir.join('pypy-%s' % release)))) + if kind == 'jit': + kind = '' + else: + kind = '-' + kind + olddir = os.getcwd() + name = 'pypy-%s-%s%s.tar.bz2' % (release, platform, kind) + print "Building %s" % name + t = tarfile.open(name, 'w:bz2') + t.add('pypy-%s' % release) + alltars.append(name) + t.close() + shutil.rmtree(str(tmpdir.join('pypy-1.3'))) + for name in alltars: + print "Uploading %s" % name + os.system('scp %s codespeak.net:/www/pypy.org/htdocs/download' % name) + finally: + os.chdir(olddir) if __name__ == '__main__': if len(sys.argv) != 2: Modified: pypy/branch/x86-64-jit-backend/pypy/tool/release/package.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/release/package.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/release/package.py Fri Jul 2 17:44:46 2010 @@ -32,27 +32,37 @@ pass def package(basedir, name='pypy-nightly', rename_pypy_c='pypy-c', - override_pypy_c = None): + copy_to_dir = None, override_pypy_c = None): basedir = py.path.local(basedir) + if sys.platform == 'win32': + basename = 'pypy-c.exe' + else: + basename = 'pypy-c' if override_pypy_c is None: - pypy_c = basedir.join('pypy', 'translator', 'goal', 'pypy-c') + pypy_c = basedir.join('pypy', 'translator', 'goal', basename) else: pypy_c = py.path.local(override_pypy_c) if not pypy_c.check(): raise PyPyCNotFound('Please compile pypy first, using translate.py') builddir = udir.ensure("build", dir=True) pypydir = builddir.ensure(name, dir=True) + # Careful: to copy lib_pypy, copying just the svn-tracked files + # would not be enough: there are also ctypes_config_cache/_*_cache.py. shutil.copytree(str(basedir.join('lib-python')), str(pypydir.join('lib-python')), - ignore=ignore_patterns('.svn', '*.pyc', '*~')) - # Careful: to copy pypy/lib, copying just the svn-tracked files - # would not be enough: there are also ctypes_config_cache/_*_cache.py. - pypydir.ensure('pypy', dir=True) - shutil.copytree(str(basedir.join('pypy', 'lib')), - str(pypydir.join('pypy', 'lib')), + ignore=ignore_patterns('.svn', 'py', '*.pyc', '*~')) + shutil.copytree(str(basedir.join('lib_pypy')), + str(pypydir.join('lib_pypy')), ignore=ignore_patterns('.svn', 'py', '*.pyc', '*~')) for file in ['LICENSE', 'README']: shutil.copy(str(basedir.join(file)), str(pypydir)) + pypydir.ensure('include', dir=True) + # we want to put there all *.h and *.inl from trunk/include + # and from pypy/_interfaces + includedir = basedir.join('include') + headers = includedir.listdir('*.h') + includedir.listdir('*.inl') + for n in headers: + shutil.copy(str(n), str(pypydir.join('include'))) pypydir.ensure('bin', dir=True) archive_pypy_c = pypydir.join('bin', rename_pypy_c) shutil.copy(str(pypy_c), str(archive_pypy_c)) @@ -64,10 +74,13 @@ " " + name) finally: os.chdir(old_dir) + if copy_to_dir is not None: + print "Copying to %s" % copy_to_dir + shutil.copy(str(builddir.join(name + '.tar.bz2')), copy_to_dir) return builddir # for tests if __name__ == '__main__': - if len(sys.argv) == 1 or len(sys.argv) > 4: + if len(sys.argv) == 1: print >>sys.stderr, __doc__ sys.exit(1) else: Modified: pypy/branch/x86-64-jit-backend/pypy/tool/release/test/test_package.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/release/test/test_package.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/release/test/test_package.py Fri Jul 2 17:44:46 2010 @@ -2,6 +2,7 @@ import py from pypy.tool.autopath import pypydir from pypy.tool.release.package import package +from pypy.module.sys.version import CPYTHON_VERSION import tarfile, os def test_dir_structure(): @@ -14,15 +15,27 @@ fake_pypy_c = False try: builddir = package(py.path.local(pypydir).dirpath(), 'test') - assert builddir.join('test', 'lib-python', '2.5.2', 'test').check() - assert builddir.join('test', 'bin', 'pypy-c').check() - assert builddir.join('test', 'pypy', 'lib', 'syslog.py').check() - assert not builddir.join('test', 'pypy', 'lib', 'py').check() - assert not builddir.join('test', 'pypy', 'lib', 'ctypes_configure').check() - assert builddir.join('test', 'LICENSE').check() - assert builddir.join('test', 'README').check() + prefix = builddir.join('test') + cpyver = '%d.%d.%d' % CPYTHON_VERSION[:3] + assert prefix.join('lib-python', cpyver, 'test').check() + assert prefix.join('bin', 'pypy-c').check() + assert prefix.join('lib_pypy', 'syslog.py').check() + assert not prefix.join('lib_pypy', 'py').check() + assert not prefix.join('lib_pypy', 'ctypes_configure').check() + assert prefix.join('LICENSE').check() + assert prefix.join('README').check() th = tarfile.open(str(builddir.join('test.tar.bz2'))) - assert th.getmember('test/pypy/lib/syslog.py') + assert th.getmember('test/lib_pypy/syslog.py') + + # the headers file could be not there, because they are copied into + # trunk/include only during translation + includedir = py.path.local(pypydir).dirpath().join('include') + def check_include(name): + if includedir.join(name).check(file=True): + assert th.getmember('test/include/%s' % name) + check_include('Python.h') + check_include('modsupport.inl') + check_include('pypy_decl.h') finally: if fake_pypy_c: pypy_c.remove() Modified: pypy/branch/x86-64-jit-backend/pypy/tool/stdlib___future__.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/stdlib___future__.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/stdlib___future__.py Fri Jul 2 17:44:46 2010 @@ -1,8 +1,8 @@ # load __future__.py constants def load_module(): - import py - module_path = py.path.local(__file__).dirpath().dirpath().dirpath('lib-python/2.5.2/__future__.py') + from pypy.tool.lib_pypy import LIB_PYTHON_VANILLA + module_path = LIB_PYTHON_VANILLA.join('__future__.py') execfile(str(module_path), globals()) load_module() Modified: pypy/branch/x86-64-jit-backend/pypy/tool/stdlib_opcode.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/stdlib_opcode.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/stdlib_opcode.py Fri Jul 2 17:44:46 2010 @@ -106,8 +106,8 @@ opmap as host_opmap, HAVE_ARGUMENT as host_HAVE_ARGUMENT) def load_pypy_opcode(): - import py - opcode_path = py.path.local(__file__).dirpath().dirpath().dirpath('lib-python/modified-2.5.2/opcode.py') + from pypy.tool.lib_pypy import LIB_PYTHON_MODIFIED + opcode_path = LIB_PYTHON_MODIFIED.join('opcode.py') d = {} execfile(str(opcode_path), d) for name in __all__: Modified: pypy/branch/x86-64-jit-backend/pypy/tool/test/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/test/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/test/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/tool/test/test_killsubprocess.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/test/test_killsubprocess.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/test/test_killsubprocess.py Fri Jul 2 17:44:46 2010 @@ -1,5 +1,5 @@ import sys, time -from py.compat import subprocess +import subprocess from pypy.tool.killsubprocess import killsubprocess def waitdead(process): Modified: pypy/branch/x86-64-jit-backend/pypy/translator/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/backendopt/test/test_mallocprediction.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/backendopt/test/test_mallocprediction.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/backendopt/test/test_mallocprediction.py Fri Jul 2 17:44:46 2010 @@ -169,7 +169,7 @@ t, graph = rtype(entrypoint, [int]) total0 = preparation(t, t.graphs, heuristic=heuristic) total = clever_inlining_and_malloc_removal(t) - assert total0 + total == 10 + assert total == 5 # XXX total0 appears to vary def test_richards(): from pypy.translator.goal.richards import entry_point Modified: pypy/branch/x86-64-jit-backend/pypy/translator/benchmark/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/benchmark/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/benchmark/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/c/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/c/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/c/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/c/database.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/c/database.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/c/database.py Fri Jul 2 17:44:46 2010 @@ -16,7 +16,7 @@ from pypy.translator.c.extfunc import do_the_getting from pypy import conftest from pypy.translator.c import gc -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict class NoCorrespondingNode(Exception): Modified: pypy/branch/x86-64-jit-backend/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/c/funcgen.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/c/funcgen.py Fri Jul 2 17:44:46 2010 @@ -13,7 +13,7 @@ from pypy.rpython.lltypesystem.llmemory import Address from pypy.translator.backendopt.ssa import SSI_to_SSA from pypy.translator.backendopt.innerloop import find_inner_loops -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict PyObjPtr = Ptr(PyObject) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/c/gcc/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/c/gcc/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/c/gcc/autopath.py Fri Jul 2 17:44:46 2010 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -127,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/c/gcc/trackgcroot.py Fri Jul 2 17:44:46 2010 @@ -17,6 +17,7 @@ class FunctionGcRootTracker(object): skip = 0 + COMMENT = "([#;].*)?" @classmethod def init_regexp(cls): @@ -25,10 +26,10 @@ cls.r_globllabel = re.compile(cls.LABEL+r"=[.][+]%d\s*$"%cls.OFFSET_LABELS) cls.r_insn = re.compile(r"\t([a-z]\w*)\s") - cls.r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+cls.OPERAND+")\s*$") + cls.r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+cls.OPERAND+")\s*" + cls.COMMENT + "$") cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+"),\s*(?P"+cls.OPERAND+")\s*$") - cls.r_jump = re.compile(r"\tj\w+\s+"+cls.LABEL+"\s*$") + cls.r_jump = re.compile(r"\tj\w+\s+"+cls.LABEL+"\s*" + cls.COMMENT + "$") cls.r_jmp_switch = re.compile(r"\tjmp\t[*]"+cls.LABEL+"[(]") cls.r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") @@ -616,7 +617,7 @@ # tail-calls are equivalent to RET for us return InsnRet(self.CALLEE_SAVE_REGISTERS) return InsnStop() - + def register_jump_to(self, label): if not isinstance(self.insns[-1], InsnStop): self.labels[label].previous_insns.append(self.insns[-1]) @@ -641,6 +642,7 @@ self.register_jump_to(label) return [] + visit_jmpl = visit_jmp visit_je = conditional_jump visit_jne = conditional_jump visit_jg = conditional_jump Modified: pypy/branch/x86-64-jit-backend/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/c/genc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/c/genc.py Fri Jul 2 17:44:46 2010 @@ -33,7 +33,7 @@ def first(self): platform = self.compiler.platform - if platform.name == 'darwin': + if platform.name.startswith('darwin'): # XXX incredible hack for darwin cfiles = self.compiler.cfiles STR = '/*--no-profiling-for-this-file!--*/' Modified: pypy/branch/x86-64-jit-backend/pypy/translator/c/src/support.h ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/c/src/support.h (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/c/src/support.h Fri Jul 2 17:44:46 2010 @@ -26,7 +26,7 @@ PyString_FromStringAndSize(_RPyString_AsString(rpystr), RPyString_Size(rpystr)) #define PyUnicode_FromRPyUnicode(rpystr) \ - PyUnicode_FromUnicode(_RPyUnicode_AsUnicode(rpystr), RPyUnicode_Size(rpystr)) + _PyUnicode_FromRPyUnicode(_RPyUnicode_AsUnicode(rpystr), RPyUnicode_Size(rpystr)) #define PyString_ToRPyString(s, rpystr) \ memcpy(_RPyString_AsString(rpystr), PyString_AS_STRING(s), \ @@ -128,6 +128,7 @@ PyObject *PyTuple_GetItem_WithIncref(PyObject *tuple, int index); int PyTuple_SetItem_WithIncref(PyObject *tuple, int index, PyObject *o); int PySequence_Contains_with_exc(PyObject *seq, PyObject *ob); +PyObject* _PyUnicode_FromRPyUnicode(wchar_t *items, long length); /* implementations */ @@ -501,6 +502,17 @@ return ret; } +PyObject* _PyUnicode_FromRPyUnicode(wchar_t *items, long length) +{ + PyObject *u = PyUnicode_FromUnicode(NULL, length); + long i; + for (i=0; i> sys.stderr, ('debug: WARNING: library path not found, ' + 'using compiled-in sys.path') + newpath = sys.path[:] + break + newpath = sys.pypy_initial_path(dirname) + if newpath is None: + search = dirname # walk to the parent directory + continue + break # found! + return newpath def setup_initial_paths(executable, nanos): # a substituted os if we are translated global os os = nanos - AUTOSUBPATH = 'share' + os.sep + 'pypy-%d.%d' # find the full path to the executable, assuming that if there is no '/' # in the provided one then we must look along the $PATH if we_are_translated() and IS_WINDOWS and not executable.lower().endswith('.exe'): @@ -204,24 +219,7 @@ break sys.executable = os.path.abspath(executable) - # set up a sys.path that depends on the local machine - autosubpath = AUTOSUBPATH % sys.pypy_version_info[:2] - search = executable - while 1: - dirname = resolvedirof(search) - if dirname == search: - # not found! let's hope that the compiled-in path is ok - print >> sys.stderr, ('debug: WARNING: library path not found, ' - 'using compiled-in sys.path') - newpath = sys.path[:] - break - newpath = sys.pypy_initial_path(dirname) - if newpath is None: - newpath = sys.pypy_initial_path(os.path.join(dirname, autosubpath)) - if newpath is None: - search = dirname # walk to the parent directory - continue - break # found! + newpath = get_library_path(executable) path = os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath Modified: pypy/branch/x86-64-jit-backend/pypy/translator/goal/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/goal/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/goal/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/goal/targetpypystandalone.py Fri Jul 2 17:44:46 2010 @@ -225,7 +225,8 @@ return PyPyJitPolicy() def get_entry_point(self, config): - from pypy.lib.ctypes_config_cache import rebuild + from pypy.tool.lib_pypy import import_from_lib_pypy + rebuild = import_from_lib_pypy('ctypes_config_cache/rebuild') rebuild.try_rebuild() space = make_objspace(config) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/goal/test2/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/goal/test2/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/goal/test2/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/goal/test2/test_app_main.py Fri Jul 2 17:44:46 2010 @@ -8,19 +8,6 @@ banner = sys.version.splitlines()[0] -def relpath(path): - # force 'path' to be a relative path, for testing purposes - curdir = py.path.local() - p = py.path.local(path) - result = [] - while not p.relto(curdir): - result.append(os.pardir) - if curdir == curdir.dirpath(): - return str(path) # no relative path found, give up - curdir = curdir.dirpath() - result.append(p.relto(curdir)) - return os.path.join(*result) - app_main = os.path.join(autopath.this_dir, os.pardir, 'app_main.py') app_main = os.path.abspath(app_main) @@ -30,7 +17,8 @@ p = udir.join('demo_test_app_main_%d.py' % (_counter,)) _counter += 1 p.write(str(py.code.Source(source))) - return relpath(p) + # return relative path for testing purposes + return py.path.local().bestrelpath(p) demo_script = getscript(""" @@ -460,3 +448,57 @@ data = child_out_err.read(11) assert data == '\x00(STDOUT)\n\x00' # from stdout child_out_err.close() + + +class AppTestAppMain: + + def setup_class(self): + # ------------------------------------ + # setup code for test_get_library_path + # ------------------------------------ + from pypy.module.sys.version import CPYTHON_VERSION, PYPY_VERSION + cpy_ver = '%d.%d.%d' % CPYTHON_VERSION[:3] + + goal_dir = os.path.dirname(app_main) + # build a directory hierarchy like which contains both bin/pypy-c and + # lib/pypy1.2/* + prefix = udir.join('pathtest').ensure(dir=1) + fake_exe = prefix.join('bin/pypy-c').ensure(file=1) + expected_path = [str(prefix.join(subdir).ensure(dir=1)) + for subdir in ('lib_pypy', + 'lib-python/modified-%s' % cpy_ver, + 'lib-python/%s' % cpy_ver)] + + self.w_goal_dir = self.space.wrap(goal_dir) + self.w_fake_exe = self.space.wrap(str(fake_exe)) + self.w_expected_path = self.space.wrap(expected_path) + self.w_trunkdir = self.space.wrap(os.path.dirname(autopath.pypydir)) + + def test_get_library_path(self): + import sys + import os + sys.path.append(self.goal_dir) + try: + import app_main + app_main.os = os + newpath = app_main.get_library_path('/tmp/pypy-c') # stdlib not found + assert newpath == sys.path + newpath = app_main.get_library_path(self.fake_exe) + assert newpath == self.expected_path + finally: + sys.path.pop() + + def test_trunk_can_be_prefix(self): + import sys + import os + sys.path.append(self.goal_dir) + try: + import app_main + app_main.os = os + pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c') + newpath = app_main.get_library_path(pypy_c) + assert len(newpath) == 3 + for p in newpath: + assert p.startswith(self.trunkdir) + finally: + sys.path.pop() Modified: pypy/branch/x86-64-jit-backend/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/jvm/genjvm.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/jvm/genjvm.py Fri Jul 2 17:44:46 2010 @@ -6,7 +6,7 @@ import os import py -from py.compat import subprocess +import subprocess from pypy.tool.udir import udir from pypy.translator.translator import TranslationContext from pypy.translator.oosupport.genoo import GenOO Modified: pypy/branch/x86-64-jit-backend/pypy/translator/jvm/test/runtest.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/jvm/test/runtest.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/jvm/test/runtest.py Fri Jul 2 17:44:46 2010 @@ -2,7 +2,7 @@ import platform import py -from py.compat import subprocess +import subprocess from pypy.tool.udir import udir from pypy.rpython.test.tool import BaseRtypingTest, OORtypeMixin from pypy.rpython.lltypesystem.lltype import typeOf Modified: pypy/branch/x86-64-jit-backend/pypy/translator/microbench/pybench/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/microbench/pybench/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/microbench/pybench/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/oosupport/function.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/oosupport/function.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/oosupport/function.py Fri Jul 2 17:44:46 2010 @@ -7,7 +7,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.translator.oosupport.treebuilder import SubOperation from pypy.translator.oosupport.metavm import InstructionList, StoreResult -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict def render_sub_op(sub_op, db, generator): Modified: pypy/branch/x86-64-jit-backend/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/platform/windows.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/platform/windows.py Fri Jul 2 17:44:46 2010 @@ -168,7 +168,7 @@ # Tell the linker to generate a manifest file temp_manifest = ofile.dirpath().join( ofile.purebasename + '.manifest') - args += ["/MANIFESTFILE:%s" % (temp_manifest,)] + args += ["/MANIFEST", "/MANIFESTFILE:%s" % (temp_manifest,)] self._execute_c_compiler(self.link, args, exe_name) @@ -277,7 +277,7 @@ '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS)') else: m.rule('$(TARGET)', '$(OBJECTS)', - ['$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFESTFILE:$*.manifest', + ['$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST /MANIFESTFILE:$*.manifest', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) @@ -290,7 +290,7 @@ 'int main(int argc, char* argv[]) ' '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); } > $@') m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.obj'], - ['$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFESTFILE:$*.manifest', + ['$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFEST /MANIFESTFILE:$*.manifest', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/pypy_interact.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/pypy_interact.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/pypy_interact.py Fri Jul 2 17:44:46 2010 @@ -26,6 +26,7 @@ from pypy.translator.sandbox.sandlib import SimpleIOSandboxedProc from pypy.translator.sandbox.sandlib import VirtualizedSandboxedProc from pypy.translator.sandbox.vfs import Dir, RealDir, RealFile +from pypy.tool.lib_pypy import LIB_ROOT class PyPySandboxedProc(VirtualizedSandboxedProc, SimpleIOSandboxedProc): debug = True @@ -50,17 +51,15 @@ tmpdirnode = Dir({}) else: tmpdirnode = RealDir(self.tmpdir, exclude=exclude) - pypydist = os.path.dirname(os.path.abspath(autopath.pypydir)) + libroot = str(LIB_ROOT) return Dir({ 'bin': Dir({ 'pypy-c': RealFile(self.executable), - 'lib-python': RealDir(os.path.join(pypydist, 'lib-python'), + 'lib-python': RealDir(os.path.join(libroot, 'lib-python'), + exclude=exclude), + 'lib_pypy': RealDir(os.path.join(libroot, 'lib_pypy'), exclude=exclude), - 'pypy': Dir({ - 'lib': RealDir(os.path.join(pypydist, 'pypy', 'lib'), - exclude=exclude), - }), }), 'tmp': tmpdirnode, }) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/sandlib.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/sandlib.py Fri Jul 2 17:44:46 2010 @@ -6,7 +6,6 @@ import py import sys, os, posixpath, errno, stat, time -from pypy.lib import marshal # see below from pypy.rpython.module.ll_os_stat import s_StatResult from pypy.tool.ansi_print import AnsiLog from pypy.rlib.rarithmetic import r_longlong @@ -26,12 +25,14 @@ py.log.setconsumer("sandlib", MyAnsiLog()) -# Note: we use pypy.lib.marshal instead of the built-in marshal +# Note: we use lib_pypy/marshal.py instead of the built-in marshal # for two reasons. The built-in module could be made to segfault # or be attackable in other ways by sending malicious input to # load(). Also, marshal.load(f) blocks with the GIL held when # f is a pipe with no data immediately avaialble, preventing the # _waiting_thread to run. +from pypy.tool.lib_pypy import import_from_lib_pypy +marshal = import_from_lib_pypy('marshal') def read_message(f, timeout=None): # warning: 'timeout' is not really reliable and should only be used Modified: pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/test/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/test/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/test/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/test/test_pypy_interact.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/test/test_pypy_interact.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/test/test_pypy_interact.py Fri Jul 2 17:44:46 2010 @@ -24,7 +24,7 @@ assert_(argv[0] == '/bin/pypy-c', "bad argv[0]") st = os.lstat('/bin/pypy-c') assert_(stat.S_ISREG(st.st_mode), "bad st_mode for /bin/pypy-c") - for dirname in ['/bin/lib-python/2.5.2', '/bin/pypy/lib']: + for dirname in ['/bin/lib-python/2.5.2', '/bin/lib_pypy']: st = os.stat(dirname) assert_(stat.S_ISDIR(st.st_mode), "bad st_mode for " + dirname) assert_(os.environ.get('PYTHONPATH') is None, "unexpected $PYTHONPATH") @@ -70,7 +70,7 @@ def setup_module(mod): t = Translation(mini_pypy_like_entry_point, backend='c', - standalone=True, sandbox=True) + standalone=True, sandbox=True) mod.executable = str(t.compile()) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/test/test_sandbox.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/test/test_sandbox.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/sandbox/test/test_sandbox.py Fri Jul 2 17:44:46 2010 @@ -145,7 +145,9 @@ g = pipe.stdin f = pipe.stdout expect(f, g, "ll_os.ll_os_getenv", ("PYPY_GENERATIONGC_NURSERY",), None) - expect(f, g, "ll_os.ll_os_open", ("/proc/cpuinfo", 0, 420), OSError(5232, "xyz")) + if sys.platform == 'linux2': # on Mac, uses another (sandboxsafe) approach + expect(f, g, "ll_os.ll_os_open", ("/proc/cpuinfo", 0, 420), + OSError(5232, "xyz")) g.close() tail = f.read() f.close() Modified: pypy/branch/x86-64-jit-backend/pypy/translator/test/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/test/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/test/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/tool/autopath.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/tool/autopath.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/tool/autopath.py Fri Jul 2 17:44:46 2010 @@ -126,7 +126,7 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') -import py +import py # note: py is imported only AFTER the path has been set libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) Modified: pypy/branch/x86-64-jit-backend/pypy/translator/tool/lltracker.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/tool/lltracker.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/tool/lltracker.py Fri Jul 2 17:44:46 2010 @@ -8,7 +8,7 @@ from pypy.rpython.memory.gcheader import header2obj from pypy.translator.tool.reftracker import BaseRefTrackerPage, MARKER from pypy.tool.uid import uid -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict class LLRefTrackerPage(BaseRefTrackerPage): From arigo at codespeak.net Fri Jul 2 18:17:25 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 18:17:25 +0200 (CEST) Subject: [pypy-svn] r75779 - pypy/branch/rsre2 Message-ID: <20100702161725.99EE9282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 18:17:24 2010 New Revision: 75779 Added: pypy/branch/rsre2/ - copied from r75778, pypy/trunk/ Log: A branch in which to rewrite rsre. From arigo at codespeak.net Fri Jul 2 18:20:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 18:20:52 +0200 (CEST) Subject: [pypy-svn] r75780 - pypy/branch/rsre2/pypy/rlib/rsre Message-ID: <20100702162052.B2841282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 18:20:51 2010 New Revision: 75780 Removed: pypy/branch/rsre2/pypy/rlib/rsre/ Log: Kill rsre. From arigo at codespeak.net Fri Jul 2 18:21:19 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 18:21:19 +0200 (CEST) Subject: [pypy-svn] r75781 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100702162119.DDEEF282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 18:21:18 2010 New Revision: 75781 Added: pypy/branch/rsre2/pypy/rlib/rsre/ - copied from r75674, user/arigo/hack/pypy-hack/rsre/ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py - copied, changed from r75765, user/arigo/hack/pypy-hack/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/rsre_char.py - copied unchanged from r75706, user/arigo/hack/pypy-hack/rsre/rsre_char.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py - copied, changed from r75765, user/arigo/hack/pypy-hack/rsre/test/test_match.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py - copied unchanged from r75762, user/arigo/hack/pypy-hack/rsre/test/test_search.py Log: Copy the current rsre from my working copy. Copied: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (from r75765, user/arigo/hack/pypy-hack/rsre/rsre.py) ============================================================================== --- user/arigo/hack/pypy-hack/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Fri Jul 2 18:21:18 2010 @@ -9,6 +9,8 @@ OPCODE_ASSERT_NOT = 5 OPCODE_AT = 6 OPCODE_BRANCH = 7 +#OPCODE_CALL = 8 +OPCODE_CATEGORY = 9 OPCODE_IN = 15 OPCODE_INFO = 17 OPCODE_JUMP = 18 @@ -121,6 +123,14 @@ ppos += ctx.pat(ppos) return False + elif op == OPCODE_CATEGORY: + # match at given category (a single char) + # + xxx #if (ptr >= end || !sre_category(pattern[0], ptr[0])) + # return 0; + #pattern++; + #ptr++; + elif op == OPCODE_INFO: # optimization info block # <0=skip> <1=flags> <2=min> ... Copied: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (from r75765, user/arigo/hack/pypy-hack/rsre/test/test_match.py) ============================================================================== --- user/arigo/hack/pypy-hack/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Fri Jul 2 18:21:18 2010 @@ -87,3 +87,10 @@ assert rsre.match(r, "abc") assert not rsre.match(r, "abcd") assert not rsre.match(r, "ab") + + def test_category(self): + r, _ = get_code(r"ab\dcd") + assert rsre.match(r, "ab0cd") + assert rsre.match(r, "ab9cd") + assert not rsre.match(r, "abXcd") + assert not rsre.match(r, "ab+cd") From jcreigh at codespeak.net Fri Jul 2 18:28:06 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 2 Jul 2010 18:28:06 +0200 (CEST) Subject: [pypy-svn] r75783 - pypy/branch/x86-64-jit-backend/pypy/lib Message-ID: <20100702162806.A8986282BEF@codespeak.net> Author: jcreigh Date: Fri Jul 2 18:28:05 2010 New Revision: 75783 Removed: pypy/branch/x86-64-jit-backend/pypy/lib/ Log: remove pypy/lib, which was not done by svn merge for some reason From arigo at codespeak.net Fri Jul 2 19:01:08 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 19:01:08 +0200 (CEST) Subject: [pypy-svn] r75784 - in pypy/trunk/pypy: annotation module/marshal module/termios objspace/std rpython rpython/module rpython/test Message-ID: <20100702170108.0D9BB282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 19:01:06 2010 New Revision: 75784 Modified: pypy/trunk/pypy/annotation/model.py pypy/trunk/pypy/annotation/unaryop.py pypy/trunk/pypy/module/marshal/interp_marshal.py pypy/trunk/pypy/module/termios/interp_termios.py pypy/trunk/pypy/objspace/std/stringobject.py pypy/trunk/pypy/rpython/module/ll_termios.py pypy/trunk/pypy/rpython/rstr.py pypy/trunk/pypy/rpython/test/test_rstr.py Log: Remove the logic to support 'ord(string)' in RPython, only keeping 'ord(char)'. The issue was that the implementation of the former was just 'ord(string[0])', succeeding with strange results if the string is actually of length != 1 at runtime. Fix a few places in pypy that use this. Modified: pypy/trunk/pypy/annotation/model.py ============================================================================== --- pypy/trunk/pypy/annotation/model.py (original) +++ pypy/trunk/pypy/annotation/model.py Fri Jul 2 19:01:06 2010 @@ -229,11 +229,15 @@ class SomeChar(SomeString): "Stands for an object known to be a string of length 1." + can_be_None = False + def __init__(self): # no 'can_be_None' argument here + pass class SomeUnicodeCodePoint(SomeUnicodeString): "Stands for an object known to be a unicode codepoint." - def can_be_none(self): - return False + can_be_None = False + def __init__(self): # no 'can_be_None' argument here + pass SomeString.basestringclass = SomeString SomeString.basecharclass = SomeChar Modified: pypy/trunk/pypy/annotation/unaryop.py ============================================================================== --- pypy/trunk/pypy/annotation/unaryop.py (original) +++ pypy/trunk/pypy/annotation/unaryop.py Fri Jul 2 19:01:06 2010 @@ -5,7 +5,7 @@ from types import MethodType from pypy.annotation.model import \ SomeObject, SomeInteger, SomeBool, SomeString, SomeChar, SomeList, \ - SomeDict, SomeTuple, SomeImpossibleValue, \ + SomeDict, SomeTuple, SomeImpossibleValue, SomeUnicodeCodePoint, \ SomeInstance, SomeBuiltin, SomeFloat, SomeIterator, SomePBC, \ SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \ s_ImpossibleValue, s_Bool, s_None, \ @@ -494,9 +494,6 @@ def getanyitem(str): return str.basecharclass() - def ord(str): - return SomeInteger(nonneg=True) - def method_split(str, patt): # XXX getbookkeeper().count("str_split", str, patt) return getbookkeeper().newlist(str.basestringclass()) @@ -541,11 +538,16 @@ return SomeUnicodeString() method_decode.can_only_throw = [UnicodeDecodeError] -class __extend__(SomeChar): +class __extend__(SomeChar, SomeUnicodeCodePoint): def len(chr): return immutablevalue(1) + def ord(str): + return SomeInteger(nonneg=True) + +class __extend__(SomeChar): + def method_isspace(chr): return s_Bool Modified: pypy/trunk/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/trunk/pypy/module/marshal/interp_marshal.py (original) +++ pypy/trunk/pypy/module/marshal/interp_marshal.py Fri Jul 2 19:01:06 2010 @@ -365,8 +365,8 @@ return self.reader.read(n) def get1(self): - # convince typer to use a char - return chr(ord(self.get(1))) + # the [0] is used to convince the annotator to return a char + return self.get(1)[0] def atom_str(self, typecode): self.start(typecode) Modified: pypy/trunk/pypy/module/termios/interp_termios.py ============================================================================== --- pypy/trunk/pypy/module/termios/interp_termios.py (original) +++ pypy/trunk/pypy/module/termios/interp_termios.py Fri Jul 2 19:01:06 2010 @@ -60,8 +60,8 @@ # last one need to be chosen carefully cc_w = [space.wrap(i) for i in cc] if lflag & termios.ICANON: - cc_w[termios.VMIN] = space.wrap(ord(cc[termios.VMIN])) - cc_w[termios.VTIME] = space.wrap(ord(cc[termios.VTIME])) + cc_w[termios.VMIN] = space.wrap(ord(cc[termios.VMIN][0])) + cc_w[termios.VTIME] = space.wrap(ord(cc[termios.VTIME][0])) w_cc = space.newlist(cc_w) l_w.append(w_cc) return space.newlist(l_w) Modified: pypy/trunk/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/stringobject.py (original) +++ pypy/trunk/pypy/objspace/std/stringobject.py Fri Jul 2 19:01:06 2010 @@ -863,7 +863,7 @@ space.w_TypeError, "ord() expected a character, but string " "of length %d found", len(u_str)) - return space.wrap(ord(u_str)) + return space.wrap(ord(u_str[0])) def getnewargs__String(space, w_str): return space.newtuple([wrapstr(space, w_str._value)]) Modified: pypy/trunk/pypy/rpython/module/ll_termios.py ============================================================================== --- pypy/trunk/pypy/rpython/module/ll_termios.py (original) +++ pypy/trunk/pypy/rpython/module/ll_termios.py Fri Jul 2 19:01:06 2010 @@ -85,7 +85,7 @@ c_struct.c_c_lflag, ispeed, ospeed, cc = attributes try: for i in range(NCCS): - c_struct.c_c_cc[i] = rffi.r_uchar(ord(cc[i])) + c_struct.c_c_cc[i] = rffi.r_uchar(ord(cc[i][0])) error = c_cfsetispeed(c_struct, ispeed) if error == -1: raise termios.error(error, 'tcsetattr failed') Modified: pypy/trunk/pypy/rpython/rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/rstr.py (original) +++ pypy/trunk/pypy/rpython/rstr.py Fri Jul 2 19:01:06 2010 @@ -80,17 +80,6 @@ # defaults to checking the length return super(AbstractStringRepr, self).rtype_is_true(hop) - def rtype_ord(self, hop): - string_repr = hop.args_r[0].repr - v_str, = hop.inputargs(string_repr) - c_zero = inputconst(Signed, 0) - v_chr = hop.gendirectcall(self.ll.ll_stritem_nonneg, v_str, c_zero) - if string_repr is hop.rtyper.type_system.rstr.string_repr: - return hop.genop('cast_char_to_int', [v_chr], resulttype=Signed) - else: - assert string_repr is hop.rtyper.type_system.rstr.unicode_repr - return hop.genop('cast_unichar_to_int', [v_chr], resulttype=Signed) - def rtype_method_startswith(self, hop): str1_repr, str2_repr = self._str_reprs(hop) v_str, v_value = hop.inputargs(str1_repr, str2_repr) Modified: pypy/trunk/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rstr.py (original) +++ pypy/trunk/pypy/rpython/test/test_rstr.py Fri Jul 2 19:01:06 2010 @@ -131,7 +131,7 @@ s = c * mul res = 0 for i in range(len(s)): - res = res*10 + ord(const(s[i])) - ord(const('0')) + res = res*10 + ord(const(s[i])[0]) - ord(const('0')[0]) c2 = c c2 *= mul res = 10 * res + (c2 == s) @@ -577,7 +577,7 @@ sum = 0 for num in l: if len(num): - sum += ord(num) - ord(const('0')) + sum += ord(num[0]) - ord(const('0')[0]) return sum + len(l) * 100 for i in range(5): res = self.interpret(fn, [i]) From arigo at codespeak.net Fri Jul 2 19:27:11 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 19:27:11 +0200 (CEST) Subject: [pypy-svn] r75785 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100702172711.9E3FC282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 19:27:10 2010 New Revision: 75785 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Log: Progress. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Fri Jul 2 19:27:10 2010 @@ -123,13 +123,17 @@ ppos += ctx.pat(ppos) return False - elif op == OPCODE_CATEGORY: - # match at given category (a single char) - # - xxx #if (ptr >= end || !sre_category(pattern[0], ptr[0])) - # return 0; - #pattern++; - #ptr++; + #elif op == OPCODE_CATEGORY: + # seems to be never produced + + elif op == OPCODE_IN: + # match set member (or non_member) + # + if (ptr >= ctx.end + or not check_charset(ctx.pattern, ppos+1, ctx.str(ptr))): + return False + ppos += ctx.pat(ppos) + ptr += 1 elif op == OPCODE_INFO: # optimization info block Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Fri Jul 2 19:27:10 2010 @@ -6,6 +6,7 @@ class GotIt(Exception): pass def my_compile(pattern, flags, code, *args): + print code raise GotIt(code) saved = _sre.compile try: @@ -14,6 +15,8 @@ re.compile(regexp) except GotIt, e: pass + else: + raise ValueError("did not reach _sre.compile()!") finally: _sre.compile = saved return e.args[0], re.compile(regexp) @@ -88,9 +91,13 @@ assert not rsre.match(r, "abcd") assert not rsre.match(r, "ab") + def test_repeated_set(self): + r, _ = get_code(r"[a0x]+f") + assert rsre.match(r, "a0af") + assert not rsre.match(r, "a0yaf") + def test_category(self): - r, _ = get_code(r"ab\dcd") - assert rsre.match(r, "ab0cd") - assert rsre.match(r, "ab9cd") - assert not rsre.match(r, "abXcd") - assert not rsre.match(r, "ab+cd") + r, _ = get_code(r"[\sx]") + assert rsre.match(r, "x") + assert rsre.match(r, " ") + assert not rsre.match(r, "n") From arigo at codespeak.net Fri Jul 2 19:40:01 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 19:40:01 +0200 (CEST) Subject: [pypy-svn] r75786 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100702174001.689A0282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 19:39:59 2010 New Revision: 75786 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Log: GROUPREF. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Fri Jul 2 19:39:59 2010 @@ -10,7 +10,10 @@ OPCODE_AT = 6 OPCODE_BRANCH = 7 #OPCODE_CALL = 8 -OPCODE_CATEGORY = 9 +#OPCODE_CATEGORY = 9 +#OPCODE_CHARSET = 10 +#OPCODE_BIGCHARSET = 11 +OPCODE_GROUPREF = 12 OPCODE_IN = 15 OPCODE_INFO = 17 OPCODE_JUMP = 18 @@ -42,12 +45,7 @@ def get_mark(self, gid): """Use this for testing.""" - mark = self.match_marks - while mark is not None: - if mark.gid == gid: - return mark.position - mark = mark.prev - return -1 + return find_mark(self.match_marks, gid) class Mark(object): @@ -58,6 +56,13 @@ self.position = position self.prev = prev # chained list +def find_mark(mark, gid): + while mark is not None: + if mark.gid == gid: + return mark.position + mark = mark.prev + return -1 + def match(pattern, string): ctx = MatchContext(pattern, string) @@ -126,6 +131,22 @@ #elif op == OPCODE_CATEGORY: # seems to be never produced + elif op == OPCODE_GROUPREF: + # match backreference + # + gid = ctx.pat(ppos) * 2 + startptr = find_mark(marks, gid) + if startptr < 0: + return False + endptr = find_mark(marks, gid + 1) + if endptr < startptr: # also includes the case "endptr == -1" + return False + for i in range(startptr, endptr): + if ptr >= ctx.end or ctx.str(ptr) != ctx.str(i): + return False + ptr += 1 + ppos += 1 + elif op == OPCODE_IN: # match set member (or non_member) # Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Fri Jul 2 19:39:59 2010 @@ -101,3 +101,14 @@ assert rsre.match(r, "x") assert rsre.match(r, " ") assert not rsre.match(r, "n") + + def test_groupref(self): + r, _ = get_code(r"(xx+)\1+$") # match non-prime numbers of x + assert not rsre.match(r, "xx") + assert not rsre.match(r, "xxx") + assert rsre.match(r, "xxxx") + assert not rsre.match(r, "xxxxx") + assert rsre.match(r, "xxxxxx") + assert not rsre.match(r, "xxxxxxx") + assert rsre.match(r, "xxxxxxxx") + assert rsre.match(r, "xxxxxxxxx") From arigo at codespeak.net Fri Jul 2 19:49:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 19:49:47 +0200 (CEST) Subject: [pypy-svn] r75787 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100702174947.ACD86282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 19:49:46 2010 New Revision: 75787 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Log: LITERAL_IGNORE, GROUPREF_IGNORE Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Fri Jul 2 19:49:46 2010 @@ -1,5 +1,5 @@ from pypy.rlib.debug import check_nonneg -from rsre_char import check_charset, is_linebreak, is_word +import rsre_char OPCODE_SUCCESS = 1 @@ -14,10 +14,13 @@ #OPCODE_CHARSET = 10 #OPCODE_BIGCHARSET = 11 OPCODE_GROUPREF = 12 +OPCODE_GROUPREF_EXISTS = 13 +OPCODE_GROUPREF_IGNORE = 14 OPCODE_IN = 15 OPCODE_INFO = 17 OPCODE_JUMP = 18 OPCODE_LITERAL = 19 +OPCODE_LITERAL_IGNORE = 20 OPCODE_MARK = 21 OPCODE_MAX_UNTIL = 22 OPCODE_MIN_UNTIL = 23 @@ -30,10 +33,11 @@ match_end = 0 match_marks = None - def __init__(self, pattern, string): + def __init__(self, pattern, string, flags): self.pattern = pattern self.string = string self.end = len(string) + self.flags = flags def pat(self, index): check_nonneg(index) @@ -43,6 +47,10 @@ check_nonneg(index) return ord(self.string[index]) + def lowstr(self, index): + c = self.str(index) + return rsre_char.getlower(c, self.flags) + def get_mark(self, gid): """Use this for testing.""" return find_mark(self.match_marks, gid) @@ -64,8 +72,8 @@ return -1 -def match(pattern, string): - ctx = MatchContext(pattern, string) +def match(pattern, string, flags=0): + ctx = MatchContext(pattern, string, flags) if sre_match(ctx, 0, 0, None): return ctx return None @@ -85,7 +93,7 @@ elif op == OPCODE_ANY: # match anything (except a newline) # - if ptr >= ctx.end or is_linebreak(ctx.str(ptr)): + if ptr >= ctx.end or rsre_char.is_linebreak(ctx.str(ptr)): return False ptr += 1 @@ -147,11 +155,28 @@ ptr += 1 ppos += 1 + elif op == OPCODE_GROUPREF_IGNORE: + # match backreference + # + gid = ctx.pat(ppos) * 2 + startptr = find_mark(marks, gid) + if startptr < 0: + return False + endptr = find_mark(marks, gid + 1) + if endptr < startptr: # also includes the case "endptr == -1" + return False + for i in range(startptr, endptr): + if ptr >= ctx.end or ctx.lowstr(ptr) != ctx.lowstr(i): + return False + ptr += 1 + ppos += 1 + elif op == OPCODE_IN: # match set member (or non_member) # - if (ptr >= ctx.end - or not check_charset(ctx.pattern, ppos+1, ctx.str(ptr))): + if ptr >= ctx.end or not rsre_char.check_charset(ctx.pattern, + ppos+1, + ctx.str(ptr)): return False ppos += ctx.pat(ppos) ptr += 1 @@ -174,6 +199,14 @@ ppos += 1 ptr += 1 + elif op == OPCODE_LITERAL_IGNORE: + # match literal string, ignoring case + # + if ptr >= ctx.end or ctx.lowstr(ptr) != ctx.pat(ppos): + return False + ppos += 1 + ptr += 1 + elif op == OPCODE_MARK: # set mark # @@ -321,7 +354,7 @@ if op == OPCODE_ANY: # repeated dot wildcard. - while ptr < end and not is_linebreak(ctx.str(ptr)): + while ptr < end and not rsre_char.is_linebreak(ctx.str(ptr)): ptr += 1 elif op == OPCODE_ANY_ALL: @@ -331,7 +364,8 @@ elif op == OPCODE_IN: # repeated set - while ptr < end and check_charset(ctx.pattern, ppos+2, ctx.str(ptr)): + while ptr < end and rsre_char.check_charset(ctx.pattern, ppos+2, + ctx.str(ptr)): ptr += 1 elif op == OPCODE_LITERAL: @@ -339,6 +373,11 @@ while ptr < end and ctx.str(ptr) == chr: ptr += 1 + elif op == OPCODE_LITERAL_IGNORE: + chr = ctx.pat(ppos+1) + while ptr < end and ctx.lowstr(ptr) == chr: + ptr += 1 + else: assert 0, "XXX %d" % op @@ -367,7 +406,7 @@ elif atcode == AT_BEGINNING_LINE: prevptr = ptr - 1 - return prevptr < 0 or is_linebreak(ctx.str(prevptr)) + return prevptr < 0 or rsre_char.is_linebreak(ctx.str(prevptr)) elif atcode == AT_BOUNDARY: if ctx.end == 0: @@ -388,10 +427,10 @@ elif atcode == AT_END: remaining_chars = ctx.end - ptr return remaining_chars <= 0 or ( - remaining_chars == 1 and is_linebreak(ctx.str(ptr))) + remaining_chars == 1 and rsre_char.is_linebreak(ctx.str(ptr))) elif atcode == AT_END_LINE: - return ptr == ctx.end or is_linebreak(ctx.str(ptr)) + return ptr == ctx.end or rsre_char.is_linebreak(ctx.str(ptr)) elif atcode == AT_END_STRING: return ptr == ctx.end Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Fri Jul 2 19:49:46 2010 @@ -112,3 +112,14 @@ assert not rsre.match(r, "xxxxxxx") assert rsre.match(r, "xxxxxxxx") assert rsre.match(r, "xxxxxxxxx") + + def test_groupref_ignore(self): + r, _ = get_code(r"(?i)(xx+)\1+$") # match non-prime numbers of x + assert not rsre.match(r, "xX") + assert not rsre.match(r, "xxX") + assert rsre.match(r, "Xxxx") + assert not rsre.match(r, "xxxXx") + assert rsre.match(r, "xXxxxx") + assert not rsre.match(r, "xxxXxxx") + assert rsre.match(r, "xxxxxxXx") + assert rsre.match(r, "xxxXxxxxx") From arigo at codespeak.net Fri Jul 2 19:52:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 19:52:45 +0200 (CEST) Subject: [pypy-svn] r75788 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100702175245.E943E282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 19:52:44 2010 New Revision: 75788 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Log: IN_IGNORE Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Fri Jul 2 19:52:44 2010 @@ -17,6 +17,7 @@ OPCODE_GROUPREF_EXISTS = 13 OPCODE_GROUPREF_IGNORE = 14 OPCODE_IN = 15 +OPCODE_IN_IGNORE = 16 OPCODE_INFO = 17 OPCODE_JUMP = 18 OPCODE_LITERAL = 19 @@ -181,6 +182,16 @@ ppos += ctx.pat(ppos) ptr += 1 + elif op == OPCODE_IN_IGNORE: + # match set member (or non_member), ignoring case + # + if ptr >= ctx.end or not rsre_char.check_charset(ctx.pattern, + ppos+1, + ctx.lowstr(ptr)): + return False + ppos += ctx.pat(ppos) + ptr += 1 + elif op == OPCODE_INFO: # optimization info block # <0=skip> <1=flags> <2=min> ... @@ -368,6 +379,12 @@ ctx.str(ptr)): ptr += 1 + elif op == OPCODE_IN_IGNORE: + # repeated set + while ptr < end and rsre_char.check_charset(ctx.pattern, ppos+2, + ctx.lowstr(ptr)): + ptr += 1 + elif op == OPCODE_LITERAL: chr = ctx.pat(ppos+1) while ptr < end and ctx.str(ptr) == chr: Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Fri Jul 2 19:52:44 2010 @@ -123,3 +123,13 @@ assert not rsre.match(r, "xxxXxxx") assert rsre.match(r, "xxxxxxXx") assert rsre.match(r, "xxxXxxxxx") + + def test_in_ignore(self): + r, _ = get_code(r"(?i)[a-f]") + assert rsre.match(r, "b") + assert rsre.match(r, "C") + assert not rsre.match(r, "g") + r, _ = get_code(r"(?i)[a-f]+$") + assert rsre.match(r, "bCdEf") + assert not rsre.match(r, "g") + assert not rsre.match(r, "aaagaaa") From arigo at codespeak.net Fri Jul 2 19:57:33 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 19:57:33 +0200 (CEST) Subject: [pypy-svn] r75789 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100702175733.666CE282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 19:57:31 2010 New Revision: 75789 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Log: NOT_LITERAL. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Fri Jul 2 19:57:31 2010 @@ -25,6 +25,8 @@ OPCODE_MARK = 21 OPCODE_MAX_UNTIL = 22 OPCODE_MIN_UNTIL = 23 +OPCODE_NOT_LITERAL = 24 +OPCODE_NOT_LITERAL_IGNORE = 25 OPCODE_REPEAT = 28 OPCODE_REPEAT_ONE = 29 OPCODE_MIN_REPEAT_ONE = 31 @@ -225,6 +227,14 @@ marks = Mark(gid, ptr, marks) ppos += 1 + elif op == OPCODE_NOT_LITERAL: + # match if it's not a literal string + # + if ptr >= ctx.end or ctx.lowstr(ptr) == ctx.pat(ppos): + return False + ppos += 1 + ptr += 1 + elif op == OPCODE_REPEAT: # general repeat. in this version of the re module, all the work # is done here, and not on the later UNTIL operator. @@ -395,6 +405,11 @@ while ptr < end and ctx.lowstr(ptr) == chr: ptr += 1 + elif op == OPCODE_NOT_LITERAL: + chr = ctx.pat(ppos+1) + while ptr < end and ctx.str(ptr) != chr: + ptr += 1 + else: assert 0, "XXX %d" % op Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Fri Jul 2 19:57:31 2010 @@ -133,3 +133,11 @@ assert rsre.match(r, "bCdEf") assert not rsre.match(r, "g") assert not rsre.match(r, "aaagaaa") + + def test_not_literal(self): + r, _ = get_code(r"[^a-f]") + assert rsre.match(r, "B") + assert not rsre.match(r, "b") + r, _ = get_code(r"[^a-f]+$") + assert rsre.match(r, "Bx123") + assert not rsre.match(r, "--f--") From arigo at codespeak.net Fri Jul 2 20:02:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 20:02:45 +0200 (CEST) Subject: [pypy-svn] r75790 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100702180245.AB2A2282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 20:02:43 2010 New Revision: 75790 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Log: Oups, by fault for not seeing the test fail. I did not implement NOT_LITERAL correctly. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Fri Jul 2 20:02:43 2010 @@ -230,7 +230,7 @@ elif op == OPCODE_NOT_LITERAL: # match if it's not a literal string # - if ptr >= ctx.end or ctx.lowstr(ptr) == ctx.pat(ppos): + if ptr >= ctx.end or ctx.str(ptr) == ctx.pat(ppos): return False ppos += 1 ptr += 1 Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Fri Jul 2 20:02:43 2010 @@ -135,9 +135,19 @@ assert not rsre.match(r, "aaagaaa") def test_not_literal(self): - r, _ = get_code(r"[^a-f]") - assert rsre.match(r, "B") - assert not rsre.match(r, "b") - r, _ = get_code(r"[^a-f]+$") + r, _ = get_code(r"[^a]") + assert rsre.match(r, "A") + assert not rsre.match(r, "a") + r, _ = get_code(r"[^a]+$") assert rsre.match(r, "Bx123") - assert not rsre.match(r, "--f--") + assert not rsre.match(r, "--a--") + + def test_not_literal_ignore(self): + import py; py.test.skip("in-progress") + r, _ = get_code(r"(?i)[^a]") + assert rsre.match(r, "G") + assert not rsre.match(r, "a") + assert not rsre.match(r, "A") + r, _ = get_code(r"(?i)[^a]+$") + assert rsre.match(r, "Gx123") + assert not rsre.match(r, "--A--") From arigo at codespeak.net Fri Jul 2 20:03:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 20:03:45 +0200 (CEST) Subject: [pypy-svn] r75791 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100702180345.3BF20282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 20:03:43 2010 New Revision: 75791 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Log: NOT_LITERAL_IGNORE. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Fri Jul 2 20:03:43 2010 @@ -235,6 +235,14 @@ ppos += 1 ptr += 1 + elif op == OPCODE_NOT_LITERAL_IGNORE: + # match if it's not a literal string, ignoring case + # + if ptr >= ctx.end or ctx.lowstr(ptr) == ctx.pat(ppos): + return False + ppos += 1 + ptr += 1 + elif op == OPCODE_REPEAT: # general repeat. in this version of the re module, all the work # is done here, and not on the later UNTIL operator. @@ -410,6 +418,11 @@ while ptr < end and ctx.str(ptr) != chr: ptr += 1 + elif op == OPCODE_NOT_LITERAL_IGNORE: + chr = ctx.pat(ppos+1) + while ptr < end and ctx.lowstr(ptr) != chr: + ptr += 1 + else: assert 0, "XXX %d" % op Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Fri Jul 2 20:03:43 2010 @@ -143,7 +143,6 @@ assert not rsre.match(r, "--a--") def test_not_literal_ignore(self): - import py; py.test.skip("in-progress") r, _ = get_code(r"(?i)[^a]") assert rsre.match(r, "G") assert not rsre.match(r, "a") From arigo at codespeak.net Fri Jul 2 20:06:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 20:06:37 +0200 (CEST) Subject: [pypy-svn] r75792 - pypy/branch/rsre2/pypy/rlib/rsre Message-ID: <20100702180637.90507282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 20:06:35 2010 New Revision: 75792 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Log: A few remaining opcodes, not used here. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Fri Jul 2 20:06:35 2010 @@ -2,6 +2,7 @@ import rsre_char +OPCODE_FAILURE = 0 OPCODE_SUCCESS = 1 OPCODE_ANY = 2 OPCODE_ANY_ALL = 3 @@ -27,8 +28,11 @@ OPCODE_MIN_UNTIL = 23 OPCODE_NOT_LITERAL = 24 OPCODE_NOT_LITERAL_IGNORE = 25 +#OPCODE_NEGATE = 26 +#OPCODE_RANGE = 27 OPCODE_REPEAT = 28 OPCODE_REPEAT_ONE = 29 +#OPCODE_SUBPATTERN = 30 OPCODE_MIN_REPEAT_ONE = 31 @@ -86,6 +90,9 @@ op = ctx.pat(ppos) ppos += 1 + if op == OPCODE_FAILURE: + return False + if (op == OPCODE_SUCCESS or op == OPCODE_MAX_UNTIL or op == OPCODE_MIN_UNTIL): From jcreigh at codespeak.net Fri Jul 2 20:08:33 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 2 Jul 2010 20:08:33 +0200 (CEST) Subject: [pypy-svn] r75793 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/test Message-ID: <20100702180833.E41B0282BEF@codespeak.net> Author: jcreigh Date: Fri Jul 2 20:08:31 2010 New Revision: 75793 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py Log: increase number of arguments used in test_assembler_call to better cover x86-64 calling convention Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py Fri Jul 2 20:08:31 2010 @@ -1714,7 +1714,7 @@ def test_assembler_call(self): called = [] def assembler_helper(failindex, virtualizable): - assert self.cpu.get_latest_value_int(0) == 10 + assert self.cpu.get_latest_value_int(0) == 97 called.append(failindex) return 4 + 9 @@ -1727,33 +1727,41 @@ _assembler_helper_ptr) ops = ''' - [i0, i1] - i2 = int_add(i0, i1) - finish(i2)''' + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, i1) + i11 = int_add(i10, i2) + i12 = int_add(i11, i3) + i13 = int_add(i12, i4) + i14 = int_add(i13, i5) + i15 = int_add(i14, i6) + i16 = int_add(i15, i7) + i17 = int_add(i16, i8) + i18 = int_add(i17, i9) + finish(i18)''' loop = parse(ops) looptoken = LoopToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) - ARGS = [lltype.Signed, lltype.Signed] + ARGS = [lltype.Signed] * 10 RES = lltype.Signed self.cpu.portal_calldescr = self.cpu.calldescrof( lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES) - self.cpu.set_future_value_int(0, 1) - self.cpu.set_future_value_int(1, 2) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_int(0) == 3 + assert self.cpu.get_latest_value_int(0) == 55 ops = ''' - [i4, i5] - i6 = int_add(i4, 1) - i3 = call_assembler(i6, i5, descr=looptoken) + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, 42) + i11 = call_assembler(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, descr=looptoken) guard_not_forced()[] - finish(i3) + finish(i11) ''' loop = parse(ops, namespace=locals()) othertoken = LoopToken() self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken) - self.cpu.set_future_value_int(0, 4) - self.cpu.set_future_value_int(1, 5) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(othertoken) assert self.cpu.get_latest_value_int(0) == 13 assert called From jcreigh at codespeak.net Fri Jul 2 20:33:29 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 2 Jul 2010 20:33:29 +0200 (CEST) Subject: [pypy-svn] r75794 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100702183329.78B17282BEF@codespeak.net> Author: jcreigh Date: Fri Jul 2 20:33:26 2010 New Revision: 75794 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Log: try to fix 64-bit call_assembler Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Fri Jul 2 20:33:26 2010 @@ -444,8 +444,13 @@ return adr_stackadjust def _assemble_bootstrap_direct_call_64(self, arglocs, jmpadr, stackdepth): - # XXX: Is this even remotely correct? Might arglocs contain some of - # the same locations that the calling convention uses? + # XXX: Very similar to _emit_call_64 + + src_locs = [] + dst_locs = [] + xmm_src_locs = [] + xmm_dst_locs = [] + get_from_stack = [] # In reverse order for use with pop() unused_gpr = [r9, r8, ecx, edx, esi, edi] @@ -455,30 +460,42 @@ adr_stackadjust = self._call_header() self._patch_stackadjust(adr_stackadjust, stackdepth) + # The lists are padded with Nones + assert len(nonfloatlocs) == len(floatlocs) + for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if loc is not None: if len(unused_gpr) > 0: - self.mc.MOV(loc, unused_gpr.pop()) + src_locs.append(unused_gpr.pop()) + dst_locs.append(loc) else: - self.mc.ensure_bytes_available(32) - self.mc.MOV_rb(X86_64_SCRATCH_REG.value, (2 + i) * WORD) - self.mc.MOV(loc, X86_64_SCRATCH_REG) + get_from_stack.append((loc, False)) - for i in range(len(floatlocs)): - loc = floatlocs[i] - if loc is not None: + floc = floatlocs[i] + if floc is not None: if len(unused_xmm) > 0: - self.mc.MOVSD(loc, unused_xmm.pop()) + xmm_src_locs.append(unused_xmm.pop()) + xmm_dst_locs.append(floc) else: - self.mc.MOVSD_xb(X86_64_XMM_SCRATCH_REG.value, (2 + i) * WORD) - self.mc.MOVSD(loc, X86_64_XMM_SCRATCH_REG) + get_from_stack.append((floc, True)) - self.mc.JMP(imm(jmpadr)) + remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) - return adr_stackadjust + for i in range(len(get_from_stack)): + loc, is_xmm = get_from_stack[i] + self.mc.ensure_bytes_available(32) + if is_xmm: + self.mc.MOVSD_xb(X86_64_XMM_SCRATCH_REG.value, (2 + i) * WORD) + self.mc.MOVSD(loc, X86_64_XMM_SCRATCH_REG) + else: + self.mc.MOV_rb(X86_64_SCRATCH_REG.value, (2 + i) * WORD) + self.mc.MOV(loc, X86_64_SCRATCH_REG) + self.mc.JMP(imm(jmpadr)) + return adr_stackadjust def _assemble_bootstrap_code(self, inputargs, arglocs): nonfloatlocs, floatlocs = arglocs From benjamin at codespeak.net Fri Jul 2 21:37:04 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 2 Jul 2010 21:37:04 +0200 (CEST) Subject: [pypy-svn] r75796 - in pypy/branch/fast-forward/pypy/interpreter: astcompiler test Message-ID: <20100702193704.25FA6282BEF@codespeak.net> Author: benjamin Date: Fri Jul 2 21:37:01 2010 New Revision: 75796 Modified: pypy/branch/fast-forward/pypy/interpreter/astcompiler/assemble.py pypy/branch/fast-forward/pypy/interpreter/test/test_compiler.py Log: don't mix -0.0 and 0.0 in complex or float cases Modified: pypy/branch/fast-forward/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/astcompiler/assemble.py Fri Jul 2 21:37:01 2010 @@ -8,6 +8,7 @@ from pypy.interpreter.error import OperationError from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib import rarithmetic class Instruction(object): @@ -209,7 +210,35 @@ # To avoid confusing equal but separate types, we hash store the type of # the constant in the dictionary. if w_key is None: - w_key = space.newtuple([obj, space.type(obj)]) + # We have to keep the difference between -0.0 and 0.0 floats. + w_type = space.type(obj) + if space.is_w(w_type, space.w_float): + val = space.float_w(obj) + if val == 0.0 and rarithmetic.copysign(1., val) < 0: + w_key = space.newtuple([obj, space.w_float, space.w_None]) + else: + w_key = space.newtuple([obj, space.w_float]) + elif space.is_w(w_type, space.w_complex): + w_real = space.getattr(obj, space.wrap("real")) + w_imag = space.getattr(obj, space.wrap("imag")) + real = space.float_w(w_real) + imag = space.float_w(w_imag) + real_negzero = (real == 0.0 and + rarithmetic.copysign(1., real) < 0) + imag_negzero = (imag == 0.0 and + rarithmetic.copysign(1., imag) < 0) + if real_negzero and imag_negzero: + tup = [obj, space.w_complex, space.w_None, space.w_None, + space.w_None] + elif imag_negzero: + tup = [obj, space.w_complex, space.w_None, space.w_None] + elif real_negzero: + tup = [obj, space.w_complex, space.w_None] + else: + tup = [obj, space.w_complex] + w_key = space.newtuple(tup) + else: + w_key = space.newtuple([obj, w_type]) w_len = space.finditem(self.w_consts, w_key) if w_len is None: w_len = space.len(self.w_consts) Modified: pypy/branch/fast-forward/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/test/test_compiler.py Fri Jul 2 21:37:01 2010 @@ -704,6 +704,20 @@ self.compiler = self.space.getexecutioncontext().compiler +class AppTestCompiler: + + def test_zeros_not_mixed(self): + import math + code = compile("x = -0.0; y = 0.0", "", "exec") + consts = code.co_consts + assert len(consts) == 3 + assert math.copysign(1, consts[0]) != math.copysign(1, consts[1]) + ns = {} + exec "z1, z2 = 0j, -0j" in ns + assert math.atan2(ns["z1"].imag, -1.) == math.atan2(0., -1.) + assert math.atan2(ns["z2"].imag, -1.) == math.atan2(-0., -1.) + + ##class TestPythonAstCompiler(BaseTestCompiler): ## def setup_method(self, method): ## self.compiler = PythonAstCompiler(self.space, "2.4") From jcreigh at codespeak.net Fri Jul 2 22:31:51 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 2 Jul 2010 22:31:51 +0200 (CEST) Subject: [pypy-svn] r75797 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100702203151.F370B282BEF@codespeak.net> Author: jcreigh Date: Fri Jul 2 22:31:50 2010 New Revision: 75797 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Log: assert that inputargs are unique Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Fri Jul 2 22:31:50 2010 @@ -253,6 +253,10 @@ _x86_param_depth _x86_arglocs """ + if not we_are_translated(): + # Arguments should be unique + assert len(dict.fromkeys(inputargs)) == len(inputargs) + funcname = self._find_debug_merge_point(operations) self.make_sure_mc_exists() @@ -284,6 +288,10 @@ self.write_pending_failure_recoveries() def assemble_bridge(self, faildescr, inputargs, operations): + if not we_are_translated(): + # Arguments should be unique + assert len(dict.fromkeys(inputargs)) == len(inputargs) + funcname = self._find_debug_merge_point(operations) self.make_sure_mc_exists() From benjamin at codespeak.net Fri Jul 2 22:32:45 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 2 Jul 2010 22:32:45 +0200 (CEST) Subject: [pypy-svn] r75798 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100702203245.BA9BE282BEF@codespeak.net> Author: benjamin Date: Fri Jul 2 22:32:44 2010 New Revision: 75798 Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py pypy/branch/fast-forward/pypy/objspace/std/floattype.py pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Log: add float.fromhex and float.hex Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Fri Jul 2 22:32:44 2010 @@ -7,6 +7,7 @@ from pypy.objspace.std.register_all import register_all from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.longobject import W_LongObject +from pypy.module.sys import system from pypy.rlib.rarithmetic import ovfcheck_float_to_int, intmask, isinf, isnan from pypy.rlib.rarithmetic import formatd, LONG_BIT, FL_MAXINT, FL_MININT from pypy.rlib.rbigint import rbigint @@ -86,6 +87,44 @@ def float_w__Float(space, w_float): return w_float.floatval +def _char_from_hex(number): + return "0123456789abcdef"[number] + +TOHEX_NBITS = system.DBL_MANT_DIG + 3 - (system.DBL_MANT_DIG + 2) % 4 + +def float_hex__Float(space, w_float): + value = w_float.floatval + if isinf(value) or isnan(value): + return str__Float(space, w_float) + if value == 0.0: + if math.copysign(1., value) == -1.: + return space.wrap("-0x0.0p+0") + else: + return space.wrap("0x0.0p+0") + mant, exp = math.frexp(value) + shift = 1 - max(system.DBL_MIN_EXP - exp, 0) + mant = math.ldexp(mant, shift) + mant = abs(mant) + exp -= shift + result = ['\0'] * ((TOHEX_NBITS - 1) // 4 + 2) + result[0] = _char_from_hex(int(mant)) + mant -= int(mant) + result[1] = "." + for i in range((TOHEX_NBITS - 1) // 4): + mant *= 16.0 + result[i + 2] = _char_from_hex(int(mant)) + mant -= int(mant) + if exp < 0: + sign = "-" + else: + sign = "+" + exp = abs(exp) + s = ''.join(result) + if value < 0.0: + return space.wrap("-0x%sp%s%d" % (s, sign, exp)) + else: + return space.wrap("0x%sp%s%d" % (s, sign, exp)) + def float2string(space, w_float, format): x = w_float.floatval # we special-case explicitly inf and nan here @@ -417,7 +456,8 @@ def getnewargs__Float(space, w_float): return space.newtuple([W_FloatObject(w_float.floatval)]) -register_all(vars()) +from pypy.objspace.std import floattype +register_all(vars(), floattype) # pow delegation for negative 2nd arg def pow_neg__Long_Long_None(space, w_int1, w_int2, thirdarg): Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floattype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floattype.py Fri Jul 2 22:32:44 2010 @@ -1,9 +1,17 @@ +import math +import sys +from pypy.rlib.unroll import unrolling_iterable from pypy.interpreter import gateway from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.error import OperationError -from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.objspace.std.stdtypedef import StdTypeDef, SMM from pypy.objspace.std.strutil import ParseStringError from pypy.objspace.std.strutil import interp_string_to_float +from pypy.module.sys import system + + +float_hex = SMM("hex", 1) + def descr__new__(space, w_floattype, w_x=0.0): from pypy.objspace.std.floatobject import W_FloatObject @@ -47,6 +55,157 @@ raise OperationError(space.w_ValueError, space.wrap("only float and double are valid")) +_alpha = zip("abcdef", range(10, 16)) + zip("ABCDEF", range(10, 16)) +_hex_to_int = zip("0123456789", range(10)) + _alpha +_hex_to_int_iterable = unrolling_iterable(_hex_to_int) +def _hex_from_char(c): + for h, v in _hex_to_int_iterable: + if h == c: + return v + return -1 + +def _hex_digit(s, j, co_end, float_digits): + if j < float_digits: + i = co_end - j + else: + i = co_end - 1 - j + return _hex_from_char(s[i]) + +def descr_fromhex(space, w_cls, s): + length = len(s) + i = 0 + while i < length and s[i].isspace(): + i += 1 + if i == length: + raise OperationError(space.w_ValueError, + space.wrap("invalid hex string")) + sign = 1 + if s[i] == "-": + sign = -1 + i += 1 + elif s[i] == "+": + i += 1 + if length == i: + raise OperationError(space.w_ValueError, + space.wrap("invalid hex string")) + if s[i] == "i" or s[i] == "I": + i += 1 + if length - i >= 2 and s[i:i + 2].lower() == "nf": + i += 2 + value = float("inf") + if length - i >= 5 and s[i:i + 5].lower() == "inity": + i += 5 + elif s[i] == "n" or s[i] == "N": + i += 1 + if length - i >= 2 and s[i:i + 2].lower() == "an": + i += 2 + value = float("nan") + else: + if (s[i] == "0" and length - i > 1 and + (s[i + 1] == "x" or s[i + 1] == "X")): + i += 2 + co_start = i + while i < length and _hex_from_char(s[i]) >= 0: + i += 1 + whole_end = i + if i < length and s[i] == ".": + i += 1 + while i < length and _hex_from_char(s[i]) >= 0: + i += 1 + co_end = i - 1 + else: + co_end = i + total_digits = co_end - co_start + float_digits = co_end - whole_end + if not total_digits: + raise OperationError(space.w_ValueError, + space.wrap("invalid hex string")) + const_one = system.DBL_MIN_EXP - system.DBL_MANT_DIG + sys.maxint // 2 + const_two = sys.maxint // 2 + 1 - system.DBL_MAX_EXP + if total_digits > min(const_one, const_two) // 4: + raise OperationError(space.w_ValueError, space.wrap("way too long")) + if i < length and (s[i] == "p" or s[i] == "P"): + if i == length: + raise OperationError(space.w_ValueError, + space.wrap("invalid hex string")) + i += 1 + exp_start = i + if s[i] == "-" or s[i] == "+": + i += 1 + if i == length: + raise OperationError(space.w_ValueError, + space.wrap("invalid hex string")) + if not s[i].isdigit(): + raise OperationError(space.w_ValueError, + space.wrap("invalid hex string")) + i += 1 + while i < length and s[i].isdigit(): + i += 1 + exp = int(s[exp_start:i]) + else: + exp = 0 + while (total_digits and + _hex_digit(s, total_digits - 1, co_end, float_digits) == 0): + total_digits -= 1 + if not total_digits or exp < -sys.maxint / 2: + value = 0.0 + elif exp > sys.maxint // 2: + raise OperationError(space.w_OverflowError, space.wrap("too large")) + else: + exp -= 4 * float_digits + top_exp = exp + 4 * (total_digits - 1) + digit = _hex_digit(s, total_digits - 1, co_end, float_digits) + while digit: + top_exp += 1 + digit //= 2 + if top_exp < system.DBL_MIN_EXP - system.DBL_MANT_DIG: + value = 0.0 + elif top_exp > system.DBL_MAX_EXP: + raise OperationError(space.w_OverflowError, + space.wrap("too large")) + else: + lsb = max(top_exp, system.DBL_MIN_EXP) - system.DBL_MANT_DIG + value = 0 + if exp >= lsb: + for j in range(total_digits - 1, -1, -1): + value = 16.0 * value + _hex_digit(s, j, co_end, + float_digits) + value = math.ldexp(value, exp) + else: + half_eps = 1 << ((lsb - exp - 1) % 4) + key_digit = (lsb - exp - 1) // 4 + for j in range(total_digits - 1, key_digit, -1): + value = 16.0 * value + _hex_digit(s, j, co_end, + float_digits) + digit = _hex_digit(s, key_digit, co_end, float_digits) + value = 16.0 * value + (digit & (16 - 2*half_eps)) + if digit & half_eps: + round_up = False + if (digit & (3 * half_eps - 1) or + (half_eps == 8 and + _hex_digit(s, key_digit + 1, co_end, float_digits) & 1)): + round_up = True + else: + for j in range(key_digit - 1, -1, -1): + if _hex_digit(s, j, co_end, float_digits): + round_up = True + break + if round_up: + value += 2 * half_eps + mant_dig = system.DBL_MANT_DIG + if (top_exp == system.DBL_MAX_EXP and + value == math.ldexp(2 * half_eps, mant_dig)): + raise OperationError(space.w_OverflowError, + space.wrap("too large")) + value = math.ldexp(value, (exp + 4*key_digit)) + while i < length and s[i].isspace(): + i += 1 + if i != length: + raise OperationError(space.w_ValueError, + space.wrap("invalid hex string")) + w_float = space.wrap(sign * value) + return space.call_function(w_cls, w_float) + # ____________________________________________________________ @@ -58,4 +217,8 @@ __getformat__ = gateway.interp2app(descr___getformat__, unwrap_spec=[ObjSpace, W_Root, str], as_classmethod=True), + fromhex = gateway.interp2app(descr_fromhex, + unwrap_spec=[ObjSpace, W_Root, str], + as_classmethod=True), ) +float_typedef.registermethods(globals()) Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Fri Jul 2 22:32:44 2010 @@ -343,3 +343,337 @@ #if hasattr(long, '__eq__'): # for py.test -A: CPython is inconsistent # assert 5L .__eq__(3.14) is NotImplemented # assert 3.14 .__eq__(5L) is False + + +class AppTestFloatHex: + + def setup_class(cls): + space = cls.space + cls.w_identical = space.appexec((), """(): + import math + def identical(x, y): + # check that floats x and y are identical, or that both + # are NaNs + if math.isnan(x) or math.isnan(y): + if math.isnan(x) == math.isnan(y): + return + assert (x == y and (x != 0.0 or + math.copysign(1.0, x) == math.copysign(1.0, y))) + return identical + """) + + + def test_from_hex(self): + fromHex = float.fromhex + import sys + INF = float("inf") + NAN = float("nan") + MIN = sys.float_info.min + MAX = sys.float_info.max + TINY = fromHex('0x0.0000000000001p-1022') # min subnormal + EPS = sys.float_info.epsilon + + # two spellings of infinity, with optional signs; case-insensitive + self.identical(fromHex('inf'), INF) + self.identical(fromHex('+Inf'), INF) + self.identical(fromHex('-INF'), -INF) + self.identical(fromHex('iNf'), INF) + self.identical(fromHex('Infinity'), INF) + self.identical(fromHex('+INFINITY'), INF) + self.identical(fromHex('-infinity'), -INF) + self.identical(fromHex('-iNFiNitY'), -INF) + + # nans with optional sign; case insensitive + self.identical(fromHex('nan'), NAN) + self.identical(fromHex('+NaN'), NAN) + self.identical(fromHex('-NaN'), NAN) + self.identical(fromHex('-nAN'), NAN) + + # variations in input format + self.identical(fromHex('1'), 1.0) + self.identical(fromHex('+1'), 1.0) + self.identical(fromHex('1.'), 1.0) + self.identical(fromHex('1.0'), 1.0) + self.identical(fromHex('1.0p0'), 1.0) + self.identical(fromHex('01'), 1.0) + self.identical(fromHex('01.'), 1.0) + self.identical(fromHex('0x1'), 1.0) + self.identical(fromHex('0x1.'), 1.0) + self.identical(fromHex('0x1.0'), 1.0) + self.identical(fromHex('+0x1.0'), 1.0) + self.identical(fromHex('0x1p0'), 1.0) + self.identical(fromHex('0X1p0'), 1.0) + self.identical(fromHex('0X1P0'), 1.0) + self.identical(fromHex('0x1P0'), 1.0) + self.identical(fromHex('0x1.p0'), 1.0) + self.identical(fromHex('0x1.0p0'), 1.0) + self.identical(fromHex('0x.1p4'), 1.0) + self.identical(fromHex('0x.1p04'), 1.0) + self.identical(fromHex('0x.1p004'), 1.0) + self.identical(fromHex('0x1p+0'), 1.0) + self.identical(fromHex('0x1P-0'), 1.0) + self.identical(fromHex('+0x1p0'), 1.0) + self.identical(fromHex('0x01p0'), 1.0) + self.identical(fromHex('0x1p00'), 1.0) + self.identical(fromHex(u'0x1p0'), 1.0) + self.identical(fromHex(' 0x1p0 '), 1.0) + self.identical(fromHex('\n 0x1p0'), 1.0) + self.identical(fromHex('0x1p0 \t'), 1.0) + self.identical(fromHex('0xap0'), 10.0) + self.identical(fromHex('0xAp0'), 10.0) + self.identical(fromHex('0xaP0'), 10.0) + self.identical(fromHex('0xAP0'), 10.0) + self.identical(fromHex('0xbep0'), 190.0) + self.identical(fromHex('0xBep0'), 190.0) + self.identical(fromHex('0xbEp0'), 190.0) + self.identical(fromHex('0XBE0P-4'), 190.0) + self.identical(fromHex('0xBEp0'), 190.0) + self.identical(fromHex('0xB.Ep4'), 190.0) + self.identical(fromHex('0x.BEp8'), 190.0) + self.identical(fromHex('0x.0BEp12'), 190.0) + + # moving the point around + pi = fromHex('0x1.921fb54442d18p1') + self.identical(fromHex('0x.006487ed5110b46p11'), pi) + self.identical(fromHex('0x.00c90fdaa22168cp10'), pi) + self.identical(fromHex('0x.01921fb54442d18p9'), pi) + self.identical(fromHex('0x.03243f6a8885a3p8'), pi) + self.identical(fromHex('0x.06487ed5110b46p7'), pi) + self.identical(fromHex('0x.0c90fdaa22168cp6'), pi) + self.identical(fromHex('0x.1921fb54442d18p5'), pi) + self.identical(fromHex('0x.3243f6a8885a3p4'), pi) + self.identical(fromHex('0x.6487ed5110b46p3'), pi) + self.identical(fromHex('0x.c90fdaa22168cp2'), pi) + self.identical(fromHex('0x1.921fb54442d18p1'), pi) + self.identical(fromHex('0x3.243f6a8885a3p0'), pi) + self.identical(fromHex('0x6.487ed5110b46p-1'), pi) + self.identical(fromHex('0xc.90fdaa22168cp-2'), pi) + self.identical(fromHex('0x19.21fb54442d18p-3'), pi) + self.identical(fromHex('0x32.43f6a8885a3p-4'), pi) + self.identical(fromHex('0x64.87ed5110b46p-5'), pi) + self.identical(fromHex('0xc9.0fdaa22168cp-6'), pi) + self.identical(fromHex('0x192.1fb54442d18p-7'), pi) + self.identical(fromHex('0x324.3f6a8885a3p-8'), pi) + self.identical(fromHex('0x648.7ed5110b46p-9'), pi) + self.identical(fromHex('0xc90.fdaa22168cp-10'), pi) + self.identical(fromHex('0x1921.fb54442d18p-11'), pi) + # ... + self.identical(fromHex('0x1921fb54442d1.8p-47'), pi) + self.identical(fromHex('0x3243f6a8885a3p-48'), pi) + self.identical(fromHex('0x6487ed5110b46p-49'), pi) + self.identical(fromHex('0xc90fdaa22168cp-50'), pi) + self.identical(fromHex('0x1921fb54442d18p-51'), pi) + self.identical(fromHex('0x3243f6a8885a30p-52'), pi) + self.identical(fromHex('0x6487ed5110b460p-53'), pi) + self.identical(fromHex('0xc90fdaa22168c0p-54'), pi) + self.identical(fromHex('0x1921fb54442d180p-55'), pi) + + + # results that should overflow... + raises(OverflowError, fromHex, '-0x1p1024') + raises(OverflowError, fromHex, '0x1p+1025') + raises(OverflowError, fromHex, '+0X1p1030') + raises(OverflowError, fromHex, '-0x1p+1100') + raises(OverflowError, fromHex, '0X1p123456789123456789') + raises(OverflowError, fromHex, '+0X.8p+1025') + raises(OverflowError, fromHex, '+0x0.8p1025') + raises(OverflowError, fromHex, '-0x0.4p1026') + raises(OverflowError, fromHex, '0X2p+1023') + raises(OverflowError, fromHex, '0x2.p1023') + raises(OverflowError, fromHex, '-0x2.0p+1023') + raises(OverflowError, fromHex, '+0X4p+1022') + raises(OverflowError, fromHex, '0x1.ffffffffffffffp+1023') + raises(OverflowError, fromHex, '-0X1.fffffffffffff9p1023') + raises(OverflowError, fromHex, '0X1.fffffffffffff8p1023') + raises(OverflowError, fromHex, '+0x3.fffffffffffffp1022') + raises(OverflowError, fromHex, '0x3fffffffffffffp+970') + raises(OverflowError, fromHex, '0x10000000000000000p960') + raises(OverflowError, fromHex, '-0Xffffffffffffffffp960') + + # ...and those that round to +-max float + self.identical(fromHex('+0x1.fffffffffffffp+1023'), MAX) + self.identical(fromHex('-0X1.fffffffffffff7p1023'), -MAX) + self.identical(fromHex('0X1.fffffffffffff7fffffffffffffp1023'), MAX) + + # zeros + self.identical(fromHex('0x0p0'), 0.0) + self.identical(fromHex('0x0p1000'), 0.0) + self.identical(fromHex('-0x0p1023'), -0.0) + self.identical(fromHex('0X0p1024'), 0.0) + self.identical(fromHex('-0x0p1025'), -0.0) + self.identical(fromHex('0X0p2000'), 0.0) + self.identical(fromHex('0x0p123456789123456789'), 0.0) + self.identical(fromHex('-0X0p-0'), -0.0) + self.identical(fromHex('-0X0p-1000'), -0.0) + self.identical(fromHex('0x0p-1023'), 0.0) + self.identical(fromHex('-0X0p-1024'), -0.0) + self.identical(fromHex('-0x0p-1025'), -0.0) + self.identical(fromHex('-0x0p-1072'), -0.0) + self.identical(fromHex('0X0p-1073'), 0.0) + self.identical(fromHex('-0x0p-1074'), -0.0) + self.identical(fromHex('0x0p-1075'), 0.0) + self.identical(fromHex('0X0p-1076'), 0.0) + self.identical(fromHex('-0X0p-2000'), -0.0) + self.identical(fromHex('-0x0p-123456789123456789'), -0.0) + + # values that should underflow to 0 + self.identical(fromHex('0X1p-1075'), 0.0) + self.identical(fromHex('-0X1p-1075'), -0.0) + self.identical(fromHex('-0x1p-123456789123456789'), -0.0) + self.identical(fromHex('0x1.00000000000000001p-1075'), TINY) + self.identical(fromHex('-0x1.1p-1075'), -TINY) + self.identical(fromHex('0x1.fffffffffffffffffp-1075'), TINY) + + # check round-half-even is working correctly near 0 ... + self.identical(fromHex('0x1p-1076'), 0.0) + self.identical(fromHex('0X2p-1076'), 0.0) + self.identical(fromHex('0X3p-1076'), TINY) + self.identical(fromHex('0x4p-1076'), TINY) + self.identical(fromHex('0X5p-1076'), TINY) + self.identical(fromHex('0X6p-1076'), 2*TINY) + self.identical(fromHex('0x7p-1076'), 2*TINY) + self.identical(fromHex('0X8p-1076'), 2*TINY) + self.identical(fromHex('0X9p-1076'), 2*TINY) + self.identical(fromHex('0xap-1076'), 2*TINY) + self.identical(fromHex('0Xbp-1076'), 3*TINY) + self.identical(fromHex('0xcp-1076'), 3*TINY) + self.identical(fromHex('0Xdp-1076'), 3*TINY) + self.identical(fromHex('0Xep-1076'), 4*TINY) + self.identical(fromHex('0xfp-1076'), 4*TINY) + self.identical(fromHex('0x10p-1076'), 4*TINY) + self.identical(fromHex('-0x1p-1076'), -0.0) + self.identical(fromHex('-0X2p-1076'), -0.0) + self.identical(fromHex('-0x3p-1076'), -TINY) + self.identical(fromHex('-0X4p-1076'), -TINY) + self.identical(fromHex('-0x5p-1076'), -TINY) + self.identical(fromHex('-0x6p-1076'), -2*TINY) + self.identical(fromHex('-0X7p-1076'), -2*TINY) + self.identical(fromHex('-0X8p-1076'), -2*TINY) + self.identical(fromHex('-0X9p-1076'), -2*TINY) + self.identical(fromHex('-0Xap-1076'), -2*TINY) + self.identical(fromHex('-0xbp-1076'), -3*TINY) + self.identical(fromHex('-0xcp-1076'), -3*TINY) + self.identical(fromHex('-0Xdp-1076'), -3*TINY) + self.identical(fromHex('-0xep-1076'), -4*TINY) + self.identical(fromHex('-0Xfp-1076'), -4*TINY) + self.identical(fromHex('-0X10p-1076'), -4*TINY) + + # ... and near MIN ... + self.identical(fromHex('0x0.ffffffffffffd6p-1022'), MIN-3*TINY) + self.identical(fromHex('0x0.ffffffffffffd8p-1022'), MIN-2*TINY) + self.identical(fromHex('0x0.ffffffffffffdap-1022'), MIN-2*TINY) + self.identical(fromHex('0x0.ffffffffffffdcp-1022'), MIN-2*TINY) + self.identical(fromHex('0x0.ffffffffffffdep-1022'), MIN-2*TINY) + self.identical(fromHex('0x0.ffffffffffffe0p-1022'), MIN-2*TINY) + self.identical(fromHex('0x0.ffffffffffffe2p-1022'), MIN-2*TINY) + self.identical(fromHex('0x0.ffffffffffffe4p-1022'), MIN-2*TINY) + self.identical(fromHex('0x0.ffffffffffffe6p-1022'), MIN-2*TINY) + self.identical(fromHex('0x0.ffffffffffffe8p-1022'), MIN-2*TINY) + self.identical(fromHex('0x0.ffffffffffffeap-1022'), MIN-TINY) + self.identical(fromHex('0x0.ffffffffffffecp-1022'), MIN-TINY) + self.identical(fromHex('0x0.ffffffffffffeep-1022'), MIN-TINY) + self.identical(fromHex('0x0.fffffffffffff0p-1022'), MIN-TINY) + self.identical(fromHex('0x0.fffffffffffff2p-1022'), MIN-TINY) + self.identical(fromHex('0x0.fffffffffffff4p-1022'), MIN-TINY) + self.identical(fromHex('0x0.fffffffffffff6p-1022'), MIN-TINY) + self.identical(fromHex('0x0.fffffffffffff8p-1022'), MIN) + self.identical(fromHex('0x0.fffffffffffffap-1022'), MIN) + self.identical(fromHex('0x0.fffffffffffffcp-1022'), MIN) + self.identical(fromHex('0x0.fffffffffffffep-1022'), MIN) + self.identical(fromHex('0x1.00000000000000p-1022'), MIN) + self.identical(fromHex('0x1.00000000000002p-1022'), MIN) + self.identical(fromHex('0x1.00000000000004p-1022'), MIN) + self.identical(fromHex('0x1.00000000000006p-1022'), MIN) + self.identical(fromHex('0x1.00000000000008p-1022'), MIN) + self.identical(fromHex('0x1.0000000000000ap-1022'), MIN+TINY) + self.identical(fromHex('0x1.0000000000000cp-1022'), MIN+TINY) + self.identical(fromHex('0x1.0000000000000ep-1022'), MIN+TINY) + self.identical(fromHex('0x1.00000000000010p-1022'), MIN+TINY) + self.identical(fromHex('0x1.00000000000012p-1022'), MIN+TINY) + self.identical(fromHex('0x1.00000000000014p-1022'), MIN+TINY) + self.identical(fromHex('0x1.00000000000016p-1022'), MIN+TINY) + self.identical(fromHex('0x1.00000000000018p-1022'), MIN+2*TINY) + + # ... and near 1.0. + self.identical(fromHex('0x0.fffffffffffff0p0'), 1.0-EPS) + self.identical(fromHex('0x0.fffffffffffff1p0'), 1.0-EPS) + self.identical(fromHex('0X0.fffffffffffff2p0'), 1.0-EPS) + self.identical(fromHex('0x0.fffffffffffff3p0'), 1.0-EPS) + self.identical(fromHex('0X0.fffffffffffff4p0'), 1.0-EPS) + self.identical(fromHex('0X0.fffffffffffff5p0'), 1.0-EPS/2) + self.identical(fromHex('0X0.fffffffffffff6p0'), 1.0-EPS/2) + self.identical(fromHex('0x0.fffffffffffff7p0'), 1.0-EPS/2) + self.identical(fromHex('0x0.fffffffffffff8p0'), 1.0-EPS/2) + self.identical(fromHex('0X0.fffffffffffff9p0'), 1.0-EPS/2) + self.identical(fromHex('0X0.fffffffffffffap0'), 1.0-EPS/2) + self.identical(fromHex('0x0.fffffffffffffbp0'), 1.0-EPS/2) + self.identical(fromHex('0X0.fffffffffffffcp0'), 1.0) + self.identical(fromHex('0x0.fffffffffffffdp0'), 1.0) + self.identical(fromHex('0X0.fffffffffffffep0'), 1.0) + self.identical(fromHex('0x0.ffffffffffffffp0'), 1.0) + self.identical(fromHex('0X1.00000000000000p0'), 1.0) + self.identical(fromHex('0X1.00000000000001p0'), 1.0) + self.identical(fromHex('0x1.00000000000002p0'), 1.0) + self.identical(fromHex('0X1.00000000000003p0'), 1.0) + self.identical(fromHex('0x1.00000000000004p0'), 1.0) + self.identical(fromHex('0X1.00000000000005p0'), 1.0) + self.identical(fromHex('0X1.00000000000006p0'), 1.0) + self.identical(fromHex('0X1.00000000000007p0'), 1.0) + self.identical(fromHex('0x1.00000000000007ffffffffffffffffffffp0'), + 1.0) + self.identical(fromHex('0x1.00000000000008p0'), 1.0) + self.identical(fromHex('0x1.00000000000008000000000000000001p0'), + 1+EPS) + self.identical(fromHex('0X1.00000000000009p0'), 1.0+EPS) + self.identical(fromHex('0x1.0000000000000ap0'), 1.0+EPS) + self.identical(fromHex('0x1.0000000000000bp0'), 1.0+EPS) + self.identical(fromHex('0X1.0000000000000cp0'), 1.0+EPS) + self.identical(fromHex('0x1.0000000000000dp0'), 1.0+EPS) + self.identical(fromHex('0x1.0000000000000ep0'), 1.0+EPS) + self.identical(fromHex('0X1.0000000000000fp0'), 1.0+EPS) + self.identical(fromHex('0x1.00000000000010p0'), 1.0+EPS) + self.identical(fromHex('0X1.00000000000011p0'), 1.0+EPS) + self.identical(fromHex('0x1.00000000000012p0'), 1.0+EPS) + self.identical(fromHex('0X1.00000000000013p0'), 1.0+EPS) + self.identical(fromHex('0X1.00000000000014p0'), 1.0+EPS) + self.identical(fromHex('0x1.00000000000015p0'), 1.0+EPS) + self.identical(fromHex('0x1.00000000000016p0'), 1.0+EPS) + self.identical(fromHex('0X1.00000000000017p0'), 1.0+EPS) + self.identical(fromHex('0x1.00000000000017ffffffffffffffffffffp0'), + 1.0+EPS) + self.identical(fromHex('0x1.00000000000018p0'), 1.0+2*EPS) + self.identical(fromHex('0X1.00000000000018000000000000000001p0'), + 1.0+2*EPS) + self.identical(fromHex('0x1.00000000000019p0'), 1.0+2*EPS) + self.identical(fromHex('0X1.0000000000001ap0'), 1.0+2*EPS) + self.identical(fromHex('0X1.0000000000001bp0'), 1.0+2*EPS) + self.identical(fromHex('0x1.0000000000001cp0'), 1.0+2*EPS) + self.identical(fromHex('0x1.0000000000001dp0'), 1.0+2*EPS) + self.identical(fromHex('0x1.0000000000001ep0'), 1.0+2*EPS) + self.identical(fromHex('0X1.0000000000001fp0'), 1.0+2*EPS) + self.identical(fromHex('0x1.00000000000020p0'), 1.0+2*EPS) + + def test_roundtrip(self): + def roundtrip(x): + return float.fromhex(x.hex()) + import sys + import math + TINY = float.fromhex('0x0.0000000000001p-1022') # min subnormal + + for x in [float("nan"), float("inf"), sys.float_info.max, + sys.float_info.min, sys.float_info.min-TINY, TINY, 0.0]: + self.identical(x, roundtrip(x)) + self.identical(-x, roundtrip(-x)) + + # fromHex(toHex(x)) should exactly recover x, for any non-NaN float x. + import random + for i in xrange(500): + e = random.randrange(-1200, 1200) + m = random.random() + s = random.choice([1.0, -1.0]) + try: + x = s*math.ldexp(m, e) + except OverflowError: + pass + else: + self.identical(x, float.fromhex(x.hex())) From arigo at codespeak.net Fri Jul 2 22:38:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 22:38:10 +0200 (CEST) Subject: [pypy-svn] r75799 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100702203810.41B17282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 22:38:08 2010 New Revision: 75799 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Log: Add a convenient method. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Fri Jul 2 22:38:08 2010 @@ -37,8 +37,10 @@ class MatchContext(object): + match_start = 0 match_end = 0 match_marks = None + match_marks_flat = None def __init__(self, pattern, string, flags): self.pattern = pattern @@ -59,9 +61,23 @@ return rsre_char.getlower(c, self.flags) def get_mark(self, gid): - """Use this for testing.""" return find_mark(self.match_marks, gid) + def flatten_marks(self): + # for testing + if self.match_marks_flat is None: + self.match_marks_flat = [self.match_start, self.match_end] + mark = self.match_marks + while mark is not None: + index = mark.gid + 2 + while index >= len(self.match_marks_flat): + self.match_marks_flat.append(-1) + if self.match_marks_flat[index] == -1: + self.match_marks_flat[index] = mark.position + mark = mark.prev + self.match_marks = None # clear + return self.match_marks_flat + class Mark(object): _immutable_ = True Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Fri Jul 2 22:38:08 2010 @@ -150,3 +150,13 @@ r, _ = get_code(r"(?i)[^a]+$") assert rsre.match(r, "Gx123") assert not rsre.match(r, "--A--") + + def test_repeated_single_character_pattern(self): + r, _ = get_code(r"foo(?:(?<=foo)x)+$") + assert rsre.match(r, "foox") + + def test_flatten_marks(self): + r, _ = get_code(r"a(b)c((d)(e))+$") + res = rsre.match(r, "abcdedede") + assert res.flatten_marks() == [0, 9, 1, 2, 7, 9, 7, 8, 8, 9] + assert res.flatten_marks() == [0, 9, 1, 2, 7, 9, 7, 8, 8, 9] From jcreigh at codespeak.net Fri Jul 2 22:38:23 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 2 Jul 2010 22:38:23 +0200 (CEST) Subject: [pypy-svn] r75800 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100702203823.89F5E282BEF@codespeak.net> Author: jcreigh Date: Fri Jul 2 22:38:22 2010 New Revision: 75800 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Log: use set instead of dict.fromkeys :-) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Fri Jul 2 22:38:22 2010 @@ -255,7 +255,7 @@ """ if not we_are_translated(): # Arguments should be unique - assert len(dict.fromkeys(inputargs)) == len(inputargs) + assert len(set(inputargs)) == len(inputargs) funcname = self._find_debug_merge_point(operations) @@ -290,7 +290,7 @@ def assemble_bridge(self, faildescr, inputargs, operations): if not we_are_translated(): # Arguments should be unique - assert len(dict.fromkeys(inputargs)) == len(inputargs) + assert len(set(inputargs)) == len(inputargs) funcname = self._find_debug_merge_point(operations) From arigo at codespeak.net Fri Jul 2 22:46:48 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Jul 2010 22:46:48 +0200 (CEST) Subject: [pypy-svn] r75801 - pypy/branch/rsre2/pypy/rlib/rsre/test Message-ID: <20100702204648.12794282BEF@codespeak.net> Author: arigo Date: Fri Jul 2 22:46:46 2010 New Revision: 75801 Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py Log: Update tests. Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Fri Jul 2 22:46:46 2010 @@ -1,4 +1,4 @@ -import _sre, re +import _sre, re, sre_compile import rsre @@ -12,58 +12,66 @@ try: _sre.compile = my_compile try: - re.compile(regexp) + sre_compile.compile(regexp, 0) except GotIt, e: pass else: raise ValueError("did not reach _sre.compile()!") finally: _sre.compile = saved - return e.args[0], re.compile(regexp) + return e.args[0] + +def get_code_and_re(regexp): + return get_code(regexp), re.compile(regexp) + +def test_get_code_repetition(): + c1 = get_code(r"a+") + c2 = get_code(r"a+") + assert c1 == c2 class TestMatch: def test_any(self): - r, _ = get_code(r"ab.cd") + r = get_code(r"ab.cd") assert rsre.match(r, "abXcdef") assert not rsre.match(r, "ab\ncdef") assert not rsre.match(r, "abXcDef") def test_any_repetition(self): - r, _ = get_code(r"ab.*cd") + r = get_code(r"ab.*cd") assert rsre.match(r, "abXXXXcdef") assert rsre.match(r, "abcdef") assert not rsre.match(r, "abX\nXcdef") assert not rsre.match(r, "abXXXXcDef") def test_any_all(self): - r, _ = get_code(r"(?s)ab.cd") + r = get_code(r"(?s)ab.cd") assert rsre.match(r, "abXcdef") assert rsre.match(r, "ab\ncdef") assert not rsre.match(r, "ab\ncDef") def test_any_all_repetition(self): - r, _ = get_code(r"(?s)ab.*cd") + r = get_code(r"(?s)ab.*cd") assert rsre.match(r, "abXXXXcdef") assert rsre.match(r, "abcdef") assert rsre.match(r, "abX\nXcdef") assert not rsre.match(r, "abX\nXcDef") def test_assert(self): - r, _ = get_code(r"abc(?=def)(.)") + r = get_code(r"abc(?=def)(.)") res = rsre.match(r, "abcdefghi") assert res is not None and res.get_mark(1) == 4 assert not rsre.match(r, "abcdeFghi") def test_assert_not(self): - r, _ = get_code(r"abc(?!def)(.)") + r = get_code(r"abc(?!def)(.)") res = rsre.match(r, "abcdeFghi") assert res is not None and res.get_mark(1) == 4 assert not rsre.match(r, "abcdefghi") def test_lookbehind(self): - r, _ = get_code(r"([a-z]*)(?<=de)") + r = get_code(r"([a-z]*)(?<=de)") assert rsre.match(r, "ade") res = rsre.match(r, "adefg") assert res is not None and res.get_mark(1) == 3 @@ -76,7 +84,7 @@ res = rsre.match(r, s) assert res is not None return res.get_mark(1) - r, _ = get_code(r"([a-z]*)(?\s*(.*?)') + r_code1 = get_code(r'\s*(.*?)') res = rsre.match(r_code1, " abcdef") assert res is not None #groups = state.create_regs(1) @@ -13,7 +13,7 @@ #assert groups[1] == (18, 21) def test_max_until_0_65535(self): - r_code2, _ = get_code(r'(?:xy)*xy') + r_code2 = get_code(r'(?:xy)*xy') res = rsre.match(r_code2, 'def') assert res is None res = rsre.match(r_code2, 'xydef') @@ -24,7 +24,7 @@ assert res is not None def test_max_until_3_5(self): - r_code2, r = get_code(r'(?:xy){3,5}xy') + r_code2, r = get_code_and_re(r'(?:xy){3,5}xy') for i in range(8): s = '' + 'xy'*i + 'defdefdefdefdef' assert (r.match(s) is not None) is (3 <= i-1 <= 5) @@ -32,7 +32,7 @@ assert (res is not None) is (3 <= i-1 <= 5) def test_min_until_0_65535(self): - r_code2, _ = get_code(r'(?:xy)*?xy') + r_code2 = get_code(r'(?:xy)*?xy') res = rsre.match(r_code2, 'def') assert res is None res = rsre.match(r_code2, 'xydef') @@ -43,7 +43,7 @@ assert res is not None def test_min_until_3_5(self): - r_code2, r = get_code(r'(?:xy){3,5}?xy') + r_code2, r = get_code_and_re(r'(?:xy){3,5}?xy') for i in range(8): s = '' + 'xy'*i + 'defdefdefdefdef' assert (r.match(s) is not None) is (3 <= i-1 <= 5) @@ -51,27 +51,27 @@ assert (res is not None) is (3 <= i-1 <= 5) def test_min_repeat_one(self): - r_code3, _ = get_code(r'.{3,5}?y') + r_code3 = get_code(r'.{3,5}?y') for i in range(8): res = rsre.match(r_code3, '' + 'x'*i + 'y') assert (res is not None) is (3 <= i <= 5) def test_simple_group(self): - r_code4, _ = get_code(r'(x.)') + r_code4 = get_code(r'(x.)') res = rsre.match(r_code4, 'xadef') assert res is not None assert res.get_mark(0) == 5 assert res.get_mark(1) == 7 def test_max_until_groups(self): - r_code4, _ = get_code(r'(x.)*xy') + r_code4 = get_code(r'(x.)*xy') res = rsre.match(r_code4, 'xaxbxydef') assert res is not None assert res.get_mark(0) == 7 assert res.get_mark(1) == 9 def test_group_branch(self): - r_code5, _ = get_code(r'(ab|c)') + r_code5 = get_code(r'(ab|c)') res = rsre.match(r_code5, 'abdef') assert (res.get_mark(0), res.get_mark(1)) == (5, 7) res = rsre.match(r_code5, 'cdef') @@ -80,17 +80,17 @@ assert res is None def test_group_branch_max_until(self): - r_code6, _ = get_code(r'(ab|c)*a') + r_code6 = get_code(r'(ab|c)*a') res = rsre.match(r_code6, 'ccabcccabadef') assert (res.get_mark(0), res.get_mark(1)) == (12, 14) - r_code7, _ = get_code(r'((ab)|(c))*a') + r_code7 = get_code(r'((ab)|(c))*a') res = rsre.match(r_code7, 'ccabcccabadef') assert (res.get_mark(0), res.get_mark(1)) == (12, 14) assert (res.get_mark(2), res.get_mark(3)) == (12, 14) assert (res.get_mark(4), res.get_mark(5)) == (11, 12) def test_group_7(self): - r_code7, r7 = get_code(r'((a)?(b))*') + r_code7, r7 = get_code_and_re(r'((a)?(b))*') match = r7.match('bbbabbbb') assert match.span(1) == (12, 13) assert match.span(3) == (12, 13) @@ -101,7 +101,7 @@ assert (res.get_mark(2), res.get_mark(3)) == (8, 9) def test_group_branch_repeat_complex_case(self): - r_code8, r8 = get_code(r'((a)|(b))*') + r_code8, r8 = get_code_and_re(r'((a)|(b))*') match = r8.match('ab') assert match.span(1) == (6, 7) assert match.span(3) == (6, 7) @@ -112,7 +112,7 @@ assert (res.get_mark(2), res.get_mark(3)) == (5, 6) def test_minuntil_lastmark_restore(self): - r_code9, r9 = get_code(r'(x|yz)+?(y)??c') + r_code9, r9 = get_code_and_re(r'(x|yz)+?(y)??c') match = r9.match('xyzxc') assert match.span(1) == (3, 4) assert match.span(2) == (-1, -1) @@ -121,7 +121,7 @@ assert (res.get_mark(2), res.get_mark(3)) == (-1, -1) def test_minuntil_bug(self): - r_code9, r9 = get_code(r'((x|yz)+?(y)??c)*') + r_code9, r9 = get_code_and_re(r'((x|yz)+?(y)??c)*') match = r9.match('xycxyzxc') assert match.span(2) == (6, 7) #assert match.span(3) == (1, 2) --- bug of CPython From benjamin at codespeak.net Sat Jul 3 00:08:00 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Jul 2010 00:08:00 +0200 (CEST) Subject: [pypy-svn] r75802 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100702220800.505DD282BEF@codespeak.net> Author: benjamin Date: Sat Jul 3 00:07:58 2010 New Revision: 75802 Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Log: pow(0.0, -inf) is infinity Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Sat Jul 3 00:07:58 2010 @@ -433,9 +433,13 @@ return space.wrap(1.0) else: return space.wrap( -1.0) - elif x == 0.0 and y < 0.0: - raise OperationError(space.w_ZeroDivisionError, - space.wrap("0.0 cannot be raised to a negative power")) + elif x == 0.0: + if y < 0.0: + if isinf(y): + return space.wrap(float("inf")) + raise OperationError(space.w_ZeroDivisionError, + space.wrap("0.0 cannot be raised to " + "a negative power")) raise OperationError(space.w_ValueError, space.wrap("float power")) return W_FloatObject(z) Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Sat Jul 3 00:07:58 2010 @@ -158,6 +158,7 @@ assert pw(-1.0, 2.0) == 1.0 assert pw(-1.0, 3.0) == -1.0 assert pw(-1.0, 1e200) == 1.0 + assert pw(0.0, float("-inf")) == float("inf") def test_pow_neg_base(self): def pw(x, y): From benjamin at codespeak.net Sat Jul 3 00:11:04 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Jul 2010 00:11:04 +0200 (CEST) Subject: [pypy-svn] r75803 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100702221104.6791C282BEF@codespeak.net> Author: benjamin Date: Sat Jul 3 00:11:03 2010 New Revision: 75803 Modified: pypy/branch/fast-forward/pypy/objspace/std/strutil.py pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Log: allow +inf float string Modified: pypy/branch/fast-forward/pypy/objspace/std/strutil.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/strutil.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/strutil.py Sat Jul 3 00:11:03 2010 @@ -180,7 +180,7 @@ low = s.lower() if low == "-inf": return -INFINITY - elif low == "inf": + elif low == "inf" or low == "+inf": return INFINITY elif low == "nan" or low == "-nan": return NAN Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Sat Jul 3 00:11:03 2010 @@ -94,6 +94,7 @@ assert 42.25 == float("42.25") inf = 1e200*1e200 assert float("inf") == inf + assert float("+inf") == inf assert float("-INf") == -inf assert str(inf) == "inf" assert str(-inf) == "-inf" From benjamin at codespeak.net Sat Jul 3 00:16:17 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Jul 2010 00:16:17 +0200 (CEST) Subject: [pypy-svn] r75804 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100702221617.13DAF282BEF@codespeak.net> Author: benjamin Date: Sat Jul 3 00:16:15 2010 New Revision: 75804 Modified: pypy/branch/fast-forward/pypy/objspace/std/strutil.py pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Log: allow +nan, too Modified: pypy/branch/fast-forward/pypy/objspace/std/strutil.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/strutil.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/strutil.py Sat Jul 3 00:16:15 2010 @@ -182,7 +182,7 @@ return -INFINITY elif low == "inf" or low == "+inf": return INFINITY - elif low == "nan" or low == "-nan": + elif low == "nan" or low == "-nan" or low == "+nan": return NAN # 1) parse the string into pieces. Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Sat Jul 3 00:16:15 2010 @@ -103,6 +103,7 @@ assert repr(inf) == "inf" assert repr(-inf) == "-inf" assert repr(float("nan")) == "nan" + assert repr(float("+nan")) == "nan" assert repr(float("-nAn")) == "nan" def test_float_unicode(self): From benjamin at codespeak.net Sat Jul 3 00:25:51 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Jul 2010 00:25:51 +0200 (CEST) Subject: [pypy-svn] r75805 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100702222551.BB77D282BEF@codespeak.net> Author: benjamin Date: Sat Jul 3 00:25:50 2010 New Revision: 75805 Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Log: always ask for __float__ first Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floattype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floattype.py Sat Jul 3 00:25:50 2010 @@ -16,7 +16,14 @@ def descr__new__(space, w_floattype, w_x=0.0): from pypy.objspace.std.floatobject import W_FloatObject w_value = w_x # 'x' is the keyword argument name in CPython - if space.is_true(space.isinstance(w_value, space.w_str)): + w_special = space.lookup(w_x, "__float__") + if w_special is not None: + w_obj = space.get_and_call_function(w_special, w_x) + if not space.isinstance_w(w_obj, space.w_float): + raise OperationError(space.w_TypeError, + space.wrap("__float__ returned non-float")) + return w_obj + elif space.is_true(space.isinstance(w_value, space.w_str)): strvalue = space.str_w(w_value) try: value = interp_string_to_float(space, strvalue) Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Sat Jul 3 00:25:50 2010 @@ -109,6 +109,10 @@ def test_float_unicode(self): # u00A0 and u2000 are some kind of spaces assert 42.75 == float(unichr(0x00A0)+unicode("42.75")+unichr(0x2000)) + class FloatUnicode(unicode): + def __float__(self): + return float(unicode(self)) + 1 + assert float(FloatUnicode("8")) == 9.0 def test_float_long(self): assert 42.0 == float(42L) From benjamin at codespeak.net Sat Jul 3 00:28:30 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Jul 2010 00:28:30 +0200 (CEST) Subject: [pypy-svn] r75806 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100702222830.EDDDA282BEF@codespeak.net> Author: benjamin Date: Sat Jul 3 00:28:29 2010 New Revision: 75806 Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py Log: remove now unneeded code Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floattype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floattype.py Sat Jul 3 00:28:29 2010 @@ -42,11 +42,7 @@ raise OperationError(space.w_ValueError, space.wrap(e.msg)) else: - w_obj = space.float(w_value) - if space.is_w(w_floattype, space.w_float): - return w_obj # 'float(x)' should return - # whatever x.__float__() returned - value = space.float_w(w_obj) + value = space.float_w(w_x) w_obj = space.allocate_instance(W_FloatObject, w_floattype) W_FloatObject.__init__(w_obj, value) return w_obj From benjamin at codespeak.net Sat Jul 3 00:44:48 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Jul 2010 00:44:48 +0200 (CEST) Subject: [pypy-svn] r75807 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100702224448.04F1F282BEF@codespeak.net> Author: benjamin Date: Sat Jul 3 00:44:46 2010 New Revision: 75807 Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py pypy/branch/fast-forward/pypy/objspace/std/floattype.py pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Log: add float.as_integer_ratio() Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Sat Jul 3 00:44:46 2010 @@ -460,6 +460,31 @@ def getnewargs__Float(space, w_float): return space.newtuple([W_FloatObject(w_float.floatval)]) +def float_as_integer_ratio__Float(space, w_float): + value = w_float.floatval + if isinf(value): + w_msg = space.wrap("cannot pass infinity to as_integer_ratio()") + raise OperationError(space.w_OverflowError, w_msg) + elif isnan(value): + w_msg = space.wrap("cannot pass nan to as_integer_ratio()") + raise OperationError(space.w_ValueError, w_msg) + float_part, exp = math.frexp(value) + for i in range(300): + if float_part == math.floor(float_part): + break + float_part *= 2.0 + exp -= 1 + w_num = W_LongObject.fromfloat(float_part) + w_dem = space.newlong(1) + w_exp = space.newlong(abs(exp)) + w_exp = space.lshift(w_dem, w_exp) + if exp > 0: + w_num = space.mul(w_num, w_exp) + else: + w_dem = w_exp + # Try to return int. + return space.newtuple([space.int(w_num), space.int(w_dem)]) + from pypy.objspace.std import floattype register_all(vars(), floattype) Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floattype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floattype.py Sat Jul 3 00:44:46 2010 @@ -10,6 +10,7 @@ from pypy.module.sys import system +float_as_integer_ratio = SMM("as_integer_ratio", 1) float_hex = SMM("hex", 1) Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Sat Jul 3 00:44:46 2010 @@ -118,8 +118,20 @@ assert 42.0 == float(42L) assert 10000000000.0 == float(10000000000L) raises(OverflowError, float, 10**400) - - + + def test_as_integer_ratio(self): + for f, ratio in [ + (0.875, (7, 8)), + (-0.875, (-7, 8)), + (0.0, (0, 1)), + (11.5, (23, 2)), + ]: + assert f.as_integer_ratio() == ratio + + raises(OverflowError, float('inf').as_integer_ratio) + raises(OverflowError, float('-inf').as_integer_ratio) + raises(ValueError, float('nan').as_integer_ratio) + def test_round(self): assert 1.0 == round(1.0) assert 1.0 == round(1.1) From afa at codespeak.net Sat Jul 3 01:03:59 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 3 Jul 2010 01:03:59 +0200 (CEST) Subject: [pypy-svn] r75808 - pypy/branch/unicode_filename-2 Message-ID: <20100702230359.42826282BEF@codespeak.net> Author: afa Date: Sat Jul 3 01:03:57 2010 New Revision: 75808 Added: pypy/branch/unicode_filename-2/ (props changed) - copied from r75807, pypy/trunk/ Log: Another attempt to support unicode filenames From afa at codespeak.net Sat Jul 3 01:17:26 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 3 Jul 2010 01:17:26 +0200 (CEST) Subject: [pypy-svn] r75809 - pypy/branch/unicode_filename-2/pypy/rpython/test Message-ID: <20100702231726.802A6282BEF@codespeak.net> Author: afa Date: Sat Jul 3 01:17:25 2010 New Revision: 75809 Modified: pypy/branch/unicode_filename-2/pypy/rpython/test/test_extfunc.py Log: Put all the tests in a class, and add missing docstrings. Modified: pypy/branch/unicode_filename-2/pypy/rpython/test/test_extfunc.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/test/test_extfunc.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/test/test_extfunc.py Sat Jul 3 01:17:25 2010 @@ -6,150 +6,164 @@ from pypy.annotation.policy import AnnotatorPolicy from pypy.rpython.test.test_llinterp import interpret -def b(x): - return eval("x+40") +class TestExtFuncEntry: -class BTestFuncEntry(ExtFuncEntry): - _about_ = b - name = 'b' - signature_args = [annmodel.SomeInteger()] - signature_result = annmodel.SomeInteger() - -def test_annotation_b(): - def f(): - return b(1) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def test_rtyping_b(): - def f(): - return b(2) - - res = interpret(f, []) - assert res == 42 - -def c(y, x): - yyy - -class CTestFuncEntry(ExtFuncEntry): - _about_ = c - name = 'ccc' - signature_args = [annmodel.SomeInteger()] * 2 - signature_result = annmodel.SomeInteger() - - def lltypeimpl(y, x): - return y + x - lltypeimpl = staticmethod(lltypeimpl) - -def test_interp_c(): - def f(): - return c(3, 4) - - res = interpret(f, []) - assert res == 7 - -def d(y): - return eval("y()") - -class DTestFuncEntry(ExtFuncEntry): - _about_ = d - name = 'd' - signature_args = [annmodel.SomeGenericCallable(args=[], result= - annmodel.SomeFloat())] - signature_result = annmodel.SomeFloat() - -def test_callback(): - def callback(): - return 2.5 - - def f(): - return d(callback) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeFloat) - assert a.translator._graphof(callback) - -def dd(): - pass - -register_external(dd, [int], int) - -def test_register_external_signature(): - def f(): - return dd(3) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - - -def function_with_tuple_arg(): - """ - Dummy function which is declared via register_external to take a tuple as - an argument so that register_external's behavior for tuple-taking functions - can be verified. - """ -register_external(function_with_tuple_arg, [(int,)], int) - -def test_register_external_tuple_args(): - """ - Verify the annotation of a registered external function which takes a tuple - argument. - """ - def f(): - return function_with_tuple_arg((1,)) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - - # Not a very good assertion, but at least it means _something_ happened. - assert isinstance(s, annmodel.SomeInteger) - -def function_with_list(): - pass -register_external(function_with_list, [[int]], int) - -def function_returning_list(): - pass -register_external(function_returning_list, [], [int]) - -def test_register_external_return_goes_back(): - """ - Check whether it works to pass the same list from one external - fun to another - [bookkeeper and list joining issues] - """ - def f(): - return function_with_list(function_returning_list()) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def function_withspecialcase(arg): - return repr(arg) -register_external(function_withspecialcase, args=None, result=str) - -def test_register_external_specialcase(): - def f(): - x = function_withspecialcase - return x(33) + x("aaa") + x([]) + "\n" - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeString) + def test_basic(self): + """ + A ExtFuncEntry provides an annotation for a function, no need to flow + its graph. + """ + def b(x): + "NOT_RPYTHON" + return eval("x+40") + + class BTestFuncEntry(ExtFuncEntry): + _about_ = b + name = 'b' + signature_args = [annmodel.SomeInteger()] + signature_result = annmodel.SomeInteger() + + def f(): + return b(2) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + res = interpret(f, []) + assert res == 42 + + def test_lltypeimpl(self): + """ + interpret() calls lltypeimpl instead of of the function/ + """ + def c(y, x): + yyy + + class CTestFuncEntry(ExtFuncEntry): + _about_ = c + name = 'ccc' + signature_args = [annmodel.SomeInteger()] * 2 + signature_result = annmodel.SomeInteger() + + def lltypeimpl(y, x): + return y + x + lltypeimpl = staticmethod(lltypeimpl) + + def f(): + return c(3, 4) + + res = interpret(f, []) + assert res == 7 + + def test_callback(self): + """ + Verify annotation when a callback function is in the arguments list. + """ + def d(y): + return eval("y()") + + class DTestFuncEntry(ExtFuncEntry): + _about_ = d + name = 'd' + signature_args = [annmodel.SomeGenericCallable(args=[], result= + annmodel.SomeFloat())] + signature_result = annmodel.SomeFloat() + + def callback(): + return 2.5 + + def f(): + return d(callback) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeFloat) + assert a.translator._graphof(callback) + + def test_register_external_signature(self): + """ + Test the standard interface for external functions. + """ + def dd(): + pass + register_external(dd, [int], int) + + def f(): + return dd(3) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_tuple_args(self): + """ + Verify the annotation of a registered external function which takes a + tuple argument. + """ + + def function_with_tuple_arg(): + """ + Dummy function which is declared via register_external to take a + tuple as an argument so that register_external's behavior for + tuple-taking functions can be verified. + """ + register_external(function_with_tuple_arg, [(int,)], int) + + def f(): + return function_with_tuple_arg((1,)) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + + # Not a very good assertion, but at least it means _something_ happened. + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_return_goes_back(self): + """ + Check whether it works to pass the same list from one external + fun to another + [bookkeeper and list joining issues] + """ + def function_with_list(): + pass + register_external(function_with_list, [[int]], int) + + def function_returning_list(): + pass + register_external(function_returning_list, [], [int]) + + def f(): + return function_with_list(function_returning_list()) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_specialcase(self): + """ + When args=None, the external function accepts any arguments unmodified. + """ + def function_withspecialcase(arg): + return repr(arg) + register_external(function_withspecialcase, args=None, result=str) + + def f(): + x = function_withspecialcase + return x(33) + x("aaa") + x([]) + "\n" + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeString) From benjamin at codespeak.net Sat Jul 3 01:34:21 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Jul 2010 01:34:21 +0200 (CEST) Subject: [pypy-svn] r75810 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100702233421.81A05282BEF@codespeak.net> Author: benjamin Date: Sat Jul 3 01:34:19 2010 New Revision: 75810 Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Log: creating float subclasses is a bit different Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floattype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floattype.py Sat Jul 3 01:34:19 2010 @@ -17,13 +17,15 @@ def descr__new__(space, w_floattype, w_x=0.0): from pypy.objspace.std.floatobject import W_FloatObject w_value = w_x # 'x' is the keyword argument name in CPython - w_special = space.lookup(w_x, "__float__") + w_special = space.lookup(w_value, "__float__") if w_special is not None: - w_obj = space.get_and_call_function(w_special, w_x) + w_obj = space.get_and_call_function(w_special, w_value) if not space.isinstance_w(w_obj, space.w_float): raise OperationError(space.w_TypeError, space.wrap("__float__ returned non-float")) - return w_obj + if space.is_w(w_floattype, space.gettypeobject(float_typedef)): + return w_obj + value = space.float_w(w_obj) elif space.is_true(space.isinstance(w_value, space.w_str)): strvalue = space.str_w(w_value) try: Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Sat Jul 3 01:34:19 2010 @@ -132,6 +132,12 @@ raises(OverflowError, float('-inf').as_integer_ratio) raises(ValueError, float('nan').as_integer_ratio) + def test_float_conversion(self): + class X(float): + def __float__(self): + return 42. + assert float(X()) == 42. + def test_round(self): assert 1.0 == round(1.0) assert 1.0 == round(1.1) From benjamin at codespeak.net Sat Jul 3 01:36:37 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Jul 2010 01:36:37 +0200 (CEST) Subject: [pypy-svn] r75811 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100702233637.DE492282BEF@codespeak.net> Author: benjamin Date: Sat Jul 3 01:36:36 2010 New Revision: 75811 Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Log: fix spelling Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Sat Jul 3 01:36:36 2010 @@ -475,15 +475,15 @@ float_part *= 2.0 exp -= 1 w_num = W_LongObject.fromfloat(float_part) - w_dem = space.newlong(1) + w_den = space.newlong(1) w_exp = space.newlong(abs(exp)) - w_exp = space.lshift(w_dem, w_exp) + w_exp = space.lshift(w_den, w_exp) if exp > 0: w_num = space.mul(w_num, w_exp) else: - w_dem = w_exp + w_den = w_exp # Try to return int. - return space.newtuple([space.int(w_num), space.int(w_dem)]) + return space.newtuple([space.int(w_num), space.int(w_den)]) from pypy.objspace.std import floattype register_all(vars(), floattype) From benjamin at codespeak.net Sat Jul 3 01:46:10 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Jul 2010 01:46:10 +0200 (CEST) Subject: [pypy-svn] r75812 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100702234610.F2B2D282BEF@codespeak.net> Author: benjamin Date: Sat Jul 3 01:46:09 2010 New Revision: 75812 Modified: pypy/branch/fast-forward/pypy/objspace/std/inttype.py pypy/branch/fast-forward/pypy/objspace/std/longtype.py pypy/branch/fast-forward/pypy/objspace/std/test/test_intobject.py pypy/branch/fast-forward/pypy/objspace/std/test/test_longobject.py Log: give int and long numerator and denominator attributes Modified: pypy/branch/fast-forward/pypy/objspace/std/inttype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/inttype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/inttype.py Sat Jul 3 01:46:09 2010 @@ -1,4 +1,4 @@ -from pypy.interpreter import gateway +from pypy.interpreter import gateway, typedef from pypy.interpreter.error import OperationError from pypy.objspace.std.stdtypedef import StdTypeDef from pypy.objspace.std.strutil import (string_to_int, string_to_w_long, @@ -134,6 +134,12 @@ W_IntObject.__init__(w_obj, value) return w_obj +def descr_get_numerator(space, w_obj): + return space.int(w_obj) + +def descr_get_denominator(space, w_obj): + return space.wrap(1) + # ____________________________________________________________ int_typedef = StdTypeDef("int", @@ -146,4 +152,6 @@ non-string. If the argument is outside the integer range a long object will be returned instead.''', __new__ = gateway.interp2app(descr__new__), + numerator = typedef.GetSetProperty(descr_get_numerator), + denominator = typedef.GetSetProperty(descr_get_denominator), ) Modified: pypy/branch/fast-forward/pypy/objspace/std/longtype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/longtype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/longtype.py Sat Jul 3 01:46:09 2010 @@ -1,5 +1,5 @@ from pypy.interpreter.error import OperationError -from pypy.interpreter import gateway +from pypy.interpreter import gateway, typedef from pypy.objspace.std.stdtypedef import StdTypeDef from pypy.objspace.std.strutil import string_to_w_long, ParseStringError @@ -65,6 +65,12 @@ W_LongObject.__init__(w_obj, w_value.num) return w_obj +def descr_get_numerator(space, w_obj): + return space.long(w_obj) + +def descr_get_denominator(space, w_obj): + return space.newlong(1) + # ____________________________________________________________ long_typedef = StdTypeDef("long", @@ -76,4 +82,6 @@ string, use the optional base. It is an error to supply a base when converting a non-string.''', __new__ = gateway.interp2app(descr__new__), + numerator = typedef.GetSetProperty(descr_get_numerator), + denominator = typedef.GetSetProperty(descr_get_denominator), ) Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_intobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_intobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_intobject.py Sat Jul 3 01:46:09 2010 @@ -293,6 +293,12 @@ def test_int_callable(self): assert 43 == int(43) + def test_numerator_denominator(self): + assert (1).numerator == 1 + assert (1).denominator == 1 + assert (42).numerator == 42 + assert (42).denominator == 1 + def test_int_string(self): assert 42 == int("42") assert 10000000000 == long("10000000000") Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_longobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_longobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_longobject.py Sat Jul 3 01:46:09 2010 @@ -65,6 +65,12 @@ a = 31415926L // 10000000L assert a == 3L + def test_numerator_denominator(self): + assert (1L).numerator == 1L + assert (1L).denominator == 1L + assert (42L).numerator == 42L + assert (42L).denominator == 1L + def test_compare(self): for BIG in (1L, 1L << 62, 1L << 9999): assert 0 == 0L From benjamin at codespeak.net Sat Jul 3 02:13:37 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Jul 2010 02:13:37 +0200 (CEST) Subject: [pypy-svn] r75813 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100703001337.73F72282BEF@codespeak.net> Author: benjamin Date: Sat Jul 3 02:13:35 2010 New Revision: 75813 Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py Log: simplify Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floattype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floattype.py Sat Jul 3 02:13:35 2010 @@ -23,7 +23,7 @@ if not space.isinstance_w(w_obj, space.w_float): raise OperationError(space.w_TypeError, space.wrap("__float__ returned non-float")) - if space.is_w(w_floattype, space.gettypeobject(float_typedef)): + if space.is_w(w_floattype, space.w_float): return w_obj value = space.float_w(w_obj) elif space.is_true(space.isinstance(w_value, space.w_str)): From fijal at codespeak.net Sat Jul 3 09:58:29 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 3 Jul 2010 09:58:29 +0200 (CEST) Subject: [pypy-svn] r75814 - in pypy/trunk/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20100703075829.D307D282BF6@codespeak.net> Author: fijal Date: Sat Jul 3 09:58:27 2010 New Revision: 75814 Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py pypy/trunk/pypy/jit/backend/llgraph/runner.py pypy/trunk/pypy/jit/metainterp/blackhole.py pypy/trunk/pypy/jit/metainterp/executor.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resoperation.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py Log: Teach frontend how to deal with raw values (and llgraph backend) Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Sat Jul 3 09:58:27 2010 @@ -1199,6 +1199,10 @@ array = array._obj.container return cast_to_int(array.getitem(index)) +def do_getarrayitem_raw_int(array, index): + array = array.adr.ptr._obj + return cast_to_int(array.getitem(index)) + def do_getarrayitem_gc_float(array, index): array = array._obj.container return cast_to_float(array.getitem(index)) @@ -1251,6 +1255,12 @@ newvalue = cast_from_int(ITEMTYPE, newvalue) array.setitem(index, newvalue) +def do_setarrayitem_raw_int(array, index, newvalue): + array = array.adr.ptr + ITEMTYPE = lltype.typeOf(array).TO.OF + newvalue = cast_from_int(ITEMTYPE, newvalue) + array._obj.setitem(index, newvalue) + def do_setarrayitem_gc_float(array, index, newvalue): array = array._obj.container ITEMTYPE = lltype.typeOf(array).OF Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/runner.py Sat Jul 3 09:58:27 2010 @@ -294,7 +294,6 @@ return llimpl.grab_exc_value() def arraydescrof(self, A): - assert isinstance(A, lltype.GcArray) assert A.OF != lltype.Void size = symbolic.get_size(A) token = history.getkind(A.OF) @@ -317,6 +316,9 @@ def bh_getarrayitem_gc_i(self, arraydescr, array, index): assert isinstance(arraydescr, Descr) return llimpl.do_getarrayitem_gc_int(array, index) + def bh_getarrayitem_raw_i(self, arraydescr, array, index): + assert isinstance(arraydescr, Descr) + return llimpl.do_getarrayitem_raw_int(array, index) def bh_getarrayitem_gc_r(self, arraydescr, array, index): assert isinstance(arraydescr, Descr) return llimpl.do_getarrayitem_gc_ptr(array, index) @@ -372,6 +374,10 @@ assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_int(array, index, newvalue) + def bh_setarrayitem_raw_i(self, arraydescr, array, index, newvalue): + assert isinstance(arraydescr, Descr) + llimpl.do_setarrayitem_raw_int(array, index, newvalue) + def bh_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_ptr(array, index, newvalue) Modified: pypy/trunk/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/blackhole.py (original) +++ pypy/trunk/pypy/jit/metainterp/blackhole.py Sat Jul 3 09:58:27 2010 @@ -990,6 +990,10 @@ bhimpl_getarrayitem_gc_pure_r = bhimpl_getarrayitem_gc_r bhimpl_getarrayitem_gc_pure_f = bhimpl_getarrayitem_gc_f + @arguments("cpu", "i", "d", "i", returns="i") + def bhimpl_getarrayitem_raw_i(cpu, array, arraydescr, index): + return cpu.bh_getarrayitem_raw_i(arraydescr, array, index) + @arguments("cpu", "r", "d", "i", "i") def bhimpl_setarrayitem_gc_i(cpu, array, arraydescr, index, newvalue): cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue) @@ -1000,6 +1004,12 @@ def bhimpl_setarrayitem_gc_f(cpu, array, arraydescr, index, newvalue): cpu.bh_setarrayitem_gc_f(arraydescr, array, index, newvalue) + @arguments("cpu", "i", "d", "i", "i") + def bhimpl_setarrayitem_raw_i(cpu, array, arraydescr, index, newvalue): + cpu.bh_setarrayitem_raw_i(arraydescr, array, index, newvalue) + + # note, there is no 'r' here, since it can't happen + @arguments("cpu", "r", "d", returns="i") def bhimpl_arraylen_gc(cpu, array, arraydescr): return cpu.bh_arraylen_gc(arraydescr, array) Modified: pypy/trunk/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/executor.py (original) +++ pypy/trunk/pypy/jit/metainterp/executor.py Sat Jul 3 09:58:27 2010 @@ -90,6 +90,15 @@ else: return BoxInt(cpu.bh_getarrayitem_gc_i(arraydescr, array, index)) +def do_getarrayitem_raw(cpu, _, arraybox, indexbox, arraydescr): + array = arraybox.getint() + index = indexbox.getint() + assert not arraydescr.is_array_of_pointers() + if arraydescr.is_array_of_floats(): + return BoxFloat(cpu.bh_getarrayitem_raw_f(arraydescr, array, index)) + else: + return BoxInt(cpu.bh_getarrayitem_raw_i(arraydescr, array, index)) + def do_setarrayitem_gc(cpu, _, arraybox, indexbox, itembox, arraydescr): array = arraybox.getref_base() index = indexbox.getint() @@ -101,6 +110,15 @@ else: cpu.bh_setarrayitem_gc_i(arraydescr, array, index, itembox.getint()) +def do_setarrayitem_raw(cpu, _, arraybox, indexbox, itembox, arraydescr): + array = arraybox.getint() + index = indexbox.getint() + assert not arraydescr.is_array_of_pointers() + if arraydescr.is_array_of_floats(): + cpu.bh_setarrayitem_raw_f(arraydescr, array, index, itembox.getfloat()) + else: + cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint()) + def do_getfield_gc(cpu, _, structbox, fielddescr): struct = structbox.getref_base() if fielddescr.is_pointer_field(): Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Sat Jul 3 09:58:27 2010 @@ -378,6 +378,14 @@ opimpl_getarrayitem_gc_f = _opimpl_getarrayitem_gc_any @arguments("box", "descr", "box") + def _opimpl_getarrayitem_raw_any(self, arraybox, arraydescr, indexbox): + return self.execute_with_descr(rop.GETARRAYITEM_RAW, + arraydescr, arraybox, indexbox) + + opimpl_getarrayitem_raw_i = _opimpl_getarrayitem_raw_any + opimpl_getarrayitem_raw_f = _opimpl_getarrayitem_raw_any + + @arguments("box", "descr", "box") def _opimpl_getarrayitem_gc_pure_any(self, arraybox, arraydescr, indexbox): return self.execute_with_descr(rop.GETARRAYITEM_GC_PURE, arraydescr, arraybox, indexbox) @@ -396,6 +404,15 @@ opimpl_setarrayitem_gc_r = _opimpl_setarrayitem_gc_any opimpl_setarrayitem_gc_f = _opimpl_setarrayitem_gc_any + @arguments("box", "descr", "box", "box") + def _opimpl_setarrayitem_raw_any(self, arraybox, arraydescr, + indexbox, itembox): + self.execute_with_descr(rop.SETARRAYITEM_RAW, arraydescr, arraybox, + indexbox, itembox) + + opimpl_setarrayitem_raw_i = _opimpl_setarrayitem_raw_any + opimpl_setarrayitem_raw_f = _opimpl_setarrayitem_raw_any + @arguments("box", "descr") def opimpl_arraylen_gc(self, arraybox, arraydescr): return self.execute_with_descr(rop.ARRAYLEN_GC, arraydescr, arraybox) Modified: pypy/trunk/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resoperation.py (original) +++ pypy/trunk/pypy/jit/metainterp/resoperation.py Sat Jul 3 09:58:27 2010 @@ -199,6 +199,7 @@ '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- 'GETARRAYITEM_GC/2d', + 'GETARRAYITEM_RAW/2d', 'GETFIELD_GC/1d', 'GETFIELD_RAW/1d', 'NEW/0d', @@ -209,7 +210,7 @@ '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'SETARRAYITEM_GC/3d', - 'SETARRAYITEM_RAW/3d',#only added by backend.llsupport.gc.rewrite_assembler + 'SETARRAYITEM_RAW/3d', 'SETFIELD_GC/2d', 'SETFIELD_RAW/2d', 'ARRAYCOPY/7d', # removed before it's passed to the backend Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Sat Jul 3 09:58:27 2010 @@ -1562,6 +1562,21 @@ res = self.interp_operations(f, [3, 2]) assert res == 1 + def test_raw_malloc_and_access(self): + from pypy.rpython.lltypesystem import rffi + + TP = rffi.CArray(lltype.Signed) + + def f(n): + a = lltype.malloc(TP, n, flavor='raw') + a[0] = n + res = a[0] + lltype.free(a, flavor='raw') + return res + + res = self.interp_operations(f, [10]) + assert res == 10 + class TestOOtype(BasicTests, OOJitMixin): From arigo at codespeak.net Sat Jul 3 10:00:29 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 3 Jul 2010 10:00:29 +0200 (CEST) Subject: [pypy-svn] r75815 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100703080029.DDA62282BF6@codespeak.net> Author: arigo Date: Sat Jul 3 10:00:28 2010 New Revision: 75815 Added: pypy/branch/rsre2/pypy/rlib/rsre/__init__.py (contents, props changed) pypy/branch/rsre2/pypy/rlib/rsre/test/test_external.py (contents, props changed) Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py Log: Port CPython 2.5's re_tests. Added: pypy/branch/rsre2/pypy/rlib/rsre/__init__.py ============================================================================== Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Sat Jul 3 10:00:28 2010 @@ -1,5 +1,5 @@ from pypy.rlib.debug import check_nonneg -import rsre_char +from pypy.rlib.rsre import rsre_char OPCODE_FAILURE = 0 @@ -42,10 +42,11 @@ match_marks = None match_marks_flat = None - def __init__(self, pattern, string, flags): + def __init__(self, pattern, string, match_start, flags): self.pattern = pattern self.string = string self.end = len(string) + self.match_start = match_start self.flags = flags def pat(self, index): @@ -78,6 +79,17 @@ self.match_marks = None # clear return self.match_marks_flat + def span(self, groupnum=0): + # compatibility + lst = self.flatten_marks()[groupnum*2:groupnum*2+2] + if not lst: raise IndexError + return tuple(lst) + + def group(self, group=0): + # compatibility + frm, to = self.span(group) + return self.string[frm:to] + class Mark(object): _immutable_ = True @@ -95,9 +107,9 @@ return -1 -def match(pattern, string, flags=0): - ctx = MatchContext(pattern, string, flags) - if sre_match(ctx, 0, 0, None): +def match(pattern, string, start=0, flags=0): + ctx = MatchContext(pattern, string, start, flags) + if sre_match(ctx, 0, start, None): return ctx return None Added: pypy/branch/rsre2/pypy/rlib/rsre/test/test_external.py ============================================================================== --- (empty file) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_external.py Sat Jul 3 10:00:28 2010 @@ -0,0 +1,66 @@ +import re +from test.re_tests import SUCCEED, FAIL, SYNTAX_ERROR +from pypy.rlib.rsre.test.test_match import get_code +from pypy.rlib.rsre import rsre + + +def test_external(): + from test.re_tests import tests + for t in tests: + yield run_external, t + +def run_external(t): + pattern, s, outcome = t[:3] + if len(t) == 5: + repl, expected = t[3:5] + else: + assert len(t) == 3 + print 'trying:', t + try: + obj = get_code(pattern) + except re.error: + if outcome == SYNTAX_ERROR: + return # Expected a syntax error + raise + if outcome == SYNTAX_ERROR: + raise Exception("this should have been a syntax error") + # + # Emulate a poor man's search() with repeated match()s + for i in range(len(s)+1): + result = rsre.match(obj, s, start=i) + if result: + break + # + if outcome == FAIL: + if result is not None: + raise Exception("succeeded incorrectly") + elif outcome == SUCCEED: + if result is None: + raise Exception("failed incorrectly") + # Matched, as expected, so now we compute the + # result string and compare it to our expected result. + start, end = result.span(0) + vardict={'found': result.group(0), + 'groups': result.group(), + }#'flags': result.re.flags} + for i in range(1, 100): + try: + gi = result.group(i) + # Special hack because else the string concat fails: + if gi is None: + gi = "None" + except IndexError: + gi = "Error" + vardict['g%d' % i] = gi + #for i in result.re.groupindex.keys(): + # try: + # gi = result.group(i) + # if gi is None: + # gi = "None" + # except IndexError: + # gi = "Error" + # vardict[i] = gi + repl = eval(repl, vardict) + if repl != expected: + raise Exception("grouping error: %r should be %r" % (repr, + expected)) Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Sat Jul 3 10:00:28 2010 @@ -1,5 +1,5 @@ import _sre, re, sre_compile -import rsre +from pypy.rlib.rsre import rsre def get_code(regexp): Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py Sat Jul 3 10:00:28 2010 @@ -1,5 +1,5 @@ -import rsre -from test_match import get_code, get_code_and_re +from pypy.rlib.rsre import rsre +from pypy.rlib.rsre.test.test_match import get_code, get_code_and_re class TestSearch: From arigo at codespeak.net Sat Jul 3 10:02:58 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 3 Jul 2010 10:02:58 +0200 (CEST) Subject: [pypy-svn] r75816 - pypy/branch/rsre2/pypy/rlib/rsre/test Message-ID: <20100703080258.E02A3282BF6@codespeak.net> Author: arigo Date: Sat Jul 3 10:02:57 2010 New Revision: 75816 Added: pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py - copied unchanged from r75813, pypy/trunk/lib-python/2.5.2/test/re_tests.py Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_external.py Log: Move the CPython file here. Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_external.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_external.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_external.py Sat Jul 3 10:02:57 2010 @@ -1,15 +1,15 @@ import re -from test.re_tests import SUCCEED, FAIL, SYNTAX_ERROR from pypy.rlib.rsre.test.test_match import get_code from pypy.rlib.rsre import rsre def test_external(): - from test.re_tests import tests + from pypy.rlib.rsre.test.re_tests import tests for t in tests: yield run_external, t def run_external(t): + from pypy.rlib.rsre.test.re_tests import SUCCEED, FAIL, SYNTAX_ERROR pattern, s, outcome = t[:3] if len(t) == 5: repl, expected = t[3:5] From fijal at codespeak.net Sat Jul 3 10:03:07 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 3 Jul 2010 10:03:07 +0200 (CEST) Subject: [pypy-svn] r75817 - in pypy/trunk/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20100703080307.60231282BFC@codespeak.net> Author: fijal Date: Sat Jul 3 10:03:05 2010 New Revision: 75817 Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py pypy/trunk/pypy/jit/backend/llgraph/runner.py pypy/trunk/pypy/jit/metainterp/blackhole.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py Log: Teach about floats Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Sat Jul 3 10:03:05 2010 @@ -1207,6 +1207,10 @@ array = array._obj.container return cast_to_float(array.getitem(index)) +def do_getarrayitem_raw_float(array, index): + array = array.adr.ptr._obj + return cast_to_float(array.getitem(index)) + def do_getarrayitem_gc_ptr(array, index): array = array._obj.container return cast_to_ptr(array.getitem(index)) @@ -1267,6 +1271,12 @@ newvalue = cast_from_float(ITEMTYPE, newvalue) array.setitem(index, newvalue) +def do_setarrayitem_raw_float(array, index, newvalue): + array = array.adr.ptr + ITEMTYPE = lltype.typeOf(array).TO.OF + newvalue = cast_from_int(ITEMTYPE, newvalue) + array._obj.setitem(index, newvalue) + def do_setarrayitem_gc_ptr(array, index, newvalue): array = array._obj.container ITEMTYPE = lltype.typeOf(array).OF Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/runner.py Sat Jul 3 10:03:05 2010 @@ -325,6 +325,9 @@ def bh_getarrayitem_gc_f(self, arraydescr, array, index): assert isinstance(arraydescr, Descr) return llimpl.do_getarrayitem_gc_float(array, index) + def bh_getarrayitem_raw_f(self, arraydescr, array, index): + assert isinstance(arraydescr, Descr) + return llimpl.do_getarrayitem_raw_float(array, index) def bh_getfield_gc_i(self, struct, fielddescr): assert isinstance(fielddescr, Descr) @@ -386,6 +389,10 @@ assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_float(array, index, newvalue) + def bh_setarrayitem_raw_f(self, arraydescr, array, index, newvalue): + assert isinstance(arraydescr, Descr) + llimpl.do_setarrayitem_raw_float(array, index, newvalue) + def bh_setfield_gc_i(self, struct, fielddescr, newvalue): assert isinstance(fielddescr, Descr) llimpl.do_setfield_gc_int(struct, fielddescr.ofs, newvalue) Modified: pypy/trunk/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/blackhole.py (original) +++ pypy/trunk/pypy/jit/metainterp/blackhole.py Sat Jul 3 10:03:05 2010 @@ -993,6 +993,9 @@ @arguments("cpu", "i", "d", "i", returns="i") def bhimpl_getarrayitem_raw_i(cpu, array, arraydescr, index): return cpu.bh_getarrayitem_raw_i(arraydescr, array, index) + @arguments("cpu", "i", "d", "i", returns="f") + def bhimpl_getarrayitem_raw_f(cpu, array, arraydescr, index): + return cpu.bh_getarrayitem_raw_f(arraydescr, array, index) @arguments("cpu", "r", "d", "i", "i") def bhimpl_setarrayitem_gc_i(cpu, array, arraydescr, index, newvalue): @@ -1007,6 +1010,9 @@ @arguments("cpu", "i", "d", "i", "i") def bhimpl_setarrayitem_raw_i(cpu, array, arraydescr, index, newvalue): cpu.bh_setarrayitem_raw_i(arraydescr, array, index, newvalue) + @arguments("cpu", "i", "d", "i", "f") + def bhimpl_setarrayitem_raw_f(cpu, array, arraydescr, index, newvalue): + cpu.bh_setarrayitem_raw_f(arraydescr, array, index, newvalue) # note, there is no 'r' here, since it can't happen Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Sat Jul 3 10:03:05 2010 @@ -1577,6 +1577,20 @@ res = self.interp_operations(f, [10]) assert res == 10 + def test_raw_malloc_and_access_float(self): + from pypy.rpython.lltypesystem import rffi + + TP = rffi.CArray(lltype.Float) + + def f(n, f): + a = lltype.malloc(TP, n, flavor='raw') + a[0] = f + res = a[0] + lltype.free(a, flavor='raw') + return res + + res = self.interp_operations(f, [10, 3.5]) + assert res == 3.5 class TestOOtype(BasicTests, OOJitMixin): From arigo at codespeak.net Sat Jul 3 10:34:36 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 3 Jul 2010 10:34:36 +0200 (CEST) Subject: [pypy-svn] r75818 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100703083436.B2C9A282BFC@codespeak.net> Author: arigo Date: Sat Jul 3 10:34:35 2010 New Revision: 75818 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_external.py Log: First fixes found by test_external. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Sat Jul 3 10:34:35 2010 @@ -82,12 +82,13 @@ def span(self, groupnum=0): # compatibility lst = self.flatten_marks()[groupnum*2:groupnum*2+2] - if not lst: raise IndexError - return tuple(lst) + return tuple(lst) or (-1, -1) def group(self, group=0): # compatibility frm, to = self.span(group) + if frm < 0 or to < frm: + return None return self.string[frm:to] @@ -492,16 +493,16 @@ if ctx.end == 0: return False prevptr = ptr - 1 - that = prevptr >= 0 and is_word(ctx.str(prevptr)) - this = ptr < ctx.end and is_word(ctx.str(ptr)) + that = prevptr >= 0 and rsre_char.is_word(ctx.str(prevptr)) + this = ptr < ctx.end and rsre_char.is_word(ctx.str(ptr)) return this != that elif atcode == AT_NON_BOUNDARY: if ctx.end == 0: return False prevptr = ptr - 1 - that = prevptr >= 0 and is_word(ctx.str(prevptr)) - this = ptr < ctx.end and is_word(ctx.str(ptr)) + that = prevptr >= 0 and rsre_char.is_word(ctx.str(prevptr)) + this = ptr < ctx.end and rsre_char.is_word(ctx.str(ptr)) return this == that elif atcode == AT_END: Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py Sat Jul 3 10:34:35 2010 @@ -188,7 +188,7 @@ ('ab|cd', 'abcd', SUCCEED, 'found', 'ab'), ('()ef', 'def', SUCCEED, 'found+"-"+g1', 'ef-'), ('$b', 'b', FAIL), - ('a\\(b', 'a(b', SUCCEED, 'found+"-"+g1', 'a(b-Error'), + #('a\\(b', 'a(b', SUCCEED, 'found+"-"+g1', 'a(b-Error'), ('a\\(*b', 'ab', SUCCEED, 'found', 'ab'), ('a\\(*b', 'a((b', SUCCEED, 'found', 'a((b'), ('a\\\\b', 'a\\b', SUCCEED, 'found', 'a\\b'), @@ -341,7 +341,7 @@ ('(*)b', '-', SYNTAX_ERROR), ('$b', 'b', FAIL), ('a\\', '-', SYNTAX_ERROR), - ('a\\(b', 'a(b', SUCCEED, 'found+"-"+g1', 'a(b-Error'), + #('a\\(b', 'a(b', SUCCEED, 'found+"-"+g1', 'a(b-Error'), ('a\\(*b', 'ab', SUCCEED, 'found', 'ab'), ('a\\(*b', 'a((b', SUCCEED, 'found', 'a((b'), ('a\\\\b', 'a\\b', SUCCEED, 'found', 'a\\b'), @@ -469,7 +469,7 @@ ('(?i)(*)b', '-', SYNTAX_ERROR), ('(?i)$b', 'B', FAIL), ('(?i)a\\', '-', SYNTAX_ERROR), - ('(?i)a\\(b', 'A(B', SUCCEED, 'found+"-"+g1', 'A(B-Error'), + #('(?i)a\\(b', 'A(B', SUCCEED, 'found+"-"+g1', 'A(B-Error'), ('(?i)a\\(*b', 'AB', SUCCEED, 'found', 'AB'), ('(?i)a\\(*b', 'A((B', SUCCEED, 'found', 'A((B'), ('(?i)a\\\\b', 'A\\B', SUCCEED, 'found', 'A\\B'), Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_external.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_external.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_external.py Sat Jul 3 10:34:35 2010 @@ -62,5 +62,5 @@ # vardict[i] = gi repl = eval(repl, vardict) if repl != expected: - raise Exception("grouping error: %r should be %r" % (repr, + raise Exception("grouping error: %r should be %r" % (repl, expected)) From arigo at codespeak.net Sat Jul 3 10:35:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 3 Jul 2010 10:35:00 +0200 (CEST) Subject: [pypy-svn] r75819 - pypy/branch/rsre2/pypy/rlib/rsre/test Message-ID: <20100703083500.2009D282BFC@codespeak.net> Author: arigo Date: Sat Jul 3 10:34:58 2010 New Revision: 75819 Added: pypy/branch/rsre2/pypy/rlib/rsre/test/test_zexternal.py - copied unchanged from r75818, pypy/branch/rsre2/pypy/rlib/rsre/test/test_external.py Removed: pypy/branch/rsre2/pypy/rlib/rsre/test/test_external.py Log: Rename to test_z* to have it run last. From benjamin at codespeak.net Sat Jul 3 17:02:59 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Jul 2010 17:02:59 +0200 (CEST) Subject: [pypy-svn] r75820 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100703150259.CF291282BEF@codespeak.net> Author: benjamin Date: Sat Jul 3 17:02:57 2010 New Revision: 75820 Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Log: work around < 2.7's imperfect math.pow implementation Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Sat Jul 3 17:02:57 2010 @@ -1,4 +1,5 @@ -import operator, new +import operator +import sys from pypy.interpreter import gateway from pypy.interpreter.error import OperationError from pypy.objspace.std import model, newformat @@ -442,6 +443,11 @@ "a negative power")) raise OperationError(space.w_ValueError, space.wrap("float power")) + # Should the result be negated? + if (sys.version_info < (2, 7) and z == 0.0 and x < 0.0 and + not isinf(x) and not isinf(y) and + math.fmod(abs(y), 2.0) == 1.0): + z = -z return W_FloatObject(z) Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Sat Jul 3 17:02:57 2010 @@ -185,10 +185,14 @@ assert pw(0.0, float("-inf")) == float("inf") def test_pow_neg_base(self): + import math def pw(x, y): return x ** y assert pw(-2.0, 2.0) == 4 - + res = pw(-2.0, -2001.0) + assert res == -0.0 + assert math.copysign(1., res) == -1. + def test_float_cmp(self): assert 12.5 == 12.5 assert 12.5 != -3.2 From benjamin at codespeak.net Sat Jul 3 17:11:25 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Jul 2010 17:11:25 +0200 (CEST) Subject: [pypy-svn] r75821 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100703151125.BD48E282BEF@codespeak.net> Author: benjamin Date: Sat Jul 3 17:11:24 2010 New Revision: 75821 Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Log: protect from being translated Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Sat Jul 3 17:11:24 2010 @@ -12,6 +12,7 @@ from pypy.rlib.rarithmetic import ovfcheck_float_to_int, intmask, isinf, isnan from pypy.rlib.rarithmetic import formatd, LONG_BIT, FL_MAXINT, FL_MININT from pypy.rlib.rbigint import rbigint +from pypy.rlib.objectmodel import we_are_translated from pypy.tool.sourcetools import func_with_new_name import math @@ -444,7 +445,8 @@ raise OperationError(space.w_ValueError, space.wrap("float power")) # Should the result be negated? - if (sys.version_info < (2, 7) and z == 0.0 and x < 0.0 and + if (not we_are_translated() and sys.version_info < (2, 7) and + z == 0.0 and x < 0.0 and not isinf(x) and not isinf(y) and math.fmod(abs(y), 2.0) == 1.0): z = -z From afa at codespeak.net Sat Jul 3 23:16:02 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 3 Jul 2010 23:16:02 +0200 (CEST) Subject: [pypy-svn] r75822 - pypy/branch/unicode_filename-2/pypy/rpython Message-ID: <20100703211602.E4857282BEF@codespeak.net> Author: afa Date: Sat Jul 3 23:16:00 2010 New Revision: 75822 Modified: pypy/branch/unicode_filename-2/pypy/rpython/extfunc.py Log: This is no more used nor tested, remove. (A remnant of the old extfunctable) Modified: pypy/branch/unicode_filename-2/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/extfunc.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/extfunc.py Sat Jul 3 23:16:00 2010 @@ -167,8 +167,6 @@ return signature_args def compute_result_annotation(self, *args_s): - if hasattr(self, 'ann_hook'): - self.ann_hook() self.normalize_args(*args_s) # check arguments return self.signature_result @@ -235,7 +233,6 @@ def register_external(function, args, result=None, export_name=None, llimpl=None, ooimpl=None, llfakeimpl=None, oofakeimpl=None, - annotation_hook=None, sandboxsafe=False): """ function: the RPython function that will be rendered as an external function (e.g.: math.floor) @@ -244,7 +241,6 @@ export_name: the name of the function as it will be seen by the backends llimpl, ooimpl: optional; if provided, these RPython functions are called instead of the target function llfakeimpl, oofakeimpl: optional; if provided, they are called by the llinterpreter - annotationhook: optional; a callable that is called during annotation, useful for genc hacks sandboxsafe: use True if the function performs no I/O (safe for --sandbox) """ @@ -271,8 +267,6 @@ lltypefakeimpl = staticmethod(llfakeimpl) if oofakeimpl: ootypefakeimpl = staticmethod(oofakeimpl) - if annotation_hook: - ann_hook = staticmethod(annotation_hook) if export_name: FunEntry.__name__ = export_name From hakanardo at codespeak.net Sun Jul 4 21:06:22 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sun, 4 Jul 2010 21:06:22 +0200 (CEST) Subject: [pypy-svn] r75824 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100704190622.4F935282B9D@codespeak.net> Author: hakanardo Date: Sun Jul 4 21:05:58 2010 New Revision: 75824 Added: pypy/branch/interplevel-array/pypy/module/array/interp_array_multiclass.py pypy/branch/interplevel-array/pypy/module/array/test/imtst.py pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: Array constructor in place except for fromXXX initiators. Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/__init__.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/__init__.py Sun Jul 4 21:05:58 2010 @@ -7,6 +7,7 @@ } interpleveldefs = { - 'array' : 'interp_array.array', + 'array' : 'interp_array.array', + 'sized_array' : 'interp_array.sized_array', } Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Sun Jul 4 21:05:58 2010 @@ -1,13 +1,14 @@ from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import TypeDef from pypy.rpython.lltypesystem import lltype, rffi -from pypy.interpreter.gateway import interp2app, ObjSpace +from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root from pypy.rlib.jit import dont_look_inside from pypy.rlib import rgc FloatArray=lltype.GcArray(lltype.Float) -class W_Array(Wrappable): +class W_SizedFloatArray(Wrappable): @dont_look_inside def __init__(self, space, size): self.space=space @@ -30,15 +31,183 @@ self.buffer[idx]=val descr_setitem.unwrap_spec = ['self', int, float] -W_Array.typedef = TypeDef( +W_SizedFloatArray.typedef = TypeDef( + 'SizedFloatArray', + __getitem__ = interp2app(W_SizedFloatArray.descr_getitem), + __setitem__ = interp2app(W_SizedFloatArray.descr_setitem), +) + +def sized_array(space,size): + return W_SizedFloatArray(space, size) +sized_array.unwrap_spec=(ObjSpace, int) + +types = { + 'c': lltype.GcArray(lltype.Char), + 'u': lltype.GcArray(lltype.UniChar), + 'b': lltype.GcArray(rffi.SIGNEDCHAR), + 'B': lltype.GcArray(rffi.UCHAR), + 'h': lltype.GcArray(rffi.SHORT), + 'H': lltype.GcArray(rffi.USHORT), + 'i': lltype.GcArray(lltype.Signed), + 'I': lltype.GcArray(lltype.Unsigned), + 'l': lltype.GcArray(lltype.Signed), + 'L': lltype.GcArray(lltype.Unsigned), + 'f': lltype.GcArray(lltype.SingleFloat), + 'd': lltype.GcArray(lltype.Float), + } +## def array(space, typecode, w_initializer=None): +## if +## self.len=0 +## self.buffer=None +## if w_initializer is not None and not space.is_w(w_size, space.w_None): +## self.extend(w_initializer) #FIXME: use fromlist, fromstring, ... + +## return W_SizedFloatArray(space, size) + +class W_Array(Wrappable): + def __init__(self, space, typecode): + if len(typecode) != 1: + msg='array() argument 1 must be char, not str' + raise OperationError(space.w_TypeError,space.wrap(msg)) + if typecode not in 'cbBuhHiIlLfd': + msg='bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' + raise OperationError(space.w_ValueError,space.wrap(msg)) + self.space=space + self.typecode=typecode + self.len=0 + self.buffer=None + + def item_w(self, w_item): + space=self.space + if self.typecode == 'c': + return self.space.str_w(w_item) + elif self.typecode == 'u': + return self.space.unicode_w(w_item) + + elif self.typecode == 'b': + item=self.space.int_w(w_item) + if item<-128: + msg='signed char is less than minimum' + raise OperationError(space.w_OverflowError, space.wrap(msg)) + elif item>127: + msg='signed char is greater than maximum' + raise OperationError(space.w_OverflowError, space.wrap(msg)) + return rffi.cast(rffi.SIGNEDCHAR, item) + elif self.typecode == 'B': + item=self.space.int_w(w_item) + if item<0: + msg='unsigned byte integer is less than minimum' + raise OperationError(space.w_OverflowError, space.wrap(msg)) + elif item>255: + msg='unsigned byte integer is greater than maximum' + raise OperationError(space.w_OverflowError, space.wrap(msg)) + return rffi.cast(rffi.UCHAR, item) + + elif self.typecode == 'h': + item=self.space.int_w(w_item) + if item<-32768: + msg='signed short integer is less than minimum' + raise OperationError(space.w_OverflowError, space.wrap(msg)) + elif item>32767: + msg='signed short integer is greater than maximum' + raise OperationError(space.w_OverflowError, space.wrap(msg)) + return rffi.cast(rffi.SHORT, item) + elif self.typecode == 'H': + item=self.space.int_w(w_item) + if item<0: + msg='unsigned short integer is less than minimum' + raise OperationError(space.w_OverflowError, space.wrap(msg)) + elif item>65535: + msg='unsigned short integer is greater than maximum' + raise OperationError(space.w_OverflowError, space.wrap(msg)) + return rffi.cast(rffi.USHORT, item) + + elif self.typecode in ('i', 'l'): + item=self.space.int_w(w_item) + if item<-2147483648: + msg='signed integer is less than minimum' + raise OperationError(space.w_OverflowError, space.wrap(msg)) + elif item>2147483647: + msg='signed integer is greater than maximum' + raise OperationError(space.w_OverflowError, space.wrap(msg)) + return rffi.cast(lltype.Signed, item) + elif self.typecode in ('I', 'L'): + item=self.space.int_w(w_item) + if item<0: + msg='unsigned integer is less than minimum' + raise OperationError(space.w_OverflowError, space.wrap(msg)) + elif item>4294967295: + msg='unsigned integer is greater than maximum' + raise OperationError(space.w_OverflowError, space.wrap(msg)) + return rffi.cast(lltype.Unsigned, item) + + elif self.typecode == 'f': + item=self.space.float_w(w_item) + return rffi.cast(lltype.SingleFloat, item) + elif self.typecode == 'd': + return self.space.float_w(w_item) + + def setlen(self, size): + new_buffer=lltype.malloc(types[self.typecode], size, zero=True) + for i in range(self.len): + new_buffer[i]=self.buffer[i] + self.buffer=new_buffer + self.len=size + + def descr_append(self,w_x): + x=self.item_w(w_x) + self.setlen(self.len+1) + self.buffer[self.len-1]=x + descr_append.unwrap_spec = ['self', W_Root] + + def descr_extend(self, w_initializer): + space = self.space + w_iterator = space.iter(w_initializer) + while True: + try: + w_item = space.next(w_iterator) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break + self.descr_append(w_item) + descr_extend.unwrap_spec = ['self', W_Root] + + + def descr_getitem(self, idx): + item = self.buffer[idx] + if self.typecode in ('b', 'B', 'h', 'H', 'i', 'l'): + item=rffi.cast(lltype.Signed, item) + if self.typecode == 'f': + item=float(item) + return self.space.wrap(item) + descr_getitem.unwrap_spec = ['self', int] + + def descr_setitem(self, idx, w_item): + item=self.item_w(w_item) + self.buffer[idx]=item + descr_setitem.unwrap_spec = ['self', int, W_Root] + + def descr_len(self): + return self.space.wrap(self.len) + descr_len.unwrap_spec = ['self'] + +W_Array.typedef=TypeDef( 'Array', + append = interp2app(W_Array.descr_append), + extend = interp2app(W_Array.descr_extend), + __len__ = interp2app(W_Array.descr_len), __getitem__ = interp2app(W_Array.descr_getitem), __setitem__ = interp2app(W_Array.descr_setitem), ) - - +def array(space, typecode, w_initializer=None): + a=W_Array(space, typecode) + if w_initializer is not None: + if not space.is_w(w_initializer, space.w_None): + a.descr_extend(w_initializer) #FIXME: use fromlist, fromstring, ... -def array(space,size): - return W_Array(space, size) -array.unwrap_spec=(ObjSpace, int) + return a + +array.unwrap_spec=(ObjSpace, str, W_Root) + Added: pypy/branch/interplevel-array/pypy/module/array/interp_array_multiclass.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array_multiclass.py Sun Jul 4 21:05:58 2010 @@ -0,0 +1,175 @@ +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.error import OperationError +from pypy.interpreter.typedef import TypeDef +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root +from pypy.rlib.jit import dont_look_inside +from pypy.rlib import rgc + +FloatArray=lltype.GcArray(lltype.Float) + +class W_SizedFloatArray(Wrappable): + @dont_look_inside + def __init__(self, space, size): + self.space=space + self.size=size + print "malloc" + #self.buffer = lltype.malloc(rffi.DOUBLEP.TO, size, flavor='raw', zero=True) + #self.buffer = rgc.malloc_nonmovable(lltype.GcArray(lltype.Float), size) + self.buffer = lltype.malloc(FloatArray, size, zero=True) + print "buf: ", self.buffer + + def __del__(self): + print "free" + #lltype.free(self.buffer, flavor='raw') + + def descr_getitem(self, idx): + return self.space.wrap(self.buffer[idx]) + descr_getitem.unwrap_spec = ['self', int] + + def descr_setitem(self, idx, val): + self.buffer[idx]=val + descr_setitem.unwrap_spec = ['self', int, float] + +W_SizedFloatArray.typedef = TypeDef( + 'SizedFloatArray', + __getitem__ = interp2app(W_SizedFloatArray.descr_getitem), + __setitem__ = interp2app(W_SizedFloatArray.descr_setitem), +) + +def sized_array(space,size): + return W_SizedFloatArray(space, size) +sized_array.unwrap_spec=(ObjSpace, int) + +types = { + 'c': lltype.GcArray(lltype.Char), + 'u': lltype.GcArray(lltype.UniChar), + 'f': lltype.GcArray(lltype.SingleFloat), + 'd': lltype.GcArray(lltype.Float), + } +## def array(space, typecode, w_initializer=None): +## if +## self.len=0 +## self.buffer=None +## if w_initializer is not None and not space.is_w(w_size, space.w_None): +## self.extend(w_initializer) #FIXME: use fromlist, fromstring, ... + +## return W_SizedFloatArray(space, size) + +class W_Array(Wrappable): + def __init__(self, space, typecode): + self.space=space + self.typecode=typecode + self.len=0 + self.buffer=None + + def item_w(self, w_item): + if self.typecode == 'c': + return self.space.str_w(w_item) + elif self.typecode == 'u': + self.space.unicode_w(w_item) + elif self.typecode in ('b', 'B', 'h', 'H', 'i', 'l'): + self.space.int_w(w_item) + elif self.typecode in ('I', 'L'): + self.space.long_w(w_item) + elif self.typecode in ('f', 'd'): + self.space.float_w(w_item) + + + def setlen(self, size): + new_buffer=lltype.malloc(types[self.typecode], size, zero=True) + for i in range(self.len): + new_buffer[i]=self.buffer[i] + self.buffer=new_buffer + self.len=size + + def descr_extend(self, w_initializer): + space = self.space + w_iterator = space.iter(w_initializer) + while True: + try: + w_item = space.next(w_iterator) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break + print w_item + self.descr_append(space.str_w(w_item)) + descr_extend.unwrap_spec = ['self', W_Root] + + + def descr_getitem(self, idx): + return self.space.wrap(self.buffer[idx]) + descr_getitem.unwrap_spec = ['self', int] + +class W_CharArray(W_Array): + def descr_append(self,x): + print x + self.setlen(self.len+1) + self.buffer[self.len-1]=x + descr_append.unwrap_spec = ['self', str] + + def descr_setitem(self, idx, val): + self.buffer[idx]=val + descr_setitem.unwrap_spec = ['self', int, float] + + +class W_UnicodeArray(W_Array): + def descr_append(self, x): + self.setlen(self.len+1) + self.buffer[self.len-1]=x + descr_append.unwrap_spec = ['self', unicode] + +class W_IntArray(W_Array): + def descr_append(): + pass + descr_append.unwrap_spec = ['self', int] + +class W_LongArray(W_Array): + def descr_append(): + pass + descr_append.unwrap_spec = ['self', long] + +class W_FloatArray(W_Array): + def descr_append(): + pass + descr_append.unwrap_spec = ['self', float] + +def mktypedef(name, cls): + methods={ + 'append': interp2app(cls.descr_append), + 'extend': interp2app(cls.descr_extend), + '__getitem__': interp2app(cls.descr_getitem), + } + cls.typedef = TypeDef(name, **methods) + +mktypedef('CharArray', W_CharArray) +mktypedef('UnicodeArray', W_UnicodeArray) + + +def array(space, typecode, w_initializer=None): + if len(typecode) != 1: + msg='array() argument 1 must be char, not str' + raise OperationError(space.w_TypeError,space.wrap(msg)) + if typecode == 'c': + a=W_CharArray(space, typecode) + elif typecode == 'u': + a=W_UnicodeArray(space, typecode) + elif typecode in ('b', 'B', 'h', 'H', 'i', 'l'): + a=W_IntArray(space, typecode) + elif typecode in ('I', 'L'): + a=W_LongArray(space, typecode) + elif typecode in ('f', 'd'): + a=W_FloatArray(space, typecode) + else: + msg='bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' + raise OperationError(space.w_ValueError,space.wrap(msg)) + + if w_initializer is not None: + if not space.is_w(w_initializer, space.w_None): + a.descr_extend(w_initializer) #FIXME: use fromlist, fromstring, ... + + return a + +array.unwrap_spec=(ObjSpace, str, W_Root) + Added: pypy/branch/interplevel-array/pypy/module/array/test/imtst.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/test/imtst.py Sun Jul 4 21:05:58 2010 @@ -0,0 +1,41 @@ +def f(): + a=(1,2,3) + s=0 + while s<1000: + s+=a[0] + +def g(): + s=0 + while s<1000: + s+=1 + return s + + +def h(): + s=0 + i=0 + a=7 + while i<100000: + s+=i + i+=1 + return s + +def h2(): + s=0 + i=0 + a=(1,7,42) + while i<100000: + s+=a[1] + i+=1 + return s + +def ff(): + s=0 + i=0 + while i<100000: + s+=i + i+=1 + if i>50000: + i=float(i) + +print ff() Added: pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py Sun Jul 4 21:05:58 2010 @@ -0,0 +1,15 @@ +#!/usr/bin/python +from array import array + +img=array(640*480); +def f(): + l=0 + i=0; + while i<640*480: + l+=img[i] + i+=1 + return l + + +#for l in range(500): f() +print f() Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Sun Jul 4 21:05:58 2010 @@ -1,18 +1,101 @@ from pypy.conftest import gettestobjspace -class AppTestcArray: +class AppTestSizedArray: def setup_class(cls): cls.space = gettestobjspace(usemodules=('array',)) - cls.w_array = cls.space.appexec([], """(): + cls.w_sized_array = cls.space.appexec([], """(): import array - return array.array + return array.sized_array """) def test_simple(self): - a=self.array(10) + a=self.sized_array(10) a[5]=7.42 assert a[5]==7.42 +class AppTestArray: + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array',)) + cls.w_array = cls.space.appexec([], """(): + import array + return array.array + """) + + def test_ctor(self): + raises(TypeError, self.array, 'hi') + raises(TypeError, self.array, 1) + raises(ValueError, self.array, 'q') + + a=self.array('c') + raises(TypeError, a.append, 7) + a.append('h') + assert a[0] == 'h' + assert type(a[0]) is str + assert len(a) == 1 + + a=self.array('u') + raises(TypeError, a.append, 7) + a.append(unicode('h')) + assert a[0] == unicode('h') + assert type(a[0]) is unicode + assert len(a) == 1 + + a=self.array('c', ('a', 'b', 'c')) + assert a[0]=='a' + assert a[1]=='b' + assert a[2]=='c' + assert len(a) == 3 + + def test_value_range(self): + values=(-129, 128, -128, 127, 0, 255, -1, 256, + -32768, 32767, -32769, 32768, 65535, 65536, + -2147483647, -2147483648, 2147483647, 4294967295, 4294967296, + ) + for tc,ok,pt in (('b',( -128, 34, 127), int), + ('B',( 0, 23, 255), int), + ('h',(-32768, 30535, 32767), int), + ('H',( 0, 56783, 65535), int), + ('i',(-32768, 30535, 32767), int), + ('I',( 0, 56783, 65535), long), + ('l',(-2**32/2, 34, 2**32/2-1), int), + ('L',(0, 3523532, 2**32-1), long), + ): + a=self.array(tc, ok) + assert len(a) == len(ok) + for v in ok: + a.append(v) + for i,v in enumerate(ok*2): + assert a[i]==v + assert type(a[i]) is pt + for v in ok: + a[1]=v + assert a[0]==ok[0] + assert a[1]==v + assert a[2]==ok[2] + assert len(a) == 2*len(ok) + for v in values: + try: + a[1]=v + assert a[0]==ok[0] + assert a[1]==v + assert a[2]==ok[2] + except OverflowError: + pass + + def test_float(self): + values=[0, 1, 2.5, -4.25] + for tc in 'fd': + a=self.array(tc, values) + assert len(a)==len(values) + for i,v in enumerate(values): + assert a[i]==v + assert type(a[i]) is float + a[1]=10.125 + assert a[0]==0 + assert a[1]==10.125 + assert a[2]==2.5 + assert len(a)==len(values) + ## space.sys.get('modules') From fijal at codespeak.net Mon Jul 5 00:01:08 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Jul 2010 00:01:08 +0200 (CEST) Subject: [pypy-svn] r75825 - pypy/trunk/pypy/rlib Message-ID: <20100704220108.60552282B9D@codespeak.net> Author: fijal Date: Mon Jul 5 00:01:07 2010 New Revision: 75825 Modified: pypy/trunk/pypy/rlib/libffi.py pypy/trunk/pypy/rlib/rarithmetic.py pypy/trunk/pypy/rlib/rlocale.py Log: Simplify isnan and isinf to something sane. Python 2.3 required that, but we don't support it any more (yay) Modified: pypy/trunk/pypy/rlib/libffi.py ============================================================================== --- pypy/trunk/pypy/rlib/libffi.py (original) +++ pypy/trunk/pypy/rlib/libffi.py Mon Jul 5 00:01:07 2010 @@ -402,7 +402,7 @@ restype = ffi_type_sint32 elif restype.c_size <= 8: restype = ffi_type_sint64 - + res = c_ffi_prep_cif(self.ll_cif, cc, rffi.cast(rffi.UINT, argnum), restype, self.ll_argtypes) Modified: pypy/trunk/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/trunk/pypy/rlib/rarithmetic.py (original) +++ pypy/trunk/pypy/rlib/rarithmetic.py Mon Jul 5 00:01:07 2010 @@ -54,13 +54,10 @@ NAN = INFINITY / INFINITY def isinf(x): - return x != 0.0 and x / 2 == x + return x == INFINITY or x == -INFINITY -# To get isnan, working x-platform and both on 2.3 and 2.4, is a -# horror. I think this works (for reasons I don't really want to talk -# about), and probably when implemented on top of pypy, too. def isnan(v): - return v != v*1.0 or (v == 1.0 and v == 2.0) + return v != v def intmask(n): if isinstance(n, int): Modified: pypy/trunk/pypy/rlib/rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/rlocale.py (original) +++ pypy/trunk/pypy/rlib/rlocale.py Mon Jul 5 00:01:07 2010 @@ -13,15 +13,15 @@ self.message = message HAVE_LANGINFO = sys.platform != 'win32' -HAVE_LIBINTL = sys.platform != 'win32' +#HAVE_LIBINTL = sys.platform != 'win32' class CConfig: includes = ['locale.h', 'limits.h'] if HAVE_LANGINFO: includes += ['langinfo.h'] - if HAVE_LIBINTL: - includes += ['libintl.h'] + #if HAVE_LIBINTL: + # includes += ['libintl.h'] if sys.platform == 'win32': includes += ['windows.h'] _compilation_info_ = ExternalCompilationInfo( From hakanardo at codespeak.net Mon Jul 5 08:51:08 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 5 Jul 2010 08:51:08 +0200 (CEST) Subject: [pypy-svn] r75827 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100705065108.05DD4282B9D@codespeak.net> Author: hakanardo Date: Mon Jul 5 08:51:07 2010 New Revision: 75827 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: dict-based item unwrapping Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Jul 5 08:51:07 2010 @@ -41,19 +41,30 @@ return W_SizedFloatArray(space, size) sized_array.unwrap_spec=(ObjSpace, int) + +class TypeCode(object): + def __init__(self, itemtype, unwrap, bytes, canoverflow=False, signed=False): + self.itemtype=itemtype + self.bytes = bytes + self.unwrap=unwrap + self.signed = signed + self.canoverflow = canoverflow + + + types = { - 'c': lltype.GcArray(lltype.Char), - 'u': lltype.GcArray(lltype.UniChar), - 'b': lltype.GcArray(rffi.SIGNEDCHAR), - 'B': lltype.GcArray(rffi.UCHAR), - 'h': lltype.GcArray(rffi.SHORT), - 'H': lltype.GcArray(rffi.USHORT), - 'i': lltype.GcArray(lltype.Signed), - 'I': lltype.GcArray(lltype.Unsigned), - 'l': lltype.GcArray(lltype.Signed), - 'L': lltype.GcArray(lltype.Unsigned), - 'f': lltype.GcArray(lltype.SingleFloat), - 'd': lltype.GcArray(lltype.Float), + 'c': TypeCode(lltype.Char, 'str_w', 1), + 'u': TypeCode(lltype.UniChar, 'unicode_w', 4), # FIXME: is unicode always 4 bytes? + 'b': TypeCode(rffi.SIGNEDCHAR, 'int_w', 1, True, True), + 'B': TypeCode(rffi.UCHAR, 'int_w', 1, True), + 'h': TypeCode(rffi.SHORT, 'int_w', 2, True, True), + 'H': TypeCode(rffi.USHORT, 'int_w', 2, True), + 'i': TypeCode(lltype.Signed, 'int_w', 4, True, True), + 'I': TypeCode(lltype.Unsigned, 'int_w', 4, True), + 'l': TypeCode(lltype.Signed, 'int_w', 8, True, True), + 'L': TypeCode(lltype.Unsigned, 'int_w', 8, True), + 'f': TypeCode(lltype.SingleFloat, 'float_w', 4), + 'd': TypeCode(lltype.Float, 'float_w', 8), } ## def array(space, typecode, w_initializer=None): ## if @@ -77,7 +88,29 @@ self.len=0 self.buffer=None - def item_w(self, w_item): + def item_w(self, w_item): + space=self.space + tc=types[self.typecode] + unwrap=getattr(space, tc.unwrap) + item=unwrap(w_item) + if tc.canoverflow: + msg=None + if tc.signed: + if item<-2**(tc.bytes*8)/2: + msg='signed %d-byte integer is less than minimum'%tc.bytes + elif item>2**(tc.bytes*8)/2-1: + msg='signed %d-byte integer is greater than maximum'%tc.bytes + else: + if item<0: + msg='unsigned %d-byte integer is less than minimum'%tc.bytes + elif item>2**(tc.bytes*8)-1: + msg='unsigned %d-byte integer is greater than maximum'%tc.bytes + if msg is not None: + raise OperationError(space.w_OverflowError, space.wrap(msg)) + return rffi.cast(tc.itemtype, item) + + + def olditem_w(self, w_item): space=self.space if self.typecode == 'c': return self.space.str_w(w_item) @@ -148,7 +181,7 @@ return self.space.float_w(w_item) def setlen(self, size): - new_buffer=lltype.malloc(types[self.typecode], size, zero=True) + new_buffer=lltype.malloc(lltype.GcArray(types[self.typecode].itemtype), size) for i in range(self.len): new_buffer[i]=self.buffer[i] self.buffer=new_buffer From cfbolz at codespeak.net Mon Jul 5 11:56:15 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 5 Jul 2010 11:56:15 +0200 (CEST) Subject: [pypy-svn] r75828 - pypy/branch/reflex-support Message-ID: <20100705095615.A38A8282BF3@codespeak.net> Author: cfbolz Date: Mon Jul 5 11:56:12 2010 New Revision: 75828 Added: pypy/branch/reflex-support/ (props changed) - copied from r75827, pypy/trunk/ Log: branch for the CERN sprint From hakanardo at codespeak.net Mon Jul 5 12:23:06 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 5 Jul 2010 12:23:06 +0200 (CEST) Subject: [pypy-svn] r75829 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100705102306.66CCB282B9D@codespeak.net> Author: hakanardo Date: Mon Jul 5 12:23:04 2010 New Revision: 75829 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: all types now properly sized Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Jul 5 12:23:04 2010 @@ -1,10 +1,11 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError -from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.rpython.lltypesystem import lltype, rffi from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root from pypy.rlib.jit import dont_look_inside from pypy.rlib import rgc +from pypy.rlib.rbigint import rbigint FloatArray=lltype.GcArray(lltype.Float) @@ -43,9 +44,12 @@ class TypeCode(object): - def __init__(self, itemtype, unwrap, bytes, canoverflow=False, signed=False): + def __init__(self, itemtype, unwrap, canoverflow=False, signed=False): self.itemtype=itemtype - self.bytes = bytes + if itemtype is lltype.SingleFloat: + self.bytes=4 + else: + self.bytes = rffi.sizeof(itemtype) self.unwrap=unwrap self.signed = signed self.canoverflow = canoverflow @@ -53,27 +57,19 @@ types = { - 'c': TypeCode(lltype.Char, 'str_w', 1), - 'u': TypeCode(lltype.UniChar, 'unicode_w', 4), # FIXME: is unicode always 4 bytes? - 'b': TypeCode(rffi.SIGNEDCHAR, 'int_w', 1, True, True), - 'B': TypeCode(rffi.UCHAR, 'int_w', 1, True), - 'h': TypeCode(rffi.SHORT, 'int_w', 2, True, True), - 'H': TypeCode(rffi.USHORT, 'int_w', 2, True), - 'i': TypeCode(lltype.Signed, 'int_w', 4, True, True), - 'I': TypeCode(lltype.Unsigned, 'int_w', 4, True), - 'l': TypeCode(lltype.Signed, 'int_w', 8, True, True), - 'L': TypeCode(lltype.Unsigned, 'int_w', 8, True), - 'f': TypeCode(lltype.SingleFloat, 'float_w', 4), - 'd': TypeCode(lltype.Float, 'float_w', 8), + 'c': TypeCode(lltype.Char, 'str_w'), + 'u': TypeCode(lltype.UniChar, 'unicode_w'), + 'b': TypeCode(rffi.SIGNEDCHAR, 'int_w', True, True), + 'B': TypeCode(rffi.UCHAR, 'int_w', True), + 'h': TypeCode(rffi.SHORT, 'int_w', True, True), + 'H': TypeCode(rffi.USHORT, 'int_w', True), + 'i': TypeCode(rffi.INT, 'int_w', True, True), + 'I': TypeCode(rffi.UINT, 'int_w', True), + 'l': TypeCode(rffi.LONG, 'int_w', True, True), + 'L': TypeCode(rffi.ULONG, 'bigint_w', True), + 'f': TypeCode(lltype.SingleFloat, 'float_w'), + 'd': TypeCode(lltype.Float, 'float_w'), } -## def array(space, typecode, w_initializer=None): -## if -## self.len=0 -## self.buffer=None -## if w_initializer is not None and not space.is_w(w_size, space.w_None): -## self.extend(w_initializer) #FIXME: use fromlist, fromstring, ... - -## return W_SizedFloatArray(space, size) class W_Array(Wrappable): def __init__(self, space, typecode): @@ -87,12 +83,21 @@ self.typecode=typecode self.len=0 self.buffer=None + self.itemsize = types[typecode].bytes def item_w(self, w_item): space=self.space tc=types[self.typecode] unwrap=getattr(space, tc.unwrap) item=unwrap(w_item) + if tc.unwrap=='bigint_w': + try: + if tc.signed: item=item.tolonglong() + else: item=item.toulonglong() + except (ValueError, OverflowError): + msg='unsigned %d-byte integer out of range'%tc.bytes + raise OperationError(space.w_OverflowError, space.wrap(msg)) + if tc.canoverflow: msg=None if tc.signed: @@ -109,77 +114,6 @@ raise OperationError(space.w_OverflowError, space.wrap(msg)) return rffi.cast(tc.itemtype, item) - - def olditem_w(self, w_item): - space=self.space - if self.typecode == 'c': - return self.space.str_w(w_item) - elif self.typecode == 'u': - return self.space.unicode_w(w_item) - - elif self.typecode == 'b': - item=self.space.int_w(w_item) - if item<-128: - msg='signed char is less than minimum' - raise OperationError(space.w_OverflowError, space.wrap(msg)) - elif item>127: - msg='signed char is greater than maximum' - raise OperationError(space.w_OverflowError, space.wrap(msg)) - return rffi.cast(rffi.SIGNEDCHAR, item) - elif self.typecode == 'B': - item=self.space.int_w(w_item) - if item<0: - msg='unsigned byte integer is less than minimum' - raise OperationError(space.w_OverflowError, space.wrap(msg)) - elif item>255: - msg='unsigned byte integer is greater than maximum' - raise OperationError(space.w_OverflowError, space.wrap(msg)) - return rffi.cast(rffi.UCHAR, item) - - elif self.typecode == 'h': - item=self.space.int_w(w_item) - if item<-32768: - msg='signed short integer is less than minimum' - raise OperationError(space.w_OverflowError, space.wrap(msg)) - elif item>32767: - msg='signed short integer is greater than maximum' - raise OperationError(space.w_OverflowError, space.wrap(msg)) - return rffi.cast(rffi.SHORT, item) - elif self.typecode == 'H': - item=self.space.int_w(w_item) - if item<0: - msg='unsigned short integer is less than minimum' - raise OperationError(space.w_OverflowError, space.wrap(msg)) - elif item>65535: - msg='unsigned short integer is greater than maximum' - raise OperationError(space.w_OverflowError, space.wrap(msg)) - return rffi.cast(rffi.USHORT, item) - - elif self.typecode in ('i', 'l'): - item=self.space.int_w(w_item) - if item<-2147483648: - msg='signed integer is less than minimum' - raise OperationError(space.w_OverflowError, space.wrap(msg)) - elif item>2147483647: - msg='signed integer is greater than maximum' - raise OperationError(space.w_OverflowError, space.wrap(msg)) - return rffi.cast(lltype.Signed, item) - elif self.typecode in ('I', 'L'): - item=self.space.int_w(w_item) - if item<0: - msg='unsigned integer is less than minimum' - raise OperationError(space.w_OverflowError, space.wrap(msg)) - elif item>4294967295: - msg='unsigned integer is greater than maximum' - raise OperationError(space.w_OverflowError, space.wrap(msg)) - return rffi.cast(lltype.Unsigned, item) - - elif self.typecode == 'f': - item=self.space.float_w(w_item) - return rffi.cast(lltype.SingleFloat, item) - elif self.typecode == 'd': - return self.space.float_w(w_item) - def setlen(self, size): new_buffer=lltype.malloc(lltype.GcArray(types[self.typecode].itemtype), size) for i in range(self.len): @@ -225,6 +159,13 @@ return self.space.wrap(self.len) descr_len.unwrap_spec = ['self'] + def descr_fromstring(self, s): + import struct + + +def descr_itemsize(space, self): + return space.wrap(self.itemsize) + W_Array.typedef=TypeDef( 'Array', append = interp2app(W_Array.descr_append), @@ -232,6 +173,7 @@ __len__ = interp2app(W_Array.descr_len), __getitem__ = interp2app(W_Array.descr_getitem), __setitem__ = interp2app(W_Array.descr_setitem), + itemsize = GetSetProperty(descr_itemsize, cls=W_Array) ) def array(space, typecode, w_initializer=None): Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Mon Jul 5 12:23:04 2010 @@ -51,6 +51,10 @@ -32768, 32767, -32769, 32768, 65535, 65536, -2147483647, -2147483648, 2147483647, 4294967295, 4294967296, ) + for bb in (8, 16, 32, 64, 128, 256, 512, 1024): + for b in (bb-1, bb, bb+1): + values += ( 2**b, 2**b+1, 2**b-1, -2**b, -2**b+1, -2**b-1) + for tc,ok,pt in (('b',( -128, 34, 127), int), ('B',( 0, 23, 255), int), ('h',(-32768, 30535, 32767), int), @@ -96,6 +100,31 @@ assert a[2]==2.5 assert len(a)==len(values) + def test_itemsize(self): + for t in 'cbB': assert(self.array(t).itemsize>=1) + for t in 'uhHiI': assert(self.array(t).itemsize>=2) + for t in 'lLf': assert(self.array(t).itemsize>=4) + for t in 'd': assert(self.array(t).itemsize>=8) + + inttypes='bhil' + for t in inttypes: + a=self.array(t, [1,2,3]) + b=a.itemsize + for v in (-2**(8*b)/2, 2**(8*b)/2-1): + a[1]=v + assert a[0]==1 and a[1]==v and a[2]==3 + raises(OverflowError, a.append, -2**(8*b)/2-1) + raises(OverflowError, a.append, 2**(8*b)/2) + + a=self.array(t.upper(), [1,2,3]) + b=a.itemsize + for v in (0, 2**(8*b)-1): + print b, v + a[1]=v + assert a[0]==1 and a[1]==v and a[2]==3 + raises(OverflowError, a.append, -1) + raises(OverflowError, a.append, 2**(8*b)) + ## space.sys.get('modules') From afa at codespeak.net Mon Jul 5 12:51:49 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 5 Jul 2010 12:51:49 +0200 (CEST) Subject: [pypy-svn] r75830 - in pypy/trunk/pypy/rpython/lltypesystem: . test Message-ID: <20100705105149.561E6282B9D@codespeak.net> Author: afa Date: Mon Jul 5 12:51:47 2010 New Revision: 75830 Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py pypy/trunk/pypy/rpython/lltypesystem/test/test_rffi.py Log: Add support for rffi.sizeof(lltype.SingleFloat) Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rffi.py Mon Jul 5 12:51:47 2010 @@ -786,6 +786,8 @@ return r_wchar_t.BITS/8 if tp is lltype.Float: return 8 + if tp is lltype.SingleFloat: + return 4 assert isinstance(tp, lltype.Number) if tp is lltype.Signed: return ULONG._type.BITS/8 Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_rffi.py Mon Jul 5 12:51:47 2010 @@ -707,6 +707,7 @@ lltype.UniChar: ctypes.c_wchar, lltype.Char: ctypes.c_ubyte, DOUBLE: ctypes.c_double, + FLOAT: ctypes.c_float, SIGNEDCHAR: ctypes.c_byte, UCHAR: ctypes.c_ubyte, SHORT: ctypes.c_short, From cfbolz at codespeak.net Mon Jul 5 13:54:03 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 5 Jul 2010 13:54:03 +0200 (CEST) Subject: [pypy-svn] r75831 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100705115403.E6CF5282BF3@codespeak.net> Author: cfbolz Date: Mon Jul 5 13:54:02 2010 New Revision: 75831 Added: pypy/branch/reflex-support/pypy/module/cppyy/ pypy/branch/reflex-support/pypy/module/cppyy/__init__.py (contents, props changed) pypy/branch/reflex-support/pypy/module/cppyy/test/ pypy/branch/reflex-support/pypy/module/cppyy/test/__init__.py (contents, props changed) Log: (arigo, antocuni, cfbolz, wlav) the empty module that we want to write Added: pypy/branch/reflex-support/pypy/module/cppyy/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/__init__.py Mon Jul 5 13:54:02 2010 @@ -0,0 +1,10 @@ +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + """ """ + + interpleveldefs = { + } + + appleveldefs = { + } Added: pypy/branch/reflex-support/pypy/module/cppyy/test/__init__.py ============================================================================== From hakanardo at codespeak.net Mon Jul 5 13:56:45 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 5 Jul 2010 13:56:45 +0200 (CEST) Subject: [pypy-svn] r75832 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100705115645.385CF282BF6@codespeak.net> Author: hakanardo Date: Mon Jul 5 13:56:43 2010 New Revision: 75832 Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: unrolling_iterable and some whitespace maniac to keep pep8 happy Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/__init__.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/__init__.py Mon Jul 5 13:56:43 2010 @@ -1,13 +1,13 @@ # Package initialisation from pypy.interpreter.mixedmodule import MixedModule + class Module(MixedModule): appleveldefs = { } - + interpleveldefs = { - 'array' : 'interp_array.array', - 'sized_array' : 'interp_array.sized_array', + 'array': 'interp_array.array', + 'sized_array': 'interp_array.sized_array', } - Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Jul 5 13:56:43 2010 @@ -5,18 +5,21 @@ from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root from pypy.rlib.jit import dont_look_inside from pypy.rlib import rgc -from pypy.rlib.rbigint import rbigint +from pypy.rlib.unroll import unrolling_iterable + +FloatArray = lltype.GcArray(lltype.Float) -FloatArray=lltype.GcArray(lltype.Float) class W_SizedFloatArray(Wrappable): @dont_look_inside def __init__(self, space, size): - self.space=space - self.size=size + self.space = space + self.size = size print "malloc" - #self.buffer = lltype.malloc(rffi.DOUBLEP.TO, size, flavor='raw', zero=True) - #self.buffer = rgc.malloc_nonmovable(lltype.GcArray(lltype.Float), size) + #self.buffer = lltype.malloc(rffi.DOUBLEP.TO, size, flavor='raw', + # zero=True) + #self.buffer = rgc.malloc_nonmovable(lltype.GcArray(lltype.Float), + # size) self.buffer = lltype.malloc(FloatArray, size, zero=True) print "buf: ", self.buffer @@ -27,33 +30,33 @@ def descr_getitem(self, idx): return self.space.wrap(self.buffer[idx]) descr_getitem.unwrap_spec = ['self', int] - + def descr_setitem(self, idx, val): - self.buffer[idx]=val + self.buffer[idx] = val descr_setitem.unwrap_spec = ['self', int, float] W_SizedFloatArray.typedef = TypeDef( 'SizedFloatArray', - __getitem__ = interp2app(W_SizedFloatArray.descr_getitem), - __setitem__ = interp2app(W_SizedFloatArray.descr_setitem), + __getitem__=interp2app(W_SizedFloatArray.descr_getitem), + __setitem__=interp2app(W_SizedFloatArray.descr_setitem), ) - -def sized_array(space,size): + + +def sized_array(space, size): return W_SizedFloatArray(space, size) -sized_array.unwrap_spec=(ObjSpace, int) +sized_array.unwrap_spec = (ObjSpace, int) class TypeCode(object): def __init__(self, itemtype, unwrap, canoverflow=False, signed=False): - self.itemtype=itemtype + self.itemtype = itemtype if itemtype is lltype.SingleFloat: - self.bytes=4 + self.bytes = 4 else: self.bytes = rffi.sizeof(itemtype) - self.unwrap=unwrap + self.unwrap = unwrap self.signed = signed self.canoverflow = canoverflow - types = { @@ -70,61 +73,67 @@ 'f': TypeCode(lltype.SingleFloat, 'float_w'), 'd': TypeCode(lltype.Float, 'float_w'), } +unroll_typecodes = unrolling_iterable(types.keys()) + class W_Array(Wrappable): def __init__(self, space, typecode): if len(typecode) != 1: - msg='array() argument 1 must be char, not str' - raise OperationError(space.w_TypeError,space.wrap(msg)) + msg = 'array() argument 1 must be char, not str' + raise OperationError(space.w_TypeError, space.wrap(msg)) if typecode not in 'cbBuhHiIlLfd': - msg='bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' - raise OperationError(space.w_ValueError,space.wrap(msg)) - self.space=space - self.typecode=typecode - self.len=0 - self.buffer=None + msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' + raise OperationError(space.w_ValueError, space.wrap(msg)) + self.space = space + self.typecode = typecode + self.len = 0 + self.buffer = None self.itemsize = types[typecode].bytes - def item_w(self, w_item): - space=self.space - tc=types[self.typecode] - unwrap=getattr(space, tc.unwrap) - item=unwrap(w_item) - if tc.unwrap=='bigint_w': - try: - if tc.signed: item=item.tolonglong() - else: item=item.toulonglong() - except (ValueError, OverflowError): - msg='unsigned %d-byte integer out of range'%tc.bytes - raise OperationError(space.w_OverflowError, space.wrap(msg)) - - if tc.canoverflow: - msg=None - if tc.signed: - if item<-2**(tc.bytes*8)/2: - msg='signed %d-byte integer is less than minimum'%tc.bytes - elif item>2**(tc.bytes*8)/2-1: - msg='signed %d-byte integer is greater than maximum'%tc.bytes - else: - if item<0: - msg='unsigned %d-byte integer is less than minimum'%tc.bytes - elif item>2**(tc.bytes*8)-1: - msg='unsigned %d-byte integer is greater than maximum'%tc.bytes - if msg is not None: - raise OperationError(space.w_OverflowError, space.wrap(msg)) - return rffi.cast(tc.itemtype, item) + def item_w(self, w_item): + space = self.space + for c in unroll_typecodes: + if self.typecode == c: + tc = types[c] + unwrap = getattr(space, tc.unwrap) + item = unwrap(w_item) + if tc.unwrap == 'bigint_w': + try: + if tc.signed: + item = item.tolonglong() + else: + item = item.toulonglong() + except (ValueError, OverflowError): + msg = 'unsigned %d-byte integer out of range' % tc.bytes + raise OperationError(space.w_OverflowError, space.wrap(msg)) + + if tc.canoverflow: + msg = None + if tc.signed: + if item < -2 ** (tc.bytes * 8) / 2: + msg = 'signed %d-byte integer is less than minimum' % tc.bytes + elif item > 2 ** (tc.bytes * 8) / 2 - 1: + msg = 'signed %d-byte integer is greater than maximum' % tc.bytes + else: + if item < 0: + msg = 'unsigned %d-byte integer is less than minimum' % tc.bytes + elif item > 2 ** (tc.bytes * 8) - 1: + msg = 'unsigned %d-byte integer is greater than maximum' % tc.bytes + if msg is not None: + raise OperationError(space.w_OverflowError, space.wrap(msg)) + return rffi.cast(tc.itemtype, item) def setlen(self, size): - new_buffer=lltype.malloc(lltype.GcArray(types[self.typecode].itemtype), size) + new_buffer = lltype.malloc(lltype.GcArray(types[self.typecode].itemtype), size) for i in range(self.len): - new_buffer[i]=self.buffer[i] - self.buffer=new_buffer - self.len=size - - def descr_append(self,w_x): - x=self.item_w(w_x) - self.setlen(self.len+1) - self.buffer[self.len-1]=x + new_buffer[i] = self.buffer[i] + self.buffer = new_buffer + self.len = size + + def descr_append(self, w_x): + x = self.item_w(w_x) + self.setlen(self.len + 1) + self.buffer[self.len - 1] = x descr_append.unwrap_spec = ['self', W_Root] def descr_extend(self, w_initializer): @@ -139,22 +148,21 @@ break self.descr_append(w_item) descr_extend.unwrap_spec = ['self', W_Root] - def descr_getitem(self, idx): item = self.buffer[idx] if self.typecode in ('b', 'B', 'h', 'H', 'i', 'l'): - item=rffi.cast(lltype.Signed, item) + item = rffi.cast(lltype.Signed, item) if self.typecode == 'f': - item=float(item) + item = float(item) return self.space.wrap(item) descr_getitem.unwrap_spec = ['self', int] def descr_setitem(self, idx, w_item): - item=self.item_w(w_item) - self.buffer[idx]=item + item = self.item_w(w_item) + self.buffer[idx] = item descr_setitem.unwrap_spec = ['self', int, W_Root] - + def descr_len(self): return self.space.wrap(self.len) descr_len.unwrap_spec = ['self'] @@ -166,7 +174,7 @@ def descr_itemsize(space, self): return space.wrap(self.itemsize) -W_Array.typedef=TypeDef( +W_Array.typedef = TypeDef( 'Array', append = interp2app(W_Array.descr_append), extend = interp2app(W_Array.descr_extend), @@ -176,13 +184,12 @@ itemsize = GetSetProperty(descr_itemsize, cls=W_Array) ) + def array(space, typecode, w_initializer=None): - a=W_Array(space, typecode) + a = W_Array(space, typecode) if w_initializer is not None: if not space.is_w(w_initializer, space.w_None): - a.descr_extend(w_initializer) #FIXME: use fromlist, fromstring, ... + a.descr_extend(w_initializer) # FIXME: use fromlist, fromstring, ... return a - -array.unwrap_spec=(ObjSpace, str, W_Root) - +array.unwrap_spec = (ObjSpace, str, W_Root) Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Mon Jul 5 13:56:43 2010 @@ -1,5 +1,6 @@ from pypy.conftest import gettestobjspace + class AppTestSizedArray: def setup_class(cls): cls.space = gettestobjspace(usemodules=('array',)) @@ -9,9 +10,10 @@ """) def test_simple(self): - a=self.sized_array(10) - a[5]=7.42 - assert a[5]==7.42 + a = self.sized_array(10) + a[5] = 7.42 + assert a[5] == 7.42 + class AppTestArray: def setup_class(cls): @@ -26,113 +28,102 @@ raises(TypeError, self.array, 1) raises(ValueError, self.array, 'q') - a=self.array('c') + a = self.array('c') raises(TypeError, a.append, 7) a.append('h') assert a[0] == 'h' assert type(a[0]) is str assert len(a) == 1 - a=self.array('u') + a = self.array('u') raises(TypeError, a.append, 7) a.append(unicode('h')) assert a[0] == unicode('h') assert type(a[0]) is unicode assert len(a) == 1 - a=self.array('c', ('a', 'b', 'c')) - assert a[0]=='a' - assert a[1]=='b' - assert a[2]=='c' + a = self.array('c', ('a', 'b', 'c')) + assert a[0] == 'a' + assert a[1] == 'b' + assert a[2] == 'c' assert len(a) == 3 def test_value_range(self): - values=(-129, 128, -128, 127, 0, 255, -1, 256, - -32768, 32767, -32769, 32768, 65535, 65536, - -2147483647, -2147483648, 2147483647, 4294967295, 4294967296, - ) + values = (-129, 128, -128, 127, 0, 255, -1, 256, + -32768, 32767, -32769, 32768, 65535, 65536, + -2147483647, -2147483648, 2147483647, 4294967295, 4294967296, + ) for bb in (8, 16, 32, 64, 128, 256, 512, 1024): - for b in (bb-1, bb, bb+1): - values += ( 2**b, 2**b+1, 2**b-1, -2**b, -2**b+1, -2**b-1) - - for tc,ok,pt in (('b',( -128, 34, 127), int), - ('B',( 0, 23, 255), int), - ('h',(-32768, 30535, 32767), int), - ('H',( 0, 56783, 65535), int), - ('i',(-32768, 30535, 32767), int), - ('I',( 0, 56783, 65535), long), - ('l',(-2**32/2, 34, 2**32/2-1), int), - ('L',(0, 3523532, 2**32-1), long), - ): - a=self.array(tc, ok) + for b in (bb - 1, bb, bb + 1): + values += (2 ** b, 2 ** b + 1, 2 ** b - 1, + -2 ** b, -2 ** b + 1, -2 ** b - 1) + + for tc, ok, pt in (('b', ( -128, 34, 127), int), + ('B', ( 0, 23, 255), int), + ('h', (-32768, 30535, 32767), int), + ('H', ( 0, 56783, 65535), int), + ('i', (-32768, 30535, 32767), int), + ('I', ( 0, 56783, 65535), long), + ('l', (-2 ** 32 / 2, 34, 2 ** 32 / 2 - 1), int), + ('L', (0, 3523532, 2 ** 32 - 1), long), + ): + a = self.array(tc, ok) assert len(a) == len(ok) for v in ok: a.append(v) - for i,v in enumerate(ok*2): - assert a[i]==v + for i, v in enumerate(ok * 2): + assert a[i] == v assert type(a[i]) is pt for v in ok: - a[1]=v - assert a[0]==ok[0] - assert a[1]==v - assert a[2]==ok[2] - assert len(a) == 2*len(ok) + a[1] = v + assert a[0] == ok[0] + assert a[1] == v + assert a[2] == ok[2] + assert len(a) == 2 * len(ok) for v in values: try: - a[1]=v - assert a[0]==ok[0] - assert a[1]==v - assert a[2]==ok[2] + a[1] = v + assert a[0] == ok[0] + assert a[1] == v + assert a[2] == ok[2] except OverflowError: pass def test_float(self): - values=[0, 1, 2.5, -4.25] + values = [0, 1, 2.5, -4.25] for tc in 'fd': - a=self.array(tc, values) - assert len(a)==len(values) - for i,v in enumerate(values): - assert a[i]==v + a = self.array(tc, values) + assert len(a) == len(values) + for i, v in enumerate(values): + assert a[i] == v assert type(a[i]) is float - a[1]=10.125 - assert a[0]==0 - assert a[1]==10.125 - assert a[2]==2.5 - assert len(a)==len(values) + a[1] = 10.125 + assert a[0] == 0 + assert a[1] == 10.125 + assert a[2] == 2.5 + assert len(a) == len(values) def test_itemsize(self): - for t in 'cbB': assert(self.array(t).itemsize>=1) - for t in 'uhHiI': assert(self.array(t).itemsize>=2) - for t in 'lLf': assert(self.array(t).itemsize>=4) - for t in 'd': assert(self.array(t).itemsize>=8) - - inttypes='bhil' + for t in 'cbB': assert(self.array(t).itemsize >= 1) + for t in 'uhHiI': assert(self.array(t).itemsize >= 2) + for t in 'lLf': assert(self.array(t).itemsize >= 4) + for t in 'd': assert(self.array(t).itemsize >= 8) + + inttypes = 'bhil' for t in inttypes: - a=self.array(t, [1,2,3]) - b=a.itemsize - for v in (-2**(8*b)/2, 2**(8*b)/2-1): - a[1]=v - assert a[0]==1 and a[1]==v and a[2]==3 - raises(OverflowError, a.append, -2**(8*b)/2-1) - raises(OverflowError, a.append, 2**(8*b)/2) - - a=self.array(t.upper(), [1,2,3]) - b=a.itemsize - for v in (0, 2**(8*b)-1): + a = self.array(t, [1, 2, 3]) + b = a.itemsize + for v in (-2 ** (8 * b) / 2, 2 ** (8 * b) / 2 - 1): + a[1] = v + assert a[0] == 1 and a[1] == v and a[2] == 3 + raises(OverflowError, a.append, -2 ** (8 * b) / 2 - 1) + raises(OverflowError, a.append, 2 ** (8 * b) / 2) + + a = self.array(t.upper(), [1, 2, 3]) + b = a.itemsize + for v in (0, 2 ** (8 * b) - 1): print b, v - a[1]=v - assert a[0]==1 and a[1]==v and a[2]==3 + a[1] = v + assert a[0] == 1 and a[1] == v and a[2] == 3 raises(OverflowError, a.append, -1) - raises(OverflowError, a.append, 2**(8*b)) - - -## space.sys.get('modules') - -## w_import = space.builtin.get('__import__') -## w_array = space.call(w_import, space.newlist([space.wrap('array')])) - -## print w_array -## space.appexec([], "(): import array; print array.array(7)") -## print space.appexec([], "(): import array; return array.array(7)") -## space.appexec([], "(): import array; a=array.array(7); print a[2]") -## space.appexec([], "(): import array; a=array.array(7); a[2]=7.42; print a[2]") + raises(OverflowError, a.append, 2 ** (8 * b)) From afa at codespeak.net Mon Jul 5 15:03:38 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 5 Jul 2010 15:03:38 +0200 (CEST) Subject: [pypy-svn] r75833 - pypy/trunk/pypy/rlib Message-ID: <20100705130338.998EA282BF6@codespeak.net> Author: afa Date: Mon Jul 5 15:03:37 2010 New Revision: 75833 Modified: pypy/trunk/pypy/rlib/rlocale.py Log: Revert the part of r75825 which was not intended. Modified: pypy/trunk/pypy/rlib/rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/rlocale.py (original) +++ pypy/trunk/pypy/rlib/rlocale.py Mon Jul 5 15:03:37 2010 @@ -13,15 +13,15 @@ self.message = message HAVE_LANGINFO = sys.platform != 'win32' -#HAVE_LIBINTL = sys.platform != 'win32' +HAVE_LIBINTL = sys.platform != 'win32' class CConfig: includes = ['locale.h', 'limits.h'] if HAVE_LANGINFO: includes += ['langinfo.h'] - #if HAVE_LIBINTL: - # includes += ['libintl.h'] + if HAVE_LIBINTL: + includes += ['libintl.h'] if sys.platform == 'win32': includes += ['windows.h'] _compilation_info_ = ExternalCompilationInfo( From cfbolz at codespeak.net Mon Jul 5 15:25:55 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 5 Jul 2010 15:25:55 +0200 (CEST) Subject: [pypy-svn] r75834 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100705132555.9CB16282BF6@codespeak.net> Author: cfbolz Date: Mon Jul 5 15:25:54 2010 New Revision: 75834 Added: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (contents, props changed) pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx (contents, props changed) pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (contents, props changed) Modified: pypy/branch/reflex-support/pypy/module/cppyy/__init__.py pypy/branch/reflex-support/pypy/module/cppyy/test/ (props changed) Log: (arigo, antocuni, cfbolz, wlav): a tiny C++ class and a test of how we want the interface to look for now. Now the real fun starts. Modified: pypy/branch/reflex-support/pypy/module/cppyy/__init__.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/__init__.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/__init__.py Mon Jul 5 15:25:54 2010 @@ -4,6 +4,7 @@ """ """ interpleveldefs = { + 'load_lib': 'interp_cppyy.load_lib', } appleveldefs = { Added: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Mon Jul 5 15:25:54 2010 @@ -0,0 +1,43 @@ +from pypy.interpreter.error import OperationError +from pypy.interpreter.gateway import ObjSpace, interp2app +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.baseobjspace import Wrappable + +from pypy.rlib.libffi import CDLL + +def load_lib(space, name): + cdll = CDLL(name) + return W_CPPLibrary(space, cdll) +load_lib.unwrap_spec = [ObjSpace, str] + +class W_CPPLibrary(Wrappable): + def __init__(self, space, cdll): + self.cdll = cdll + self.space = space + + def type_byname(self, name): + return W_CPPType(self, name) + +W_CPPLibrary.typedef = TypeDef( + 'CPPLibrary', + type_byname = interp2app(W_CPPLibrary.type_byname, unwrap_spec=['self', str]), +) + + +class W_CPPType(Wrappable): + def __init__(self, cpplib, name): + self.space = cpplib.space + self.cpplib = cpplib + self.name = name + + def invoke(self, name, args_w): + xxx + + def construct(self, args_w): + xxx + +W_CPPType.typedef = TypeDef( + 'CPPType', + invoke = interp2app(W_CPPType.invoke, unwrap_spec=['self', str, 'args_w']), + construct = interp2app(W_CPPType.construct, unwrap_spec=['self', 'args_w']), +) Added: pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile Mon Jul 5 15:25:54 2010 @@ -0,0 +1,8 @@ +LD_LIBRARY_PATH := ${LD_LIBRARY_PATH}:${ROOTSYS}/lib +ROOTSYS := ${ROOTSYS} + +example01Dict.so: example01.cxx + echo $(ROOTSYS) + $(ROOTSYS)/bin/genreflex example01.cxx + g++ -o $@ example01_rflx.cpp -shared -I$(ROOTSYS)/include -L$(ROOTSYS)/lib -lReflex + Added: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx Mon Jul 5 15:25:54 2010 @@ -0,0 +1,16 @@ +#include + +class example01 { +public: + int somedata; + example01(int a) : somedata(a) { + std::cout << "constructor called" << std::endl; + } + + static int add1(int a) { + return a + 1; + } + int add(int a) { + return somedata + a; + } +}; Added: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Mon Jul 5 15:25:54 2010 @@ -0,0 +1,24 @@ +import py +import os +from pypy.conftest import gettestobjspace + +currpath = py.path.local(__file__).dirpath() + +class AppTestCPPYY: + def setup_class(cls): + cls.space = gettestobjspace(usemodules=['cppyy']) + env = os.environ + cls.w_example01 = cls.space.appexec([], """(): + import cppyy + return cppyy.load_lib(%r)""" % (str(currpath.join("example01Dict.so")), )) + + def test_example01static(self): + t = self.example01.type_byname("example01") + res = t.invoke("add1", 1) + assert res == 2 + + def test_example01method(self): + t = self.example01.type_byname("example01") + instance = t.construct(7) + res = instance.invoke("add2", 4) + assert res == 11 From cfbolz at codespeak.net Mon Jul 5 17:19:57 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 5 Jul 2010 17:19:57 +0200 (CEST) Subject: [pypy-svn] r75835 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src Message-ID: <20100705151957.19CF0282BF6@codespeak.net> Author: cfbolz Date: Mon Jul 5 17:19:56 2010 New Revision: 75835 Added: pypy/branch/reflex-support/pypy/module/cppyy/include/ pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/src/ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Log: (cfbolz, wlav, antocuni, arigo): write just enough code to be able to call a static method Added: pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h Mon Jul 5 17:19:56 2010 @@ -0,0 +1,7 @@ +#ifndef CPPYY_CPPYY +#define CPPYY_CPPYY + +#include "Reflex/Type.h" +#include "Reflex/Member.h" + +#endif // CPPYY_CPPYY Added: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Mon Jul 5 17:19:56 2010 @@ -0,0 +1,9 @@ + +#ifndef CPPYY_REFLEXCWRAPPER +#define CPPYY_REFLEXCWRAPPER + +extern "C" { + long callstatic_l(const char* class_name, const char* method_name, int numargs, void* args[]); +} + +#endif // ifndef CPPYY_REFLEXCWRAPPER Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Mon Jul 5 17:19:56 2010 @@ -1,10 +1,35 @@ +import py, os + from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, interp2app from pypy.interpreter.typedef import TypeDef from pypy.interpreter.baseobjspace import Wrappable +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rpython.lltypesystem import rffi, lltype + from pypy.rlib.libffi import CDLL + +srcpath = py.path.local(__file__).dirpath().join("src") +incpath = py.path.local(__file__).dirpath().join("include") +rootincpath = os.path.join(os.environ["ROOTSYS"], "include") +rootlibpath = os.path.join(os.environ["ROOTSYS"], "lib") + +eci = ExternalCompilationInfo( + separate_module_files=[srcpath.join("reflexcwrapper.cxx")], + include_dirs=[incpath, rootincpath], + library_dirs=[rootlibpath], + libraries=["Reflex"], + use_cpp_linker=True, +) + +callstatic_l = rffi.llexternal( + "callstatic_l", + [rffi.CCHARP, rffi.CCHARP, rffi.INT, rffi.VOIDPP], rffi.LONG, + compilation_info=eci) + + def load_lib(space, name): cdll = CDLL(name) return W_CPPLibrary(space, cdll) @@ -31,7 +56,17 @@ self.name = name def invoke(self, name, args_w): - xxx + args = lltype.malloc(rffi.CArray(rffi.VOIDP), len(args_w), flavor='raw') + for i in range(len(args_w)): + arg = self.space.int_w(args_w[i]) + x = lltype.malloc(rffi.LONGP.TO, 1, flavor='raw') + x[0] = arg + args[i] = rffi.cast(rffi.VOIDP, x) + result = callstatic_l(self.name, name, len(args_w), args) + for i in range(len(args_w)): + lltype.free(args[i], flavor='raw') + lltype.free(args, flavor='raw') + return self.space.wrap(result) def construct(self, args_w): xxx Added: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Mon Jul 5 17:19:56 2010 @@ -0,0 +1,12 @@ +#include "cppyy.h" +#include "reflexcwrapper.h" +#include + +long callstatic_l(const char* classname, const char* methodname, int numargs, void* args[]) { + long result; + std::vector arguments(args, args+numargs); + Reflex::Type t = Reflex::Type::ByName(classname); + Reflex::Member m = t.FunctionMemberByName(methodname); + m.Invoke(result, arguments); + return result; +} From hakanardo at codespeak.net Mon Jul 5 17:24:20 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 5 Jul 2010 17:24:20 +0200 (CEST) Subject: [pypy-svn] r75836 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100705152420.256A1282BF6@codespeak.net> Author: hakanardo Date: Mon Jul 5 17:24:18 2010 New Revision: 75836 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: fromfile, fromlist, fromstring and fromunicode Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Jul 5 17:24:18 2010 @@ -169,7 +169,52 @@ def descr_fromstring(self, s): import struct + if len(s)%self.itemsize !=0: + msg = 'string length not a multiple of item size' + raise OperationError(self.space.w_ValueError, self.space.wrap(msg)) + for i in range(len(s)/self.itemsize): + p = i * self.itemsize + item=struct.unpack(self.typecode, s[p:p + self.itemsize])[0] + self.descr_append(self.space.wrap(item)) + descr_fromstring.unwrap_spec = ['self', str] + + def descr_fromfile(self, w_f, n): + space=self.space + size = n*self.itemsize + w_s = space.call_function( + space.getattr(w_f, space.wrap('read')), + space.wrap(size)) + s=space.str_w(w_s) + if len(s) != size: + n = len(s) % self.itemsize + if n != 0: s = s[0:-(len(s) % self.itemsize)] + self.descr_fromstring(s) + msg='not enough items in file' + raise OperationError(self.space.w_EOFError, self.space.wrap(msg)) + else: + self.descr_fromstring(s) + descr_fromfile.unwrap_spec = ['self', W_Root, int] + + def descr_fromlist(self, w_lst): + oldbuf = self.buffer + oldlen = self.len + try: + self.descr_extend(w_lst) + except OperationError: + self.buffer = oldbuf + self.len = oldlen + raise + descr_fromlist.unwrap_spec = ['self', W_Root] + + def descr_fromunicode(self, s): + if self.typecode != 'u': + msg = "fromunicode() may only be called on type 'u' arrays" + raise OperationError(self.space.w_ValueError, self.space.wrap(msg)) + self.descr_extend(self.space.wrap(s)) + descr_fromunicode.unwrap_spec = ['self', unicode] + + def descr_itemsize(space, self): return space.wrap(self.itemsize) @@ -181,14 +226,24 @@ __len__ = interp2app(W_Array.descr_len), __getitem__ = interp2app(W_Array.descr_getitem), __setitem__ = interp2app(W_Array.descr_setitem), - itemsize = GetSetProperty(descr_itemsize, cls=W_Array) + itemsize = GetSetProperty(descr_itemsize, cls=W_Array), + fromstring = interp2app(W_Array.descr_fromstring), + fromfile = interp2app(W_Array.descr_fromfile), + fromlist = interp2app(W_Array.descr_fromlist), + fromunicode = interp2app(W_Array.descr_fromunicode), ) def array(space, typecode, w_initializer=None): a = W_Array(space, typecode) if w_initializer is not None: - if not space.is_w(w_initializer, space.w_None): + if space.is_w(space.type(w_initializer), space.w_str): + a.descr_fromstring(space.str_w(w_initializer)) + elif space.is_w(space.type(w_initializer), space.w_unicode): + a.descr_fromunicode(space.unicode_w(w_initializer)) + elif space.is_w(space.type(w_initializer), space.w_list): + a.descr_fromlist(w_initializer) + elif not space.is_w(w_initializer, space.w_None): a.descr_extend(w_initializer) # FIXME: use fromlist, fromstring, ... return a Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Mon Jul 5 17:24:18 2010 @@ -127,3 +127,72 @@ assert a[0] == 1 and a[1] == v and a[2] == 3 raises(OverflowError, a.append, -1) raises(OverflowError, a.append, 2 ** (8 * b)) + + def test_fromstring(self): + a = self.array('c') + a.fromstring('Hi!') + assert a[0] == 'H' and a[1] == 'i' and a[2] == '!' and len(a) == 3 + + for t in 'bBhHiIlLfd': + a = self.array(t) + a.fromstring('\x00' * a.itemsize*2) + assert len(a) == 2 and a[0] == 0 and a[1] == 0 + if a.itemsize > 1: + raises(ValueError, a.fromstring, '\x00' * (a.itemsize-1)) + raises(ValueError, a.fromstring, '\x00' * (a.itemsize+1)) + raises(ValueError, a.fromstring, '\x00' * (2*a.itemsize-1)) + raises(ValueError, a.fromstring, '\x00' * (2*a.itemsize+1)) + b = self.array(t, '\x00' * a.itemsize*2) + assert len(b) == 2 and b[0] == 0 and b[1] == 0 + + def test_fromfile(self): + + class myfile(object): + def __init__(self, c, s): + self.c = c + self.s = s + def read(self,n): + return self.c*min(n,self.s) + + f=myfile('\x00', 20) + for t in 'bBhHiIlLfd': + a = self.array(t) + a.fromfile(f,2) + assert len(a)==2 and a[0]==0 and a[1]==0 + + a = self.array('b') + a.fromfile(myfile('\x01', 20),2) + assert len(a)==2 and a[0]==1 and a[1]==1 + + a = self.array('h') + a.fromfile(myfile('\x01', 20),2) + assert len(a)==2 and a[0]==257 and a[1]==257 + + for i in (0,1): + a = self.array('h') + raises(EOFError, a.fromfile, myfile('\x01', 2+i),2) + assert len(a)==1 and a[0]==257 + + + def test_fromlist(self): + a = self.array('b') + raises(OverflowError, a.fromlist, [1, 2, 400]) + assert len(a) == 0 + + raises(OverflowError, a.extend, [1, 2, 400]) + assert len(a) == 2 and a[0] == 1 and a[1] == 2 + + raises(OverflowError, self.array, 'b', [1, 2, 400]) + + a = self.array('b', [1, 2]) + assert len(a) == 2 and a[0] == 1 and a[1] == 2 + + def test_fromunicode(self): + raises(ValueError, self.array('i').fromunicode, unicode('hi')) + a = self.array('u') + a.fromunicode(unicode('hi')) + assert len(a) == 2 and a[0] == 'h' and a[1]=='i' + + b = self.array('u', unicode('hi')) + assert len(b) == 2 and b[0] == 'h' and b[1]=='i' + From hakanardo at codespeak.net Mon Jul 5 18:01:45 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 5 Jul 2010 18:01:45 +0200 (CEST) Subject: [pypy-svn] r75837 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100705160145.0CB03282BF6@codespeak.net> Author: hakanardo Date: Mon Jul 5 18:01:44 2010 New Revision: 75837 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: less mallocing Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Jul 5 18:01:44 2010 @@ -172,10 +172,13 @@ if len(s)%self.itemsize !=0: msg = 'string length not a multiple of item size' raise OperationError(self.space.w_ValueError, self.space.wrap(msg)) - for i in range(len(s)/self.itemsize): + oldlen = self.len + new = len(s) / self.itemsize + self.setlen(oldlen + new) + for i in range(new): p = i * self.itemsize item=struct.unpack(self.typecode, s[p:p + self.itemsize])[0] - self.descr_append(self.space.wrap(item)) + self.descr_setitem(oldlen + i, self.space.wrap(item)) descr_fromstring.unwrap_spec = ['self', str] def descr_fromfile(self, w_f, n): @@ -196,10 +199,15 @@ descr_fromfile.unwrap_spec = ['self', W_Root, int] def descr_fromlist(self, w_lst): + space=self.space oldbuf = self.buffer oldlen = self.len try: - self.descr_extend(w_lst) + new=space.int_w(space.len(w_lst)) + self.setlen(oldlen+new) + for i in range(new): + w_item=space.getitem(w_lst, space.wrap(i)) + self.descr_setitem(oldlen + i, w_item) except OperationError: self.buffer = oldbuf self.len = oldlen Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Mon Jul 5 18:01:44 2010 @@ -187,6 +187,18 @@ a = self.array('b', [1, 2]) assert len(a) == 2 and a[0] == 1 and a[1] == 2 + a = self.array('b') + raises(OverflowError, a.fromlist, (1, 2, 400)) + assert len(a) == 0 + + raises(OverflowError, a.extend, (1, 2, 400)) + assert len(a) == 2 and a[0] == 1 and a[1] == 2 + + raises(OverflowError, self.array, 'b', (1, 2, 400)) + + a = self.array('b', (1, 2)) + assert len(a) == 2 and a[0] == 1 and a[1] == 2 + def test_fromunicode(self): raises(ValueError, self.array('i').fromunicode, unicode('hi')) a = self.array('u') From hakanardo at codespeak.net Mon Jul 5 18:05:14 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 5 Jul 2010 18:05:14 +0200 (CEST) Subject: [pypy-svn] r75838 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100705160514.35C52282BF6@codespeak.net> Author: hakanardo Date: Mon Jul 5 18:05:12 2010 New Revision: 75838 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: less mallocing Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Jul 5 18:05:12 2010 @@ -218,7 +218,7 @@ if self.typecode != 'u': msg = "fromunicode() may only be called on type 'u' arrays" raise OperationError(self.space.w_ValueError, self.space.wrap(msg)) - self.descr_extend(self.space.wrap(s)) + self.descr_fromlist(self.space.wrap(s)) descr_fromunicode.unwrap_spec = ['self', unicode] @@ -252,7 +252,7 @@ elif space.is_w(space.type(w_initializer), space.w_list): a.descr_fromlist(w_initializer) elif not space.is_w(w_initializer, space.w_None): - a.descr_extend(w_initializer) # FIXME: use fromlist, fromstring, ... + a.descr_extend(w_initializer) return a array.unwrap_spec = (ObjSpace, str, W_Root) From cfbolz at codespeak.net Mon Jul 5 18:23:25 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 5 Jul 2010 18:23:25 +0200 (CEST) Subject: [pypy-svn] r75839 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src test Message-ID: <20100705162325.8DF0A282BFC@codespeak.net> Author: cfbolz Date: Mon Jul 5 18:23:24 2010 New Revision: 75839 Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Log: (arigo, wlav, antocuni, cfbolz): hack some more until we are able to construct an object (with a complete hack) and call a method on it. This still leaks memory. Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h Mon Jul 5 18:23:24 2010 @@ -3,5 +3,6 @@ #include "Reflex/Type.h" #include "Reflex/Member.h" +#include "Reflex/Object.h" #endif // CPPYY_CPPYY Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Mon Jul 5 18:23:24 2010 @@ -4,6 +4,8 @@ extern "C" { long callstatic_l(const char* class_name, const char* method_name, int numargs, void* args[]); + void* construct(const char* class_name, int numargs, void* args[]); + long callmethod_l(const char* class_name, const char* method_name, void* self, int numargs, void* args[]); } #endif // ifndef CPPYY_REFLEXCWRAPPER Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Mon Jul 5 18:23:24 2010 @@ -28,6 +28,15 @@ "callstatic_l", [rffi.CCHARP, rffi.CCHARP, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) +construct = rffi.llexternal( + "construct", + [rffi.CCHARP, rffi.INT, rffi.VOIDPP], rffi.VOIDP, + compilation_info=eci) +callmethod_l = rffi.llexternal( + "callmethod_l", + [rffi.CCHARP, rffi.CCHARP, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.LONG, + compilation_info=eci) + def load_lib(space, name): @@ -35,6 +44,20 @@ return W_CPPLibrary(space, cdll) load_lib.unwrap_spec = [ObjSpace, str] +def prepare_arguments(space, args_w): + args = lltype.malloc(rffi.CArray(rffi.VOIDP), len(args_w), flavor='raw') + for i in range(len(args_w)): + arg = space.int_w(args_w[i]) + x = lltype.malloc(rffi.LONGP.TO, 1, flavor='raw') + x[0] = arg + args[i] = rffi.cast(rffi.VOIDP, x) + return args + +def free_arguments(args, numargs): + for i in range(numargs): + lltype.free(args[i], flavor='raw') + lltype.free(args, flavor='raw') + class W_CPPLibrary(Wrappable): def __init__(self, space, cdll): self.cdll = cdll @@ -56,23 +79,40 @@ self.name = name def invoke(self, name, args_w): - args = lltype.malloc(rffi.CArray(rffi.VOIDP), len(args_w), flavor='raw') - for i in range(len(args_w)): - arg = self.space.int_w(args_w[i]) - x = lltype.malloc(rffi.LONGP.TO, 1, flavor='raw') - x[0] = arg - args[i] = rffi.cast(rffi.VOIDP, x) + args = prepare_arguments(self.space, args_w) result = callstatic_l(self.name, name, len(args_w), args) - for i in range(len(args_w)): - lltype.free(args[i], flavor='raw') - lltype.free(args, flavor='raw') + free_arguments(args, len(args_w)) return self.space.wrap(result) def construct(self, args_w): - xxx + args = prepare_arguments(self.space, args_w) + result = construct(self.name, len(args_w), args) + free_arguments(args, len(args_w)) + return W_CPPObject(self, result) W_CPPType.typedef = TypeDef( 'CPPType', invoke = interp2app(W_CPPType.invoke, unwrap_spec=['self', str, 'args_w']), construct = interp2app(W_CPPType.construct, unwrap_spec=['self', 'args_w']), ) + + + +class W_CPPObject(Wrappable): + def __init__(self, cppclass, rawobject): + self.space = cppclass.space + self.cppclass = cppclass + self.rawobject = rawobject + + def invoke(self, method_name, args_w): + args = prepare_arguments(self.space, args_w) + result = callmethod_l(self.cppclass.name, method_name, + self.rawobject, len(args_w), args) + free_arguments(args, len(args_w)) + return self.space.wrap(result) + + +W_CPPObject.typedef = TypeDef( + 'CPPObject', + invoke = interp2app(W_CPPObject.invoke, unwrap_spec=['self', str, 'args_w']), +) Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Mon Jul 5 18:23:24 2010 @@ -2,11 +2,30 @@ #include "reflexcwrapper.h" #include -long callstatic_l(const char* classname, const char* methodname, int numargs, void* args[]) { +long callstatic_l(const char* class_name, const char* method_name, int numargs, void* args[]) { long result; std::vector arguments(args, args+numargs); - Reflex::Type t = Reflex::Type::ByName(classname); - Reflex::Member m = t.FunctionMemberByName(methodname); + Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Member m = t.FunctionMemberByName(method_name); m.Invoke(result, arguments); return result; } + +long callmethod_l(const char* class_name, const char* method_name, + void* self, int numargs, void* args[]) { + long result; + std::vector arguments(args, args+numargs); + Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Object o(t, self); + o.Invoke(method_name, result, arguments); + return result; +} + +void* construct(const char* class_name, int numargs, void* args[]) { + std::vector arguments(args, args+numargs); + Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Object r = Reflex::Object(t, t.Allocate()); + t.FunctionMemberAt(1).Invoke(r, 0, arguments); + return r.Address(); +} + Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Mon Jul 5 18:23:24 2010 @@ -20,5 +20,5 @@ def test_example01method(self): t = self.example01.type_byname("example01") instance = t.construct(7) - res = instance.invoke("add2", 4) + res = instance.invoke("add", 4) assert res == 11 From cfbolz at codespeak.net Mon Jul 5 18:48:08 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 5 Jul 2010 18:48:08 +0200 (CEST) Subject: [pypy-svn] r75843 - in pypy/branch/reflex-support/pypy/module/cppyy: include src Message-ID: <20100705164808.51E34282BFC@codespeak.net> Author: cfbolz Date: Mon Jul 5 18:48:06 2010 New Revision: 75843 Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Log: (arigo, antocuni, wlav, cfbolz): clean up the calling of constructors. Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h Mon Jul 5 18:48:06 2010 @@ -4,5 +4,6 @@ #include "Reflex/Type.h" #include "Reflex/Member.h" #include "Reflex/Object.h" +#include "Reflex/Builder/TypeBuilder.h" #endif // CPPYY_CPPYY Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Mon Jul 5 18:48:06 2010 @@ -1,6 +1,7 @@ #include "cppyy.h" #include "reflexcwrapper.h" #include +#include long callstatic_l(const char* class_name, const char* method_name, int numargs, void* args[]) { long result; @@ -24,8 +25,20 @@ void* construct(const char* class_name, int numargs, void* args[]) { std::vector arguments(args, args+numargs); Reflex::Type t = Reflex::Type::ByName(class_name); - Reflex::Object r = Reflex::Object(t, t.Allocate()); - t.FunctionMemberAt(1).Invoke(r, 0, arguments); + std::vector argtypes; + argtypes.reserve(numargs); + for (int i = 0; i < numargs; i++) { + argtypes.push_back(Reflex::Type::ByName("int")); + } + Reflex::Type constructor_type = Reflex::FunctionTypeBuilder( + Reflex::Type::ByName("void"), argtypes); + return t.Construct(constructor_type, arguments).Address(); + void* mem = t.Allocate(); + memset(mem, 41, t.SizeOf()); + Reflex::Object r = Reflex::Object(t, mem); + int i = 1; + std::cout << t.FunctionMemberAt(i).Name() << std::endl; + t.FunctionMemberAt(i).Invoke(r, 0, arguments); return r.Address(); } From benjamin at codespeak.net Mon Jul 5 18:54:13 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 5 Jul 2010 18:54:13 +0200 (CEST) Subject: [pypy-svn] r75844 - in pypy/branch/fast-forward/pypy/module/math: . test Message-ID: <20100705165413.C7521282BFC@codespeak.net> Author: benjamin Date: Mon Jul 5 18:54:12 2010 New Revision: 75844 Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py pypy/branch/fast-forward/pypy/module/math/interp_math.py pypy/branch/fast-forward/pypy/module/math/test/test_math.py Log: add math.fsum: floating point summation with partials Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/math/__init__.py Mon Jul 5 18:54:12 2010 @@ -38,5 +38,6 @@ 'isinf' : 'interp_math.isinf', 'isnan' : 'interp_math.isnan', 'trunc' : 'interp_math.trunc', + 'fsum' : 'interp_math.fsum', } Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Mon Jul 5 18:54:12 2010 @@ -299,3 +299,68 @@ """ return math1(space, math.acos, x) acos.unwrap_spec = [ObjSpace, float] + +def fsum(space, w_iterable): + """Sum an iterable of floats, trying to keep precision.""" + w_iter = space.iter(w_iterable) + inf_sum = special_sum = 0.0 + partials = [] + while True: + try: + w_value = space.next(w_iter) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break + v = space.float_w(w_value) + original = v + added = 0 + for y in partials: + if abs(v) < abs(y): + v, y = y, v + hi = v + y + yr = hi - v + lo = y - yr + if lo != 0.0: + partials[added] = lo + added += 1 + v = hi + del partials[added:] + if v != 0.0: + if rarithmetic.isinf(v) or rarithmetic.isnan(v): + if (not rarithmetic.isinf(original) and + not rarithmetic.isnan(original)): + raise OperationError(space.w_OverflowError, + space.wrap("intermediate overflow")) + if isinf(original): + inf_sum += original + special_sum += original + del partials[:] + else: + partials.append(v) + if special_sum != 0.0: + if rarithmetic.isnan(special_sum): + raise OperationError(space.w_ValueError, space.wrap("-inf + inf")) + return space.wrap(special_sum) + hi = 0.0 + if partials: + hi = partials[-1] + j = 0 + for j in range(len(partials) - 2, -1, -1): + v = hi + y = partials[j] + assert abs(y) < abs(v) + hi = v + y + yr = hi - v + lo = y - yr + if lo != 0.0: + break + if j > 0 and (lo < 0.0 and partials[j - 1] < 0.0 or + lo > 0.0 and partials[j - 1] > 0.0): + y = lo * 2.0 + v = hi + y + yr = v - hi + if y == yr: + hi = v + return space.wrap(hi) +fsum.unwrap_spec = [ObjSpace, W_Root] Modified: pypy/branch/fast-forward/pypy/module/math/test/test_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/test/test_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/test/test_math.py Mon Jul 5 18:54:12 2010 @@ -35,3 +35,41 @@ if not ok: raise AssertionError("%s(%s): got %s" % ( fnname, ', '.join(map(str, args)), got)) + + def test_fsum(self): + # Python version of math.fsum, for comparison. Uses a + # different algorithm based on frexp, ldexp and integer + # arithmetic. + import math + + test_values = [ + ([], 0.0), + ([0.0], 0.0), + ([1e100, 1.0, -1e100, 1e-100, 1e50, -1.0, -1e50], 1e-100), + ([2.0**53, -0.5, -2.0**-54], 2.0**53-1.0), + ([2.0**53, 1.0, 2.0**-100], 2.0**53+2.0), + ([2.0**53+10.0, 1.0, 2.0**-100], 2.0**53+12.0), + ([2.0**53-4.0, 0.5, 2.0**-54], 2.0**53-3.0), + ([1./n for n in range(1, 1001)], + float.fromhex('0x1.df11f45f4e61ap+2')), + ([(-1.)**n/n for n in range(1, 1001)], + float.fromhex('-0x1.62a2af1bd3624p-1')), + ([1.7**(i+1)-1.7**i for i in range(1000)] + [-1.7**1000], -1.0), + ([1e16, 1., 1e-16], 10000000000000002.0), + ([1e16-2., 1.-2.**-53, -(1e16-2.), -(1.-2.**-53)], 0.0), + # exercise code for resizing partials array + ([2.**n - 2.**(n+50) + 2.**(n+52) for n in range(-1074, 972, 2)] + + [-2.**1022], + float.fromhex('0x1.5555555555555p+970')), + ] + + for i, (vals, expected) in enumerate(test_values): + try: + actual = math.fsum(vals) + except OverflowError: + py.test.fail("test %d failed: got OverflowError, expected %r " + "for math.fsum(%.100r)" % (i, expected, vals)) + except ValueError: + py.test.fail("test %d failed: got ValueError, expected %r " + "for math.fsum(%.100r)" % (i, expected, vals)) + assert actual == expected From getxsick at codespeak.net Mon Jul 5 19:02:56 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 5 Jul 2010 19:02:56 +0200 (CEST) Subject: [pypy-svn] r75845 - in pypy/branch/fast-ctypes/pypy: module/jitffi module/jitffi/test rlib rlib/test Message-ID: <20100705170256.0AD84282BFC@codespeak.net> Author: getxsick Date: Mon Jul 5 19:02:54 2010 New Revision: 75845 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: functions with various type of arguments can be called now limitations: - can't "recall" function (stack has to be cleaned up) - rjitffi.call doesn't accept args (only module.jitffi or by using push_*()) - tests for rjitffi fails because of the above Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Mon Jul 5 19:02:54 2010 @@ -34,17 +34,27 @@ if space.is_w(w_args, space.w_None): return space.wrap(self.call()) else: - if self.args_type[0] == 'int': - args_w = [ space.int_w(w_x) for w_x in space.listview(w_args) ] - elif self.args_type[0] == 'float': - args_w = [ space.float_w(w_x) for w_x in space.listview(w_args) ] - else: - raise OperationError( - space.w_TypeError, - space.wrap('Unsupported type of argument: %s' - % self.args_type[0])) - - return space.wrap(self.call(args_w)) + i = 0 + w_iterator = space.iter(w_args) + while True: + try: + w_arg = space.next(w_iterator) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break # done + + if self.args_type[i] == 'int': + self.push_int(space.int_w(w_arg)) + elif self.args_type[i] == 'float': + self.push_float(space.float_w(w_arg)) + else: + raise OperationError( + space.w_TypeError, + space.wrap('Unsupported type of argument: %s' + % self.args_type[0])) + i += 1 + return space.wrap(self.call()) def W_Get___new__(space, w_type, cpu, lib, func, args_type, res_type): try: Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py Mon Jul 5 19:02:54 2010 @@ -20,6 +20,12 @@ return a+b; } + int add_intfloat(int a, double b) + { + int rb = (int)b; + return a+rb; + } + double return_float(int a, int b) { return a+b; @@ -46,8 +52,8 @@ ''' )) - symbols = ['add_integers', 'add_floats', 'return_float', - 'max3', 'fvoid', 'return_void'] + symbols = ['add_integers', 'add_floats', 'add_intfloat', + 'return_float', 'max3', 'fvoid', 'return_void'] eci = ExternalCompilationInfo(export_symbols=symbols) return str(platform.compile([c_file], eci, 'x', standalone=False)) @@ -90,6 +96,13 @@ func = lib.get('return_void', ['int', 'int']) assert func.call([1, 2]) is None + def test_various_type_args(self): + import jitffi + lib = jitffi.CDLL(self.lib_name) + + func = lib.get('add_intfloat', ['int', 'float'], 'int') + assert func.call([1, 2.9]) == 3 + def test_undefined_func(self): import jitffi lib = jitffi.CDLL(self.lib_name) Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Mon Jul 5 19:02:54 2010 @@ -33,6 +33,7 @@ self.res_type = res_type self.cpu = cpu self.lib = lib.handler + self.esp = 1 # 0 is a func addr if self.res_type == 'int': self.bres = BoxInt() @@ -70,19 +71,10 @@ FUNC = deref(FPTR) self.calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) - def call(self, func_args=None): - if func_args is None: - func_args = [] - - bargs = [] - for tp, value in zip(self.args_type, func_args): - if tp == 'int': - bargs.append(BoxInt(value)) - elif tp == 'float': - bargs.append(BoxFloat(value)) - elif tp == 'ref': - bargs.append(BoxPtr(value)) - inputargs = [self.bfuncaddr] + bargs + self.bargs = [] + + def call(self): + inputargs = [self.bfuncaddr] + self.bargs oplist = [ResOperation(rop.CALL, inputargs, self.bres, descr=self.calldescr), @@ -90,16 +82,7 @@ descr=BasicFailDescr(0))] looptoken = LoopToken() self.cpu.compile_loop(inputargs, oplist, looptoken) - - for i, box in enumerate(inputargs): - if i == 0: # func address - self.cpu.set_future_value_int(i, box.getint()) - elif self.args_type[i-1] == 'int': - self.cpu.set_future_value_int(i, box.getint()) - elif self.args_type[i-1] == 'float': - self.cpu.set_future_value_float(i, box.getfloat()) - elif self.args_type[i-1] == 'ref': - self.cpu.set_future_value_ref(i, box.getref()) + self.cpu.set_future_value_int(0, self.bfuncaddr.getint()) res = self.cpu.execute_token(looptoken) if res is oplist[-1].descr: @@ -118,3 +101,13 @@ else: raise ValueError(self.res_type) return r + + def push_int(self, value): + self.cpu.set_future_value_int(self.esp, value) + self.bargs.append(BoxInt(value)) + self.esp += 1 + + def push_float(self, value): + self.cpu.set_future_value_float(self.esp, value) + self.bargs.append(BoxFloat(value)) + self.esp += 1 Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Mon Jul 5 19:02:54 2010 @@ -21,6 +21,12 @@ return a+b; } + int add_intfloat(int a, double b) + { + int rb = (int)b; + return a+rb; + } + double return_float(int a, int b) { return a+b; @@ -47,8 +53,8 @@ ''' )) - symbols = ['add_integers', 'add_floats', 'return_float', - 'max3', 'fvoid', 'return_void'] + symbols = ['add_integers', 'add_floats', 'add_intfloat', + 'return_float', 'max3', 'fvoid', 'return_void'] eci = ExternalCompilationInfo(export_symbols=symbols) return str(platform.compile([c_file], eci, 'x', standalone=False)) @@ -86,6 +92,12 @@ func = lib.get('return_void', ['int', 'int']) assert func.call([1, 2]) is None + def test_various_type_args(self): + lib = rjitffi.CDLL(self.lib_name) + + func = lib.get('add_intfloat', ['int', 'float'], 'int') + assert func.call([1, 2.9]) == 3 + def test_undefined_func(self): lib = rjitffi.CDLL(self.lib_name) # xxxfoo888baryyy - not existed function From getxsick at codespeak.net Mon Jul 5 19:08:49 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 5 Jul 2010 19:08:49 +0200 (CEST) Subject: [pypy-svn] r75846 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100705170849.2562E282BFC@codespeak.net> Author: getxsick Date: Mon Jul 5 19:08:47 2010 New Revision: 75846 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: tests for rjitffi pass now Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Mon Jul 5 19:08:47 2010 @@ -73,7 +73,14 @@ self.bargs = [] - def call(self): + def call(self, func_args=None): + if func_args is not None: + for tp, value in zip(self.args_type, func_args): + if tp == 'int': + self.push_int(value) + elif tp == 'float': + self.push_float(value) + inputargs = [self.bfuncaddr] + self.bargs oplist = [ResOperation(rop.CALL, inputargs, self.bres, From cfbolz at codespeak.net Mon Jul 5 19:09:54 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 5 Jul 2010 19:09:54 +0200 (CEST) Subject: [pypy-svn] r75847 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src test Message-ID: <20100705170954.AA627282BFC@codespeak.net> Author: cfbolz Date: Mon Jul 5 19:09:53 2010 New Revision: 75847 Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Log: (antocuni, cfbolz, wlav, arigo): call the destructor too. Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Mon Jul 5 19:09:53 2010 @@ -4,8 +4,9 @@ extern "C" { long callstatic_l(const char* class_name, const char* method_name, int numargs, void* args[]); - void* construct(const char* class_name, int numargs, void* args[]); long callmethod_l(const char* class_name, const char* method_name, void* self, int numargs, void* args[]); + void* construct(const char* class_name, int numargs, void* args[]); + void destruct(const char* class_name, void* self); } #endif // ifndef CPPYY_REFLEXCWRAPPER Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Mon Jul 5 19:09:53 2010 @@ -36,6 +36,10 @@ "callmethod_l", [rffi.CCHARP, rffi.CCHARP, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) +destruct = rffi.llexternal( + "destruct", + [rffi.CCHARP, rffi.VOIDP], lltype.Void, + compilation_info=eci) @@ -111,8 +115,11 @@ free_arguments(args, len(args_w)) return self.space.wrap(result) + def destruct(self): + destruct(self.cppclass.name, self.rawobject) W_CPPObject.typedef = TypeDef( 'CPPObject', invoke = interp2app(W_CPPObject.invoke, unwrap_spec=['self', str, 'args_w']), + destruct = interp2app(W_CPPObject.destruct, unwrap_spec=['self']), ) Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Mon Jul 5 19:09:53 2010 @@ -33,12 +33,9 @@ Reflex::Type constructor_type = Reflex::FunctionTypeBuilder( Reflex::Type::ByName("void"), argtypes); return t.Construct(constructor_type, arguments).Address(); - void* mem = t.Allocate(); - memset(mem, 41, t.SizeOf()); - Reflex::Object r = Reflex::Object(t, mem); - int i = 1; - std::cout << t.FunctionMemberAt(i).Name() << std::endl; - t.FunctionMemberAt(i).Invoke(r, 0, arguments); - return r.Address(); } +void destruct(const char* class_name, void* self) { + Reflex::Type t = Reflex::Type::ByName(class_name); + t.Destruct(self, true); +} Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx Mon Jul 5 19:09:53 2010 @@ -2,15 +2,40 @@ class example01 { public: + static int count; int somedata; + + example01() : somedata(-99) { + count++; + } example01(int a) : somedata(a) { + count++; std::cout << "constructor called" << std::endl; } + example01(const example01& e) : somedata(e.somedata) { + count++; + std::cout << "copy constructor called" << std::endl; + } + example01& operator=(const example01& e) { + if (this != &e) { + somedata = e.somedata; + } + return *this; + } + ~example01() { + count--; + } static int add1(int a) { return a + 1; } + static int getcount() { + std::cout << "getcount called" << std::endl; + return count; + } int add(int a) { return somedata + a; } }; + +int example01::count = 0; Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Mon Jul 5 19:09:53 2010 @@ -19,6 +19,13 @@ def test_example01method(self): t = self.example01.type_byname("example01") + count = t.invoke("getcount") + assert count == 0 instance = t.construct(7) + count = t.invoke("getcount") + assert count == 1 res = instance.invoke("add", 4) assert res == 11 + instance.destruct() + count = t.invoke("getcount") + assert count == 0 From getxsick at codespeak.net Mon Jul 5 19:18:29 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 5 Jul 2010 19:18:29 +0200 (CEST) Subject: [pypy-svn] r75848 - in pypy/branch/fast-ctypes/pypy: module/jitffi/test rlib rlib/test Message-ID: <20100705171829.786E8282BFC@codespeak.net> Author: getxsick Date: Mon Jul 5 19:18:28 2010 New Revision: 75848 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: functions can be multiple called now Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py Mon Jul 5 19:18:28 2010 @@ -102,6 +102,7 @@ func = lib.get('add_intfloat', ['int', 'float'], 'int') assert func.call([1, 2.9]) == 3 + assert func.call([0, 1.3]) == 1 def test_undefined_func(self): import jitffi Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Mon Jul 5 19:18:28 2010 @@ -33,7 +33,7 @@ self.res_type = res_type self.cpu = cpu self.lib = lib.handler - self.esp = 1 # 0 is a func addr + self.setup_stack() if self.res_type == 'int': self.bres = BoxInt() @@ -71,8 +71,6 @@ FUNC = deref(FPTR) self.calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) - self.bargs = [] - def call(self, func_args=None): if func_args is not None: for tp, value in zip(self.args_type, func_args): @@ -97,6 +95,8 @@ else: self.guard_failed = True + self.setup_stack() # clean up the stack + if self.res_type == 'int': r = BoxInt(self.cpu.get_latest_value_int(0)).getint() elif self.res_type == 'float': @@ -109,6 +109,10 @@ raise ValueError(self.res_type) return r + def setup_stack(self): + self.bargs = [] + self.esp = 1 # 0 is a func addr + def push_int(self, value): self.cpu.set_future_value_int(self.esp, value) self.bargs.append(BoxInt(value)) Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Mon Jul 5 19:18:28 2010 @@ -97,6 +97,7 @@ func = lib.get('add_intfloat', ['int', 'float'], 'int') assert func.call([1, 2.9]) == 3 + assert func.call([0, 1.3]) == 1 def test_undefined_func(self): lib = rjitffi.CDLL(self.lib_name) From getxsick at codespeak.net Mon Jul 5 19:19:40 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 5 Jul 2010 19:19:40 +0200 (CEST) Subject: [pypy-svn] r75849 - in pypy/branch/fast-ctypes/pypy: module/jitffi rlib Message-ID: <20100705171940.33554282BFC@codespeak.net> Author: getxsick Date: Mon Jul 5 19:19:38 2010 New Revision: 75849 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: add dummy support for C pointers. it shouldn't work anyway. Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Mon Jul 5 19:19:38 2010 @@ -48,6 +48,8 @@ self.push_int(space.int_w(w_arg)) elif self.args_type[i] == 'float': self.push_float(space.float_w(w_arg)) + elif self.args_type[i] == 'ref': + self.push_ref(space.int_w(w_arg)) else: raise OperationError( space.w_TypeError, Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Mon Jul 5 19:19:38 2010 @@ -78,6 +78,8 @@ self.push_int(value) elif tp == 'float': self.push_float(value) + elif tp == 'ref': + self.push_ref(value) inputargs = [self.bfuncaddr] + self.bargs @@ -122,3 +124,8 @@ self.cpu.set_future_value_float(self.esp, value) self.bargs.append(BoxFloat(value)) self.esp += 1 + + def push_ref(self, value): + self.cpu.set_future_value_ref(self.esp, value) + self.bargs.append(BoxPtr(value)) + self.esp += 1 From benjamin at codespeak.net Mon Jul 5 19:58:46 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 5 Jul 2010 19:58:46 +0200 (CEST) Subject: [pypy-svn] r75850 - pypy/branch/fast-forward/lib-python Message-ID: <20100705175846.49126282BFC@codespeak.net> Author: benjamin Date: Mon Jul 5 19:58:44 2010 New Revision: 75850 Modified: pypy/branch/fast-forward/lib-python/ (props changed) Log: update 2.7 lib From getxsick at codespeak.net Mon Jul 5 20:00:53 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 5 Jul 2010 20:00:53 +0200 (CEST) Subject: [pypy-svn] r75851 - in pypy/branch/fast-ctypes/pypy/rlib: . test Message-ID: <20100705180053.B9045282BFC@codespeak.net> Author: getxsick Date: Mon Jul 5 20:00:52 2010 New Revision: 75851 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: remove func_args as it's not RPython. update tests to use push_*() directly Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Mon Jul 5 20:00:52 2010 @@ -71,16 +71,7 @@ FUNC = deref(FPTR) self.calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) - def call(self, func_args=None): - if func_args is not None: - for tp, value in zip(self.args_type, func_args): - if tp == 'int': - self.push_int(value) - elif tp == 'float': - self.push_float(value) - elif tp == 'ref': - self.push_ref(value) - + def call(self): inputargs = [self.bfuncaddr] + self.bargs oplist = [ResOperation(rop.CALL, inputargs, self.bres, Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Mon Jul 5 20:00:52 2010 @@ -69,35 +69,59 @@ lib = rjitffi.CDLL(self.lib_name) func = lib.get('add_integers', ['int', 'int'], 'int') - assert 3 == func.call([1,2]) + func.push_int(1) + func.push_int(2) + assert func.call() == 3 + func = lib.get('add_integers', ['int', 'int'], 'int') - assert 1 == func.call([-1,2]) + func.push_int(-1) + func.push_int(2) + assert func.call() == 1 + func = lib.get('add_integers', ['int', 'int'], 'int') - assert 0 == func.call([0,0]) + func.push_int(0) + func.push_int(0) + assert func.call() == 0 func = lib.get('max3', ['int', 'int', 'int'], 'int') - assert 8 == func.call([2, 8, 3]) + func.push_int(2) + func.push_int(8) + func.push_int(3) + assert func.call() == 8 func = lib.get('add_floats', ['float', 'float'], 'float') - assert 2.7 == func.call([1.2, 1.5]) + func.push_float(1.2) + func.push_float(1.5) + assert func.call() == 2.7 def test_get_void(self): lib = rjitffi.CDLL(self.lib_name) func = lib.get('fvoid', [], 'int') - assert 1 == func.call() + assert func.call() == 1 func = lib.get('return_void', ['int', 'int'], 'void') - assert func.call([1, 2]) is None + func.push_int(1) + func.push_int(2) + assert func.call() is None + func = lib.get('return_void', ['int', 'int']) - assert func.call([1, 2]) is None + func.push_int(1) + func.push_int(2) + assert func.call() is None def test_various_type_args(self): lib = rjitffi.CDLL(self.lib_name) func = lib.get('add_intfloat', ['int', 'float'], 'int') - assert func.call([1, 2.9]) == 3 - assert func.call([0, 1.3]) == 1 + func.push_int(1) + func.push_float(2.9) + assert func.call() == 3 + + # stack is cleaned up after calling + func.push_int(0) + func.push_float(1.3) + assert func.call() == 1 def test_undefined_func(self): lib = rjitffi.CDLL(self.lib_name) From getxsick at codespeak.net Mon Jul 5 20:19:52 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 5 Jul 2010 20:19:52 +0200 (CEST) Subject: [pypy-svn] r75853 - in pypy/branch/fast-ctypes/pypy: module/jitffi module/jitffi/test rlib rlib/test Message-ID: <20100705181952.28E05282BF6@codespeak.net> Author: getxsick Date: Mon Jul 5 20:19:51 2010 New Revision: 75853 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: use short names of types Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Mon Jul 5 20:19:51 2010 @@ -26,7 +26,7 @@ class W_Get(Wrappable, rjitffi._Get): - def __init__(self, space, cpu, lib, func, args_type, res_type='void'): + def __init__(self, space, cpu, lib, func, args_type, res_type='v'): self.space = space rjitffi._Get.__init__(self, cpu, lib, func, args_type, res_type) @@ -44,11 +44,11 @@ raise break # done - if self.args_type[i] == 'int': + if self.args_type[i] == 'i': self.push_int(space.int_w(w_arg)) - elif self.args_type[i] == 'float': + elif self.args_type[i] == 'f': self.push_float(space.float_w(w_arg)) - elif self.args_type[i] == 'ref': + elif self.args_type[i] == 'p': self.push_ref(space.int_w(w_arg)) else: raise OperationError( @@ -80,7 +80,7 @@ except OSError, e: raise OperationError(space.w_OSError, space.wrap(str(e))) - def get_w(self, space, func, w_args_type, res_type='void'): + def get_w(self, space, func, w_args_type, res_type='v'): args_type_w = [ space.str_w(w_x) for w_x in space.listview(w_args_type) ] try: Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py Mon Jul 5 20:19:51 2010 @@ -71,36 +71,36 @@ import jitffi lib = jitffi.CDLL(self.lib_name) - func = lib.get('add_integers', ['int', 'int'], 'int') + func = lib.get('add_integers', ['i', 'i'], 'i') assert 3 == func.call([1,2]) - func = lib.get('add_integers', ['int', 'int'], 'int') + func = lib.get('add_integers', ['i', 'i'], 'i') assert 1 == func.call([-1,2]) - func = lib.get('add_integers', ['int', 'int'], 'int') + func = lib.get('add_integers', ['i', 'i'], 'i') assert 0 == func.call([0,0]) - func = lib.get('max3', ['int', 'int', 'int'], 'int') + func = lib.get('max3', ['i', 'i', 'i'], 'i') assert 8 == func.call([2, 8, 3]) - func = lib.get('add_floats', ['float', 'float'], 'float') + func = lib.get('add_floats', ['f', 'f'], 'f') assert 2.7 == func.call([1.2, 1.5]) def test_get_void(self): import jitffi lib = jitffi.CDLL(self.lib_name) - func = lib.get('fvoid', [], 'int') + func = lib.get('fvoid', [], 'i') assert 1 == func.call() - func = lib.get('return_void', ['int', 'int'], 'void') + func = lib.get('return_void', ['i', 'i'], 'v') assert func.call([1, 2]) is None - func = lib.get('return_void', ['int', 'int']) + func = lib.get('return_void', ['i', 'i']) assert func.call([1, 2]) is None def test_various_type_args(self): import jitffi lib = jitffi.CDLL(self.lib_name) - func = lib.get('add_intfloat', ['int', 'float'], 'int') + func = lib.get('add_intfloat', ['i', 'f'], 'i') assert func.call([1, 2.9]) == 3 assert func.call([0, 1.3]) == 1 @@ -109,13 +109,13 @@ lib = jitffi.CDLL(self.lib_name) # xxxfoo888baryyy - not existed function raises(ValueError, lib.get, 'xxxfoo888baryyy', []) - raises(ValueError, lib.get, 'xxxfoo888baryyy', ['int'], 'int') + raises(ValueError, lib.get, 'xxxfoo888baryyy', ['i'], 'i') def test_unknown_types(self): import jitffi lib = jitffi.CDLL(self.lib_name) # xxxfoo888baryyy - not defined types (args_type, res_type etc.) raises(ValueError, lib.get, 'fvoid', ['xxxfoo888baryyy']) - raises(ValueError, lib.get, 'fvoid', ['int','xxxfoo888baryyy']) - raises(ValueError, lib.get, 'fvoid', ['xxxfoo888baryyy'],'int') + raises(ValueError, lib.get, 'fvoid', ['i','xxxfoo888baryyy']) + raises(ValueError, lib.get, 'fvoid', ['xxxfoo888baryyy'],'i') raises(ValueError, lib.get, 'fvoid', [], 'xxxfoo888baryyy') Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Mon Jul 5 20:19:51 2010 @@ -16,7 +16,7 @@ self.name = name self.cpu = CPU(None, None) - def get(self, func, args_type, res_type='void'): + def get(self, func, args_type, res_type='v'): return _Get(self.cpu, self.lib, func, args_type, res_type) class _LibHandler(object): @@ -27,7 +27,7 @@ raise OSError('%s: %s', name, e.msg or 'unspecified error') class _Get(object): - def __init__(self, cpu, lib, func, args_type, res_type='void'): + def __init__(self, cpu, lib, func, args_type, res_type='v'): assert isinstance(args_type, list) self.args_type = args_type self.res_type = res_type @@ -35,16 +35,16 @@ self.lib = lib.handler self.setup_stack() - if self.res_type == 'int': + if self.res_type == 'i': self.bres = BoxInt() res = lltype.Signed - elif self.res_type == 'float': + elif self.res_type == 'f': self.bres = BoxFloat() res = lltype.Float - elif self.res_type == 'ref': + elif self.res_type == 'p': self.bres = BoxPtr() res = lltype.Signed - elif self.res_type == 'void': + elif self.res_type == 'v': self.bres = NULLBOX res = lltype.Void else: @@ -58,11 +58,11 @@ args = [] for arg in self.args_type: - if arg == 'int': + if arg == 'i': args.append(lltype.Signed) - elif arg == 'float': + elif arg == 'f': args.append(lltype.Float) - elif arg == 'ref': + elif arg == 'p': args.append(lltype.Signed) else: raise ValueError(arg) @@ -90,13 +90,13 @@ self.setup_stack() # clean up the stack - if self.res_type == 'int': + if self.res_type == 'i': r = BoxInt(self.cpu.get_latest_value_int(0)).getint() - elif self.res_type == 'float': + elif self.res_type == 'f': r = BoxFloat(self.cpu.get_latest_value_float(0)).getfloat() - elif self.res_type == 'ref': + elif self.res_type == 'p': r = BoxPtr(self.cpu.get_latest_value_ref(0)).getref() - elif self.res_type == 'void': + elif self.res_type == 'v': r = None else: raise ValueError(self.res_type) Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Mon Jul 5 20:19:51 2010 @@ -68,28 +68,28 @@ def test_get(self): lib = rjitffi.CDLL(self.lib_name) - func = lib.get('add_integers', ['int', 'int'], 'int') + func = lib.get('add_integers', ['i', 'i'], 'i') func.push_int(1) func.push_int(2) assert func.call() == 3 - func = lib.get('add_integers', ['int', 'int'], 'int') + func = lib.get('add_integers', ['i', 'i'], 'i') func.push_int(-1) func.push_int(2) assert func.call() == 1 - func = lib.get('add_integers', ['int', 'int'], 'int') + func = lib.get('add_integers', ['i', 'i'], 'i') func.push_int(0) func.push_int(0) assert func.call() == 0 - func = lib.get('max3', ['int', 'int', 'int'], 'int') + func = lib.get('max3', ['i', 'i', 'i'], 'i') func.push_int(2) func.push_int(8) func.push_int(3) assert func.call() == 8 - func = lib.get('add_floats', ['float', 'float'], 'float') + func = lib.get('add_floats', ['f', 'f'], 'f') func.push_float(1.2) func.push_float(1.5) assert func.call() == 2.7 @@ -97,15 +97,15 @@ def test_get_void(self): lib = rjitffi.CDLL(self.lib_name) - func = lib.get('fvoid', [], 'int') + func = lib.get('fvoid', [], 'i') assert func.call() == 1 - func = lib.get('return_void', ['int', 'int'], 'void') + func = lib.get('return_void', ['i', 'i'], 'v') func.push_int(1) func.push_int(2) assert func.call() is None - func = lib.get('return_void', ['int', 'int']) + func = lib.get('return_void', ['i', 'i']) func.push_int(1) func.push_int(2) assert func.call() is None @@ -113,7 +113,7 @@ def test_various_type_args(self): lib = rjitffi.CDLL(self.lib_name) - func = lib.get('add_intfloat', ['int', 'float'], 'int') + func = lib.get('add_intfloat', ['i', 'f'], 'i') func.push_int(1) func.push_float(2.9) assert func.call() == 3 @@ -127,12 +127,12 @@ lib = rjitffi.CDLL(self.lib_name) # xxxfoo888baryyy - not existed function py.test.raises(ValueError, lib.get, 'xxxfoo888baryyy', []) - py.test.raises(ValueError, lib.get, 'xxxfoo888baryyy', ['int'], 'int') + py.test.raises(ValueError, lib.get, 'xxxfoo888baryyy', ['i'], 'i') def test_unknown_types(self): lib = rjitffi.CDLL(self.lib_name) # xxxfoo888baryyy - not defined types (args_type, res_type etc.) py.test.raises(ValueError, lib.get, 'fvoid', ['xxxfoo888baryyy']) - py.test.raises(ValueError, lib.get, 'fvoid', ['int','xxxfoo888baryyy']) - py.test.raises(ValueError, lib.get, 'fvoid', ['xxxfoo888baryyy'],'int') + py.test.raises(ValueError, lib.get, 'fvoid', ['i','xxxfoo888baryyy']) + py.test.raises(ValueError, lib.get, 'fvoid', ['xxxfoo888baryyy'],'i') py.test.raises(ValueError, lib.get, 'fvoid', [], 'xxxfoo888baryyy') From benjamin at codespeak.net Mon Jul 5 20:21:26 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 5 Jul 2010 20:21:26 +0200 (CEST) Subject: [pypy-svn] r75854 - in pypy/branch/fast-forward/pypy/module/__builtin__: . test Message-ID: <20100705182126.87682282BF6@codespeak.net> Author: benjamin Date: Mon Jul 5 20:21:24 2010 New Revision: 75854 Added: pypy/branch/fast-forward/pypy/module/__builtin__/app_operation.py (contents, props changed) Modified: pypy/branch/fast-forward/pypy/module/__builtin__/__init__.py pypy/branch/fast-forward/pypy/module/__builtin__/test/test_builtin.py Log: add bin() Modified: pypy/branch/fast-forward/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/__builtin__/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/__builtin__/__init__.py Mon Jul 5 20:21:24 2010 @@ -37,6 +37,8 @@ 'vars' : 'app_inspect.vars', 'dir' : 'app_inspect.dir', + 'bin' : 'app_operation.bin', + '__filestub' : 'app_file_stub.file', } Added: pypy/branch/fast-forward/pypy/module/__builtin__/app_operation.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/pypy/module/__builtin__/app_operation.py Mon Jul 5 20:21:24 2010 @@ -0,0 +1,4 @@ +def bin(x): + if not isinstance(x, (int, long)): + raise TypeError("must be int or long") + return format(x, "#b") Modified: pypy/branch/fast-forward/pypy/module/__builtin__/test/test_builtin.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/__builtin__/test/test_builtin.py (original) +++ pypy/branch/fast-forward/pypy/module/__builtin__/test/test_builtin.py Mon Jul 5 20:21:24 2010 @@ -32,6 +32,13 @@ raises(ValueError, chr, -1) raises(TypeError, chr, 'a') + def test_bin(self): + assert bin(0) == "0b0" + assert bin(-1) == "-0b1" + assert bin(2L) == "0b10" + assert bin(-2L) == "-0b10" + raises(TypeError, bin, 0.) + def test_unichr(self): import sys assert unichr(65) == u'A' From benjamin at codespeak.net Mon Jul 5 20:41:08 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 5 Jul 2010 20:41:08 +0200 (CEST) Subject: [pypy-svn] r75855 - in pypy/branch/fast-forward/pypy/module/math: . test Message-ID: <20100705184108.C3CCF282BF6@codespeak.net> Author: benjamin Date: Mon Jul 5 20:41:07 2010 New Revision: 75855 Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py pypy/branch/fast-forward/pypy/module/math/test/test_math.py Log: passing a large long to ldexp changes it to LONG_MAX Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Mon Jul 5 20:41:07 2010 @@ -1,4 +1,5 @@ import math +import sys from pypy.rlib import rarithmetic from pypy.interpreter.error import OperationError @@ -84,11 +85,25 @@ return math1(space, math.cosh, x) cosh.unwrap_spec = [ObjSpace, float] -def ldexp(space, x, i): +def ldexp(space, x, w_i): """ldexp(x, i) -> x * (2**i) """ - return math2(space, math.ldexp, x, i) -ldexp.unwrap_spec = [ObjSpace, float, int] + if (space.isinstance_w(w_i, space.w_int) or + space.isinstance_w(w_i, space.w_long)): + try: + exp = space.int_w(w_i) + except OperationError, e: + if not e.match(space, space.w_OverflowError): + raise + if space.is_true(space.lt(w_i, space.wrap(0))): + exp = -sys.maxint + else: + exp = sys.maxint + else: + raise OperationError(space.w_TypeError, + space.wrap("integer required for second argument")) + return math2(space, math.ldexp, x, exp) +ldexp.unwrap_spec = [ObjSpace, float, W_Root] def hypot(space, x, y): """hypot(x,y) Modified: pypy/branch/fast-forward/pypy/module/math/test/test_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/test/test_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/test/test_math.py Mon Jul 5 20:41:07 2010 @@ -36,6 +36,10 @@ raise AssertionError("%s(%s): got %s" % ( fnname, ', '.join(map(str, args)), got)) + def test_ldexp(self): + import math + assert math.ldexp(float("inf"), -10**20) == float("inf") + def test_fsum(self): # Python version of math.fsum, for comparison. Uses a # different algorithm based on frexp, ldexp and integer From benjamin at codespeak.net Mon Jul 5 20:51:02 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 5 Jul 2010 20:51:02 +0200 (CEST) Subject: [pypy-svn] r75856 - in pypy/branch/fast-forward/pypy/module/math: . test Message-ID: <20100705185102.747E6282BF6@codespeak.net> Author: benjamin Date: Mon Jul 5 20:50:59 2010 New Revision: 75856 Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py pypy/branch/fast-forward/pypy/module/math/interp_math.py pypy/branch/fast-forward/pypy/module/math/test/test_math.py Log: add math.factorial Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/math/__init__.py Mon Jul 5 20:50:59 2010 @@ -39,5 +39,6 @@ 'isnan' : 'interp_math.isnan', 'trunc' : 'interp_math.trunc', 'fsum' : 'interp_math.fsum', + 'factorial' : 'interp_math.factorial', } Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Mon Jul 5 20:50:59 2010 @@ -379,3 +379,19 @@ hi = v return space.wrap(hi) fsum.unwrap_spec = [ObjSpace, W_Root] + +def factorial(space, w_x): + """Find x!.""" + if space.isinstance_w(w_x, space.w_float): + fl = space.float_w(w_x) + if math.floor(fl) != fl: + raise OperationError(space.w_ValueError, + space.wrap("float arguments must be integral")) + w_x = space.long(w_x) + x = space.int_w(w_x) + if x < 0: + raise OperationError(space.w_ValueError, space.wrap("x must be >= 0")) + w_res = space.wrap(1) + for i in range(1, x + 1): + w_res = space.mul(w_res, space.wrap(i)) + return w_res Modified: pypy/branch/fast-forward/pypy/module/math/test/test_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/test/test_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/test/test_math.py Mon Jul 5 20:50:59 2010 @@ -41,9 +41,6 @@ assert math.ldexp(float("inf"), -10**20) == float("inf") def test_fsum(self): - # Python version of math.fsum, for comparison. Uses a - # different algorithm based on frexp, ldexp and integer - # arithmetic. import math test_values = [ @@ -77,3 +74,14 @@ py.test.fail("test %d failed: got ValueError, expected %r " "for math.fsum(%.100r)" % (i, expected, vals)) assert actual == expected + + def test_factorial(self): + import math + assert math.factorial(0) == 1 + assert math.factorial(1) == 1 + assert math.factorial(2) == 2 + assert math.factorial(5) == 120 + assert math.factorial(5.) == 120 + raises(ValueError, math.factorial, -1) + raises(ValueError, math.factorial, -1.) + raises(ValueError, math.factorial, 1.1) From getxsick at codespeak.net Mon Jul 5 20:53:33 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 5 Jul 2010 20:53:33 +0200 (CEST) Subject: [pypy-svn] r75857 - pypy/branch/fast-ctypes/pypy/module/jitffi Message-ID: <20100705185333.4D318282BF6@codespeak.net> Author: getxsick Date: Mon Jul 5 20:53:31 2010 New Revision: 75857 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Log: avoid multiple inheritance as it's not RPython Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Mon Jul 5 20:53:31 2010 @@ -4,13 +4,14 @@ from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef -class W_LibHandler(Wrappable, rjitffi._LibHandler): +class W_LibHandler(Wrappable): def __init__(self, space, name): self.space = space try: - rjitffi._LibHandler.__init__(self, name) + self.rlibhandler = rjitffi._LibHandler(name) except OSError, e: raise OperationError(space.w_OSError, space.wrap(str(e))) + self.handler = self.rlibhandler.handler def W_LibHandler___new__(space, w_type, name): try: @@ -25,14 +26,14 @@ ) -class W_Get(Wrappable, rjitffi._Get): +class W_Get(Wrappable): def __init__(self, space, cpu, lib, func, args_type, res_type='v'): self.space = space - rjitffi._Get.__init__(self, cpu, lib, func, args_type, res_type) + self.rget = rjitffi._Get(cpu, lib, func, args_type, res_type) def call_w(self, space, w_args=None): if space.is_w(w_args, space.w_None): - return space.wrap(self.call()) + return space.wrap(self.rget.call()) else: i = 0 w_iterator = space.iter(w_args) @@ -44,19 +45,19 @@ raise break # done - if self.args_type[i] == 'i': - self.push_int(space.int_w(w_arg)) - elif self.args_type[i] == 'f': - self.push_float(space.float_w(w_arg)) - elif self.args_type[i] == 'p': - self.push_ref(space.int_w(w_arg)) + if self.rget.args_type[i] == 'i': + self.rget.push_int(space.int_w(w_arg)) + elif self.rget.args_type[i] == 'f': + self.rget.push_float(space.float_w(w_arg)) + elif self.rget.args_type[i] == 'p': + self.rget.push_ref(space.int_w(w_arg)) else: raise OperationError( space.w_TypeError, space.wrap('Unsupported type of argument: %s' % self.args_type[0])) i += 1 - return space.wrap(self.call()) + return space.wrap(self.rget.call()) def W_Get___new__(space, w_type, cpu, lib, func, args_type, res_type): try: @@ -71,10 +72,10 @@ ) -class W_CDLL(Wrappable, rjitffi.CDLL): +class W_CDLL(Wrappable): def __init__(self, space, name): self.space = space - rjitffi.CDLL.__init__(self, name, load=False) + self.rcdll = rjitffi.CDLL(name, load=False) try: self.lib_w = W_LibHandler(self.space, name) except OSError, e: @@ -84,7 +85,7 @@ args_type_w = [ space.str_w(w_x) for w_x in space.listview(w_args_type) ] try: - ret = W_Get(space, self.cpu, space.wrap(self.lib_w), + ret = W_Get(space, self.rcdll.cpu, space.wrap(self.lib_w), func, args_type_w, res_type) except ValueError, e: raise OperationError(space.w_ValueError, space.wrap(str(e))) From getxsick at codespeak.net Mon Jul 5 20:56:10 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 5 Jul 2010 20:56:10 +0200 (CEST) Subject: [pypy-svn] r75858 - pypy/branch/fast-ctypes/pypy/module/jitffi Message-ID: <20100705185610.58E03282BF6@codespeak.net> Author: getxsick Date: Mon Jul 5 20:56:08 2010 New Revision: 75858 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Log: put in order classes in the source file Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Mon Jul 5 20:56:08 2010 @@ -4,6 +4,42 @@ from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef +class W_CDLL(Wrappable): + def __init__(self, space, name): + self.space = space + self.rcdll = rjitffi.CDLL(name, load=False) + try: + self.lib_w = W_LibHandler(self.space, name) + except OSError, e: + raise OperationError(space.w_OSError, space.wrap(str(e))) + + def get_w(self, space, func, w_args_type, res_type='v'): + args_type_w = [ space.str_w(w_x) + for w_x in space.listview(w_args_type) ] + try: + ret = W_Get(space, self.rcdll.cpu, space.wrap(self.lib_w), + func, args_type_w, res_type) + except ValueError, e: + raise OperationError(space.w_ValueError, space.wrap(str(e))) + return space.wrap(ret) + +def W_CDLL___new__(space, w_type, name): + try: + return space.wrap(W_CDLL(space, name)) + except OSError, e: + raise wrap_oserror(space, e) + +W_CDLL.typedef = TypeDef( + 'CDLL', + __new__ = interp2app(W_CDLL___new__,unwrap_spec=[ObjSpace,W_Root,str]), + get = interp2app(W_CDLL.get_w, unwrap_spec=['self', + ObjSpace, str, W_Root, str]), + __doc__ = """ C Dynamically loaded library +use CDLL(libname) to create a handle to a C library (the argument is processed +the same way as dlopen processes it).""" +) + + class W_LibHandler(Wrappable): def __init__(self, space, name): self.space = space @@ -70,39 +106,3 @@ #__new__ = interp2app(W_Get___new__, unwrap_spec=[ObjSpace, W_Root, W_Root, W_Root, str, W_Root, str]), call = interp2app(W_Get.call_w, unwrap_spec=['self', ObjSpace, W_Root]) ) - - -class W_CDLL(Wrappable): - def __init__(self, space, name): - self.space = space - self.rcdll = rjitffi.CDLL(name, load=False) - try: - self.lib_w = W_LibHandler(self.space, name) - except OSError, e: - raise OperationError(space.w_OSError, space.wrap(str(e))) - - def get_w(self, space, func, w_args_type, res_type='v'): - args_type_w = [ space.str_w(w_x) - for w_x in space.listview(w_args_type) ] - try: - ret = W_Get(space, self.rcdll.cpu, space.wrap(self.lib_w), - func, args_type_w, res_type) - except ValueError, e: - raise OperationError(space.w_ValueError, space.wrap(str(e))) - return space.wrap(ret) - -def W_CDLL___new__(space, w_type, name): - try: - return space.wrap(W_CDLL(space, name)) - except OSError, e: - raise wrap_oserror(space, e) - -W_CDLL.typedef = TypeDef( - 'CDLL', - __new__ = interp2app(W_CDLL___new__,unwrap_spec=[ObjSpace,W_Root,str]), - get = interp2app(W_CDLL.get_w, unwrap_spec=['self', - ObjSpace, str, W_Root, str]), - __doc__ = """ C Dynamically loaded library -use CDLL(libname) to create a handle to a C library (the argument is processed -the same way as dlopen processes it).""" -) From hakanardo at codespeak.net Mon Jul 5 21:29:42 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 5 Jul 2010 21:29:42 +0200 (CEST) Subject: [pypy-svn] r75859 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100705192942.8230C282BF6@codespeak.net> Author: hakanardo Date: Mon Jul 5 21:29:40 2010 New Revision: 75859 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: sequence interface Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Jul 5 21:29:40 2010 @@ -149,19 +149,52 @@ self.descr_append(w_item) descr_extend.unwrap_spec = ['self', W_Root] - def descr_getitem(self, idx): - item = self.buffer[idx] - if self.typecode in ('b', 'B', 'h', 'H', 'i', 'l'): - item = rffi.cast(lltype.Signed, item) - if self.typecode == 'f': - item = float(item) - return self.space.wrap(item) - descr_getitem.unwrap_spec = ['self', int] - - def descr_setitem(self, idx, w_item): - item = self.item_w(w_item) - self.buffer[idx] = item - descr_setitem.unwrap_spec = ['self', int, W_Root] + def descr_getitem(self, w_idx): + space=self.space + start, stop, step = space.decode_index(w_idx, self.len) + if step==0: + item = self.buffer[start] + if self.typecode in ('b', 'B', 'h', 'H', 'i', 'l'): + item = rffi.cast(lltype.Signed, item) + elif self.typecode == 'f': + item = float(item) + return self.space.wrap(item) + else: + size = (stop - start) / step + if (stop - start) % step > 0: size += 1 + w_a=W_Array(self.space, self.typecode) + w_a.setlen(size) + j=0 + for i in range(start, stop, step): + w_a.buffer[j]=self.buffer[i] + j+=1 + return w_a + descr_getitem.unwrap_spec = ['self', W_Root] + + def descr_setitem(self, w_idx, w_item): + start, stop, step = self.space.decode_index(w_idx, self.len) + if step==0: + item = self.item_w(w_item) + self.buffer[start] = item + else: + if isinstance(w_item, W_Array): + if self.typecode == w_item.typecode: + size = (stop - start) / step + if (stop - start) % step > 0: size += 1 + if w_item.len != size: + msg = ('attempt to assign array of size %d to ' + + 'slice of size %d') % (w_item.len, size) + raise OperationError(self.space.w_ValueError, + self.space.wrap(msg)) + j=0 + for i in range(start, stop, step): + self.buffer[i]=w_item.buffer[j] + j+=1 + return + msg='can only assign array to array slice' + raise OperationError(self.space.w_TypeError, self.space.wrap(msg)) + + descr_setitem.unwrap_spec = ['self', W_Root, W_Root] def descr_len(self): return self.space.wrap(self.len) @@ -178,7 +211,7 @@ for i in range(new): p = i * self.itemsize item=struct.unpack(self.typecode, s[p:p + self.itemsize])[0] - self.descr_setitem(oldlen + i, self.space.wrap(item)) + self.buffer[oldlen + i]=self.item_w(self.space.wrap(item)) descr_fromstring.unwrap_spec = ['self', str] def descr_fromfile(self, w_f, n): @@ -207,7 +240,7 @@ self.setlen(oldlen+new) for i in range(new): w_item=space.getitem(w_lst, space.wrap(i)) - self.descr_setitem(oldlen + i, w_item) + self.buffer[oldlen + i] = self.item_w(w_item) except OperationError: self.buffer = oldbuf self.len = oldlen Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Mon Jul 5 21:29:40 2010 @@ -208,3 +208,58 @@ b = self.array('u', unicode('hi')) assert len(b) == 2 and b[0] == 'h' and b[1]=='i' + def test_sequence(self): + a=self.array('i', [1,2,3,4]) + assert len(a)==4 + assert a[0] == 1 and a[1] == 2 and a[2] == 3 and a[3] == 4 + assert a[-4] == 1 and a[-3] == 2 and a[-2] == 3 and a[-1] == 4 + a[-2]=5 + assert a[0] == 1 and a[1] == 2 and a[2] == 5 and a[3] == 4 + + for i in (4, -5): raises(IndexError, a.__getitem__, i) + + b = a[0:2] + assert len(b) == 2 and b[0] == 1 and b[1] == 2 + b[0]=6 + assert len(b) == 2 and b[0] == 6 and b[1] == 2 + assert a[0] == 1 and a[1] == 2 and a[2] == 5 and a[3] == 4 + assert a.itemsize == b.itemsize + + b = a[0:100] + assert len(b)==4 + assert b[0] == 1 and b[1] == 2 and b[2] == 5 and b[3] == 4 + + l1 = [2 * i + 1 for i in range(10)] + a1 = self.array('i', l1) + for start in range(10): + for stop in range(start, 10): + for step in range(1,10): + l2 = l1[start:stop:step] + a2 = a1[start:stop:step] + assert len(l2) == len(a2) + for i in range(len(l2)): assert l2[i] == a2[i] + + a=self.array('i', [1,2,3,4]) + a[1:3]=self.array('i', [5,6]) + assert len(a)==4 + assert a[0] == 1 and a[1] == 5 and a[2] == 6 and a[3] == 4 + a[0:-1:2]=self.array('i', [7,8]) + assert a[0] == 7 and a[1] == 5 and a[2] == 8 and a[3] == 4 + + try: + a[1:2:4]=self.array('i', [5,6,7]) + assert False + except ValueError: + pass + + try: + a[1:3]=self.array('I', [5,6]) + assert False + except TypeError: + pass + + try: + a[1:3]=[5,6] + assert False + except TypeError: + pass From getxsick at codespeak.net Mon Jul 5 21:48:59 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 5 Jul 2010 21:48:59 +0200 (CEST) Subject: [pypy-svn] r75860 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100705194859.1E801282BF6@codespeak.net> Author: getxsick Date: Mon Jul 5 21:48:58 2010 New Revision: 75860 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: next step to be more RPython Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Mon Jul 5 21:48:58 2010 @@ -22,7 +22,7 @@ class _LibHandler(object): def __init__(self, name): try: - self.handler = rdynload.dlopen(name) + self.handler = rdynload.dlopen(rffi.str2charp(name)) except rdynload.DLOpenError, e: raise OSError('%s: %s', name, e.msg or 'unspecified error') From getxsick at codespeak.net Mon Jul 5 21:57:16 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 5 Jul 2010 21:57:16 +0200 (CEST) Subject: [pypy-svn] r75861 - pypy/branch/fast-ctypes/pypy/module/jitffi Message-ID: <20100705195716.5AD5C282BF6@codespeak.net> Author: getxsick Date: Mon Jul 5 21:57:14 2010 New Revision: 75861 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Log: use correct name of the class Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Mon Jul 5 21:57:14 2010 @@ -51,7 +51,7 @@ def W_LibHandler___new__(space, w_type, name): try: - return space.wrap(W_Lib(space, name)) + return space.wrap(W_LibHandler(space, name)) except OSError, e: raise wrap_oserror(space, e) From benjamin at codespeak.net Mon Jul 5 23:45:42 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 5 Jul 2010 23:45:42 +0200 (CEST) Subject: [pypy-svn] r75862 - pypy/branch/fast-forward/pypy/module/math/test Message-ID: <20100705214542.B2383282BF6@codespeak.net> Author: benjamin Date: Mon Jul 5 23:45:41 2010 New Revision: 75862 Modified: pypy/branch/fast-forward/pypy/module/math/test/test_math.py Log: add new math tests Modified: pypy/branch/fast-forward/pypy/module/math/test/test_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/test/test_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/test/test_math.py Mon Jul 5 23:45:41 2010 @@ -85,3 +85,135 @@ raises(ValueError, math.factorial, -1) raises(ValueError, math.factorial, -1.) raises(ValueError, math.factorial, 1.1) + + def test_mtestfile(self): + import math + import abc + import os + def _parse_mtestfile(fname): + """Parse a file with test values + + -- starts a comment + blank lines, or lines containing only a comment, are ignored + other lines are expected to have the form + id fn arg -> expected [flag]* + + """ + with open(fname) as fp: + for line in fp: + # strip comments, and skip blank lines + if '--' in line: + line = line[:line.index('--')] + if not line.strip(): + continue + + lhs, rhs = line.split('->') + id, fn, arg = lhs.split() + rhs_pieces = rhs.split() + exp = rhs_pieces[0] + flags = rhs_pieces[1:] + + yield (id, fn, float(arg), float(exp), flags) + def to_ulps(x): + """Convert a non-NaN float x to an integer, in such a way that + adjacent floats are converted to adjacent integers. Then + abs(ulps(x) - ulps(y)) gives the difference in ulps between two + floats. + + The results from this function will only make sense on platforms + where C doubles are represented in IEEE 754 binary64 format. + + """ + n = struct.unpack(' Author: benjamin Date: Tue Jul 6 00:00:33 2010 New Revision: 75863 Added: pypy/branch/fast-forward/include/ (props changed) - copied from r75862, pypy/trunk/include/ pypy/branch/fast-forward/lib_pypy/pypy_test/ pypy/branch/fast-forward/lib_pypy/pypy_test/__init__.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/inprogress_test_binascii_extra.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/no_test_pickle_extra.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_binascii.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_collections.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_coroutine.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_ctypes_support.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_datetime.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_dbm_extra.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_defaultdict.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_deque_extra.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_exception_extra.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_functools.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_grp_extra.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_hashlib.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_locale.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_marshal_extra.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_md5_extra.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_pickle_extra.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_pyexpat.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_resource.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_runpy.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_sha_extra.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_stackless.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_stackless_pickling.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_struct_extra.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_structseq.py (contents, props changed) pypy/branch/fast-forward/lib_pypy/pypy_test/test_syslog.py (contents, props changed) pypy/branch/fast-forward/pypy/jit/metainterp/jitdriver.py - copied unchanged from r75862, pypy/trunk/pypy/jit/metainterp/jitdriver.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_jitdriver.py - copied unchanged from r75862, pypy/trunk/pypy/jit/metainterp/test/test_jitdriver.py Removed: pypy/branch/fast-forward/pypy/_interfaces/ pypy/branch/fast-forward/pypy/module/_codecs/app_codecs.py pypy/branch/fast-forward/pypy/module/unicodedata/LICENSE Modified: pypy/branch/fast-forward/ (props changed) pypy/branch/fast-forward/LICENSE pypy/branch/fast-forward/dotviewer/sshgraphserver.py pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/cmd.py pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/build_ext.py pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/install.py pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py pypy/branch/fast-forward/lib-python/modified-2.5.2/site.py pypy/branch/fast-forward/lib_pypy/stackless.py pypy/branch/fast-forward/py/_process/cmdexec.py pypy/branch/fast-forward/pypy/annotation/model.py pypy/branch/fast-forward/pypy/annotation/unaryop.py pypy/branch/fast-forward/pypy/conftest.py pypy/branch/fast-forward/pypy/doc/index.txt pypy/branch/fast-forward/pypy/doc/jit/pyjitpl5.txt pypy/branch/fast-forward/pypy/doc/release-1.3.0.txt pypy/branch/fast-forward/pypy/doc/stackless.txt pypy/branch/fast-forward/pypy/interpreter/pyopcode.py pypy/branch/fast-forward/pypy/interpreter/pyparser/parsestring.py pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_parsestring.py pypy/branch/fast-forward/pypy/interpreter/test/test_module.py pypy/branch/fast-forward/pypy/interpreter/unicodehelper.py pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py pypy/branch/fast-forward/pypy/jit/backend/llgraph/runner.py pypy/branch/fast-forward/pypy/jit/backend/model.py pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py pypy/branch/fast-forward/pypy/jit/backend/test/support.py pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/fast-forward/pypy/jit/codewriter/assembler.py pypy/branch/fast-forward/pypy/jit/codewriter/call.py pypy/branch/fast-forward/pypy/jit/codewriter/codewriter.py pypy/branch/fast-forward/pypy/jit/codewriter/jitcode.py pypy/branch/fast-forward/pypy/jit/codewriter/jtransform.py pypy/branch/fast-forward/pypy/jit/codewriter/support.py pypy/branch/fast-forward/pypy/jit/codewriter/test/test_assembler.py pypy/branch/fast-forward/pypy/jit/codewriter/test/test_call.py pypy/branch/fast-forward/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/fast-forward/pypy/jit/codewriter/test/test_flatten.py pypy/branch/fast-forward/pypy/jit/metainterp/blackhole.py pypy/branch/fast-forward/pypy/jit/metainterp/compile.py pypy/branch/fast-forward/pypy/jit/metainterp/executor.py pypy/branch/fast-forward/pypy/jit/metainterp/history.py pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py pypy/branch/fast-forward/pypy/jit/metainterp/resoperation.py pypy/branch/fast-forward/pypy/jit/metainterp/resume.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_basic.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_blackhole.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_compile.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_list.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_ztranslation.py pypy/branch/fast-forward/pypy/jit/metainterp/virtualizable.py pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py pypy/branch/fast-forward/pypy/jit/metainterp/warmstate.py pypy/branch/fast-forward/pypy/jit/tl/spli/targetspli.py pypy/branch/fast-forward/pypy/jit/tl/targettlc.py pypy/branch/fast-forward/pypy/jit/tl/targettlr.py pypy/branch/fast-forward/pypy/jit/tl/tinyframe/targettinyframe.py pypy/branch/fast-forward/pypy/module/__builtin__/functional.py pypy/branch/fast-forward/pypy/module/__builtin__/test/test_functional.py pypy/branch/fast-forward/pypy/module/_codecs/__init__.py pypy/branch/fast-forward/pypy/module/_codecs/interp_codecs.py pypy/branch/fast-forward/pypy/module/_codecs/test/test_codecs.py pypy/branch/fast-forward/pypy/module/_stackless/interp_coroutine.py pypy/branch/fast-forward/pypy/module/_stackless/rcoroutine.py pypy/branch/fast-forward/pypy/module/_stackless/test/test_coroutine.py pypy/branch/fast-forward/pypy/module/conftest.py pypy/branch/fast-forward/pypy/module/cpyext/api.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_api.py pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py pypy/branch/fast-forward/pypy/module/operator/test/test_operator.py pypy/branch/fast-forward/pypy/module/sys/state.py pypy/branch/fast-forward/pypy/module/sys/test/test_initialpath.py pypy/branch/fast-forward/pypy/module/sys/version.py pypy/branch/fast-forward/pypy/module/termios/interp_termios.py pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py pypy/branch/fast-forward/pypy/objspace/std/stringobject.py pypy/branch/fast-forward/pypy/objspace/std/test/test_shadowtracking.py pypy/branch/fast-forward/pypy/objspace/std/unicodeobject.py pypy/branch/fast-forward/pypy/rlib/debug.py pypy/branch/fast-forward/pypy/rlib/libffi.py pypy/branch/fast-forward/pypy/rlib/rarithmetic.py pypy/branch/fast-forward/pypy/rlib/rcoroutine.py pypy/branch/fast-forward/pypy/rlib/rsre/test/targetrsre.py pypy/branch/fast-forward/pypy/rlib/rstring.py pypy/branch/fast-forward/pypy/rlib/runicode.py pypy/branch/fast-forward/pypy/rlib/test/test_debug.py pypy/branch/fast-forward/pypy/rlib/test/test_runicode.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/rffi.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/test/test_rffi.py pypy/branch/fast-forward/pypy/rpython/microbench/microbench.py pypy/branch/fast-forward/pypy/rpython/module/ll_termios.py pypy/branch/fast-forward/pypy/rpython/rstr.py pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py pypy/branch/fast-forward/pypy/tool/release/force-builds.py pypy/branch/fast-forward/pypy/tool/release/make_release.py pypy/branch/fast-forward/pypy/tool/release/package.py pypy/branch/fast-forward/pypy/tool/release/test/test_package.py pypy/branch/fast-forward/pypy/tool/test/test_killsubprocess.py pypy/branch/fast-forward/pypy/translator/backendopt/test/test_mallocprediction.py pypy/branch/fast-forward/pypy/translator/c/gcc/trackgcroot.py pypy/branch/fast-forward/pypy/translator/cli/gencli.py pypy/branch/fast-forward/pypy/translator/cli/query.py pypy/branch/fast-forward/pypy/translator/cli/rte.py pypy/branch/fast-forward/pypy/translator/cli/support.py pypy/branch/fast-forward/pypy/translator/cli/test/runtest.py pypy/branch/fast-forward/pypy/translator/driver.py pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py pypy/branch/fast-forward/pypy/translator/jvm/genjvm.py pypy/branch/fast-forward/pypy/translator/jvm/test/runtest.py pypy/branch/fast-forward/pypy/translator/platform/windows.py Log: merge from trunk Modified: pypy/branch/fast-forward/LICENSE ============================================================================== --- pypy/branch/fast-forward/LICENSE (original) +++ pypy/branch/fast-forward/LICENSE Tue Jul 6 00:00:33 2010 @@ -152,3 +152,26 @@ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License for 'pypy/module/unicodedata/' +====================================== + +The following files are from the website of The Unicode Consortium +at http://www.unicode.org/. For the terms of use of these files, see +http://www.unicode.org/terms_of_use.html + + CompositionExclusions-3.2.0.txt + CompositionExclusions-4.1.0.txt + CompositionExclusions-5.0.0.txt + EastAsianWidth-3.2.0.txt + EastAsianWidth-4.1.0.txt + EastAsianWidth-5.0.0.txt + UnicodeData-3.2.0.txt + UnicodeData-4.1.0.txt + UnicodeData-5.0.0.txt + +The following files are derived from files from the above website. The same +terms of use apply. + UnihanNumeric-3.2.0.txt + UnihanNumeric-4.1.0.txt + UnihanNumeric-5.0.0.txt Modified: pypy/branch/fast-forward/dotviewer/sshgraphserver.py ============================================================================== --- pypy/branch/fast-forward/dotviewer/sshgraphserver.py (original) +++ pypy/branch/fast-forward/dotviewer/sshgraphserver.py Tue Jul 6 00:00:33 2010 @@ -21,7 +21,7 @@ remoteport = random.randrange(10000, 20000) # ^^^ and just hope there is no conflict - args = ['ssh', '-C', '-R%d:127.0.0.1:%d' % (remoteport, localport)] + args = ['ssh', '-S', 'none', '-C', '-R%d:127.0.0.1:%d' % (remoteport, localport)] args = args + sshargs + ['python -u -c "exec input()"'] print ' '.join(args[:-1]) p = subprocess.Popen(args, bufsize=0, Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/cmd.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/cmd.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/cmd.py Tue Jul 6 00:00:33 2010 @@ -17,9 +17,9 @@ def _easy_install_get_site_dirs(): # return a list of 'site' dirs for easy_install from pkg_resources import normalize_path + from distutils.sysconfig import get_python_lib sitedirs = filter(None,os.environ.get('PYTHONPATH','').split(os.pathsep)) - pypylib = 'pypy%d.%d' % sys.pypy_version_info[:2] - sitedirs.append(os.path.join(sys.prefix, 'lib', pypylib, 'site-packages')) + sitedirs.append(get_python_lib(standard_lib=False)) sitedirs = map(normalize_path, sitedirs) return sitedirs Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/build_ext.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/build_ext.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/build_ext.py Tue Jul 6 00:00:33 2010 @@ -167,7 +167,7 @@ # for Release and Debug builds. # also Python's library directory must be appended to library_dirs if os.name == 'nt': - self.library_dirs.append(os.path.join(sys.prefix, 'pypy', '_interfaces')) + self.library_dirs.append(os.path.join(sys.prefix, 'include')) if self.debug: self.build_temp = os.path.join(self.build_temp, "Debug") else: Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/install.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/install.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/install.py Tue Jul 6 00:00:33 2010 @@ -67,8 +67,8 @@ 'data' : '$base', }, 'pypy': { - 'purelib': '$base/lib/pypy$pypy_version_short/site-packages', - 'platlib': '$base/lib/pypy$pypy_version_short/site-packages', + 'purelib': '$base/site-packages', + 'platlib': '$base/site-packages', 'headers': '$base/include', 'scripts': '$base/bin', 'data' : '$base', Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Tue Jul 6 00:00:33 2010 @@ -13,12 +13,7 @@ def get_python_inc(plat_specific=0, prefix=None): from os.path import join as j - cand = j(sys.pypy_prefix, 'include') - if os.path.exists(cand): - return cand - if plat_specific: - return j(sys.prefix, "pypy", "_interfaces") - return j(sys.prefix, 'pypy', 'module', 'cpyext', 'include') + return j(sys.prefix, 'include') def get_python_version(): """Return a string containing the major and minor Python version, @@ -47,8 +42,7 @@ "calls to get_python_lib(standard_lib=1) cannot succeed") if prefix is None: prefix = PREFIX - pypylib = 'pypy%d.%d' % sys.pypy_version_info[:2] - return os.path.join(prefix, 'lib', pypylib, 'site-packages') + return os.path.join(prefix, 'site-packages') _config_vars = None Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/site.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.5.2/site.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.5.2/site.py Tue Jul 6 00:00:33 2010 @@ -176,8 +176,8 @@ def addsitepackages(known_paths): """Add site-packages to sys.path, in a PyPy-specific way.""" if hasattr(sys, 'pypy_version_info'): - pypylib = 'pypy%d.%d' % sys.pypy_version_info[:2] - sitedir = os.path.join(sys.prefix, 'lib', pypylib, 'site-packages') + from distutils.sysconfig import get_python_lib + sitedir = get_python_lib(standard_lib=False) if os.path.isdir(sitedir): addsitedir(sitedir, known_paths) return None Added: pypy/branch/fast-forward/lib_pypy/pypy_test/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/__init__.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1 @@ +# Added: pypy/branch/fast-forward/lib_pypy/pypy_test/inprogress_test_binascii_extra.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/inprogress_test_binascii_extra.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,19 @@ +from __future__ import absolute_import +from .. import binascii + +def test_uu(): + assert binascii.b2a_uu('1234567') == "',3(S-#4V-P \n" + assert binascii.b2a_uu('123456789012345678901234567890123456789012345') == 'M,3(S-#4V-S at Y,#$R,S0U-C ratio: + c, line = line[0], line[1:] + else: + c, noise = noise[0], noise[1:] + res += c + return res + noise + line + res = "" + for line in map(addnoise, lines): + b = binascii.a2b_base64(line) + res += b + assert res == data + + # Test base64 with just invalid characters, which should return + # empty strings. TBD: shouldn't it raise an exception instead ? + assert binascii.a2b_base64(fillers) == '' + +def test_uu(): + MAX_UU = 45 + lines = [] + for i in range(0, len(data), MAX_UU): + b = data[i:i+MAX_UU] + a = binascii.b2a_uu(b) + lines.append(a) + res = "" + for line in lines: + b = binascii.a2b_uu(line) + res += b + assert res == data + + assert binascii.a2b_uu("\x7f") == "\x00"*31 + assert binascii.a2b_uu("\x80") == "\x00"*32 + assert binascii.a2b_uu("\xff") == "\x00"*31 + py.test.raises(binascii.Error, binascii.a2b_uu, "\xff\x00") + py.test.raises(binascii.Error, binascii.a2b_uu, "!!!!") + + py.test.raises(binascii.Error, binascii.b2a_uu, 46*"!") + +def test_crc32(): + crc = binascii.crc32("Test the CRC-32 of") + crc = binascii.crc32(" this string.", crc) + assert crc == 1571220330 + + crc = binascii.crc32('frotz\n', 0) + assert crc == -372923920 + + py.test.raises(TypeError, binascii.crc32) + +def test_hex(): + # test hexlification + s = '{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000' + t = binascii.b2a_hex(s) + u = binascii.a2b_hex(t) + assert s == u + py.test.raises(TypeError, binascii.a2b_hex, t[:-1]) + py.test.raises(TypeError, binascii.a2b_hex, t[:-1] + 'q') + + # Verify the treatment of Unicode strings + assert binascii.hexlify(unicode('a', 'ascii')) == '61' + +def test_qp(): + # A test for SF bug 534347 (segfaults without the proper fix) + try: + binascii.a2b_qp("", **{1:1}) + except TypeError: + pass + else: + fail("binascii.a2b_qp(**{1:1}) didn't raise TypeError") + assert binascii.a2b_qp("= ") == "= " + assert binascii.a2b_qp("==") == "=" + assert binascii.a2b_qp("=AX") == "=AX" + py.test.raises(TypeError, binascii.b2a_qp, foo="bar") + assert binascii.a2b_qp("=00\r\n=00") == "\x00\r\n\x00" + assert binascii.b2a_qp("\xff\r\n\xff\n\xff") == "=FF\r\n=FF\r\n=FF" + target = "0"*75+"=\r\n=FF\r\n=FF\r\n=FF" + assert binascii.b2a_qp("0"*75+"\xff\r\n\xff\r\n\xff") == target + +def test_empty_string(): + # A test for SF bug #1022953. Make sure SystemError is not raised. + for n in ['b2a_qp', 'a2b_hex', 'b2a_base64', 'a2b_uu', 'a2b_qp', + 'b2a_hex', 'unhexlify', 'hexlify', 'crc32', 'b2a_hqx', + 'a2b_hqx', 'a2b_base64', 'rlecode_hqx', 'b2a_uu', + 'rledecode_hqx']: + f = getattr(binascii, n) + f('') + binascii.crc_hqx('', 0) + +def test_qp_bug_case(): + assert binascii.b2a_qp('y'*77, False, False) == 'y'*75 + '=\nyy' + assert binascii.b2a_qp(' '*77, False, False) == ' '*75 + '=\n =20' + assert binascii.b2a_qp('y'*76, False, False) == 'y'*76 + assert binascii.b2a_qp(' '*76, False, False) == ' '*75 + '=\n=20' + +def test_wrong_padding(): + s = 'CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3IQ' + py.test.raises(binascii.Error, binascii.a2b_base64, s) + +def test_crap_after_padding(): + s = 'xxx=axxxx' + assert binascii.a2b_base64(s) == '\xc7\x1c' + +def test_wrong_args(): + # this should grow as a way longer list + py.test.raises(TypeError, binascii.a2b_base64, 42) Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_collections.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_collections.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,24 @@ +from __future__ import absolute_import +from .. import collections +import py + +def test_deque_remove_empty(): + d = collections.deque([]) + py.test.raises(ValueError, d.remove, 1) + +def test_deque_remove_mutating(): + class MutatingCmp(object): + def __eq__(self, other): + d.clear() + return True + + d = collections.deque([MutatingCmp()]) + py.test.raises(IndexError, d.remove, 1) + +class SubclassWithKwargs(collections.deque): + def __init__(self, newarg=1): + collections.deque.__init__(self) + +def test_subclass_with_kwargs(): + # SF bug #1486663 -- this used to erroneously raise a TypeError + SubclassWithKwargs(newarg=1) Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_coroutine.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_coroutine.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,214 @@ +from __future__ import absolute_import +from py.test import skip, raises + +try: + from ..stackless import coroutine +except ImportError, e: + skip('cannot import stackless: %s' % (e,)) + + +class Test_Coroutine: + + def test_is_zombie(self): + co = coroutine() + def f(): + print 'in coro' + assert not co.is_zombie + co.bind(f) + assert not co.is_zombie + co.switch() + assert not co.is_zombie + + def test_is_zombie_del_without_frame(self): + try: + import _stackless # are we on pypy with a stackless build? + except ImportError: + skip("only works on pypy-c-stackless") + import gc + res = [] + class MyCoroutine(coroutine): + def __del__(self): + res.append(self.is_zombie) + def f(): + print 'in coro' + co = MyCoroutine() + co.bind(f) + co.switch() + del co + for i in range(10): + gc.collect() + if res: + break + co = coroutine() + co.bind(f) + co.switch() + assert res[0], "is_zombie was False in __del__" + + def test_is_zombie_del_with_frame(self): + try: + import _stackless # are we on pypy with a stackless build? + except ImportError: + skip("only works on pypy-c-stackless") + import gc + res = [] + class MyCoroutine(coroutine): + def __del__(self): + res.append(self.is_zombie) + main = coroutine.getcurrent() + def f(): + print 'in coro' + main.switch() + co = MyCoroutine() + co.bind(f) + co.switch() + del co + for i in range(10): + gc.collect() + if res: + break + co = coroutine() + co.bind(f) + co.switch() + assert res[0], "is_zombie was False in __del__" + + def test_raise_propagate(self): + co = coroutine() + def f(): + return 1/0 + co.bind(f) + try: + co.switch() + except ZeroDivisionError: + pass + else: + raise AssertionError("exception not propagated") + + def test_strange_test(self): + def f(): + return 42 + def create(): + b = coroutine() + b.bind(f) + b.switch() + return b + a = coroutine() + a.bind(create) + b = a.switch() + def nothing(): + pass + a.bind(nothing) + def kill(): + a.kill() + b.bind(kill) + b.switch() + + def test_kill(self): + co = coroutine() + def f(): + pass + assert not co.is_alive + co.bind(f) + assert co.is_alive + co.kill() + assert not co.is_alive + + def test_catch_coroutineexit(self): + coroutineexit = [] + co_a = coroutine() + co_test = coroutine.getcurrent() + + def a(): + try: + co_test.switch() + except CoroutineExit: + coroutineexit.append(True) + raise + + co_a.bind(a) + co_a.switch() + assert co_a.is_alive + + co_a.kill() + assert coroutineexit == [True] + assert not co_a.is_alive + + def test_throw(self): + exceptions = [] + co = coroutine() + def f(main): + try: + main.switch() + except RuntimeError: + exceptions.append(True) + + co.bind(f, coroutine.getcurrent()) + co.switch() + co.throw(RuntimeError) + assert exceptions == [True] + + def test_propagation(self): + exceptions = [] + co = coroutine() + co2 = coroutine() + def f(main): + main.switch() + + co.bind(f, coroutine.getcurrent()) + co.switch() + + try: + co.throw(RuntimeError) + except RuntimeError: + exceptions.append(1) + + def f2(): + raise RuntimeError + + co2.bind(f2) + + try: + co2.switch() + except RuntimeError: + exceptions.append(2) + + assert exceptions == [1,2] + + def test_bogus_bind(self): + co = coroutine() + def f(): + pass + co.bind(f) + raises(ValueError, co.bind, f) + + def test_simple_task(self): + maintask = coroutine.getcurrent() + def f():pass + co = coroutine() + co.bind(f) + co.switch() + assert not co.is_alive + assert maintask is coroutine.getcurrent() + + def test_backto_main(self): + maintask = coroutine.getcurrent() + def f(task): + task.switch() + co = coroutine() + co.bind(f,maintask) + co.switch() + + def test_wrapped_main(self): + class mwrap(object): + def __init__(self, coro): + self._coro = coro + + def __getattr__(self, attr): + return getattr(self._coro, attr) + + maintask = mwrap(coroutine.getcurrent()) + def f(task): + task.switch() + co = coroutine() + co.bind(f,maintask) + co.switch() + Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_ctypes_support.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_ctypes_support.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,22 @@ +from __future__ import absolute_import + +import py +from ctypes import * +try: + from ctypes_support import standard_c_lib, get_errno, set_errno +except ImportError: # on top of cpython + from ..ctypes_support import standard_c_lib, get_errno, set_errno + + +def test_stdlib_and_errno(): + py.test.skip("this is expected on top of pypy, we need to fix ctypes in a way that is now in 2.6 in order to make this reliable") + write = standard_c_lib.write + write.argtypes = [c_int, c_char_p, c_size_t] + write.restype = c_size_t + # clear errno first + set_errno(0) + assert get_errno() == 0 + write(-345, "abc", 3) + assert get_errno() != 0 + set_errno(0) + assert get_errno() == 0 Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_datetime.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_datetime.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,18 @@ +from __future__ import absolute_import + +from .. import datetime + +def test_repr(): + print datetime + expected = "datetime.datetime(1, 2, 3, 0, 0)" + assert repr(datetime.datetime(1,2,3)) == expected + +def test_strptime(): + import time + + string = '2004-12-01 13:02:47' + format = '%Y-%m-%d %H:%M:%S' + expected = datetime.datetime(*(time.strptime(string, format)[0:6])) + got = datetime.datetime.strptime(string, format) + assert expected == got + Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_dbm_extra.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_dbm_extra.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,50 @@ +from __future__ import absolute_import +import py +from pypy.tool.udir import udir +try: + from .. import dbm +except ImportError, e: + py.test.skip(e) + +def test_get(): + path = str(udir.join('test_dbm_extra.test_get')) + d = dbm.open(path, 'c') + x = d.get("42") + assert x is None + d.close() + +def test_delitem(): + path = str(udir.join('test_dbm_extra.test_delitem')) + d = dbm.open(path, 'c') + py.test.raises(KeyError, "del d['xyz']") + +def test_nonstring(): + path = str(udir.join('test_dbm_extra.test_nonstring')) + d = dbm.open(path, 'c') + py.test.raises(TypeError, "d[123] = 'xyz'") + py.test.raises(TypeError, "d['xyz'] = 123") + py.test.raises(TypeError, "d['xyz'] = None") + py.test.raises(TypeError, "del d[123]") + py.test.raises(TypeError, "d[123]") + py.test.raises(TypeError, "123 in d") + py.test.raises(TypeError, "d.has_key(123)") + py.test.raises(TypeError, "d.setdefault(123, 'xyz')") + py.test.raises(TypeError, "d.setdefault('xyz', 123)") + py.test.raises(TypeError, "d.get(123)") + assert dict(d) == {} + d.setdefault('xyz', '123') + assert dict(d) == {'xyz': '123'} + d.close() + +def test_multiple_sets(): + path = str(udir.join('test_dbm_extra.test_multiple_sets')) + d = dbm.open(path, 'c') + d['xyz'] = '12' + d['xyz'] = '3' + d['xyz'] = '546' + assert dict(d) == {'xyz': '546'} + assert d['xyz'] == '546' + +def test_extra(): + py.test.raises(TypeError, dbm.datum, 123) + py.test.raises(TypeError, dbm.datum, False) Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_defaultdict.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_defaultdict.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,126 @@ +# defaultdict Tests +# from CPython2.5 +from __future__ import absolute_import +import py +import sys +if sys.version_info < (2, 5): + # the app-level defaultdict relies on the interp-level dict + # calling __missing__() + py.test.skip("these tests only run on top of CPython 2.5") + +import copy + +from ..collections import defaultdict + +def foobar(): + return list + +class Test_defaultdict: + + def test_basic(self): + d1 = defaultdict() + assert d1.default_factory is None + d1.default_factory = list + d1[12].append(42) + assert d1 == {12: [42]} + d1[12].append(24) + assert d1 == {12: [42, 24]} + d1[13] + d1[14] + assert d1 == {12: [42, 24], 13: [], 14: []} + assert d1[12] is not d1[13] is not d1[14] + d2 = defaultdict(list, foo=1, bar=2) + assert d2.default_factory == list + assert d2 == {"foo": 1, "bar": 2} + assert d2["foo"] == 1 + assert d2["bar"] == 2 + assert d2[42] == [] + assert "foo" in d2 + assert "foo" in d2.keys() + assert "bar" in d2 + assert "bar" in d2.keys() + assert 42 in d2 + assert 42 in d2.keys() + assert 12 not in d2 + assert 12 not in d2.keys() + d2.default_factory = None + assert d2.default_factory == None + py.test.raises(KeyError, d2.__getitem__, 15) + py.test.raises(TypeError, defaultdict, 1) + + def test_missing(self): + d1 = defaultdict() + py.test.raises(KeyError, d1.__missing__, 42) + d1.default_factory = list + assert d1.__missing__(42) == [] + + def test_repr(self): + d1 = defaultdict() + assert d1.default_factory == None + assert repr(d1) == "defaultdict(None, {})" + d1[11] = 41 + assert repr(d1) == "defaultdict(None, {11: 41})" + d2 = defaultdict(int) + assert d2.default_factory == int + d2[12] = 42 + assert repr(d2) == "defaultdict(, {12: 42})" + def foo(): return 43 + d3 = defaultdict(foo) + assert d3.default_factory is foo + d3[13] + assert repr(d3) == "defaultdict(%s, {13: 43})" % repr(foo) + d4 = defaultdict(int) + d4[14] = defaultdict() + assert repr(d4) == "defaultdict(%s, {14: defaultdict(None, {})})" % repr(int) + + def test_recursive_repr(self): + # Issue2045: stack overflow when default_factory is a bound method + class sub(defaultdict): + def __init__(self): + self.default_factory = self._factory + def _factory(self): + return [] + d = sub() + assert repr(d).startswith( + "defaultdict( 1: + cpy_dump_version = (1,) +else: + cpy_dump_version = () + + +def test_cases(): + for case in TESTCASES: + yield dumps_and_reload, case + yield loads_from_cpython, case + yield dumps_to_cpython, case + if case is not StopIteration: + yield dumps_subclass, case + yield load_from_cpython, case + yield dump_to_cpython, case + +def dumps_and_reload(case): + print 'dump_and_reload', `case` + s = marshal.dumps(case) + obj = marshal.loads(s) + assert obj == case + +def loads_from_cpython(case): + print 'load_from_cpython', `case` + try: + s = cpy_marshal.dumps(case, *cpy_dump_version) + except ValueError: + py.test.skip("this version of CPython doesn't support this object") + obj = marshal.loads(s) + assert obj == case + +def dumps_to_cpython(case): + print 'dump_to_cpython', `case` + try: + cpy_marshal.dumps(case, *cpy_dump_version) + except ValueError: + py.test.skip("this version of CPython doesn't support this object") + s = marshal.dumps(case) + obj = cpy_marshal.loads(s) + assert obj == case + +def dumps_subclass(case): + try: + class Subclass(type(case)): + pass + case = Subclass(case) + except TypeError: + py.test.skip("this version of CPython doesn't support this object") + s = marshal.dumps(case) + obj = marshal.loads(s) + assert obj == case + +def load_from_cpython(case): + p = str(udir.join('test.dat')) + + f1 = open(p, "w") + try: + try: + s = cpy_marshal.dump(case, f1, *cpy_dump_version) + finally: + f1.close() + except ValueError: + py.test.skip("this version of CPython doesn't support this object") + + f2 = open(p, "r") + try: + obj = marshal.load(f2) + finally: + f2.close() + assert obj == case + +def dump_to_cpython(case): + + try: + cpy_marshal.dumps(case, *cpy_dump_version) + except ValueError: + py.test.skip("this version of CPython doesn't support this object") + + p = str(udir.join('test.dat')) + f1 = open(p, "w") + try: + try: + s = marshal.dump(case, f1) + finally: + f1.close() + except ValueError: + py.test.skip("this version of CPython doesn't support this object") + + f2 = open(p, "r") + try: + obj = cpy_marshal.load(f2) + finally: + f2.close() + assert obj == case + + Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_md5_extra.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_md5_extra.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,228 @@ +"""A test script to compare MD5 implementations. + +A note about performance: the pure Python MD5 takes roughly +160 sec. per MB of data on a 233 MHz Intel Pentium CPU. +""" + +from __future__ import absolute_import +import md5 # CPython's implementation in C. +from .. import md5 as pymd5 + + +# Helpers... + +def formatHex(str): + "Print a string's HEX code in groups of two digits." + + d = map(None, str) + d = map(ord, d) + d = map(lambda x:"%02x" % x, d) + return ' '.join(d) + + +def format(str): + "Print a string as-is in groups of two characters." + + s = '' + for i in range(0, len(str)-1, 2): + s = s + "%03s" % str[i:i+2] + return s[1:] + + +def printDiff(message, d1, d2, expectedResult=None): + "Print different outputs for same message." + + print "Message: '%s'" % message + print "Message length: %d" % len(message) + if expectedResult: + print "%-48s (expected)" % format(expectedResult) + print "%-48s (Std. lib. MD5)" % formatHex(d1) + print "%-48s (Pure Python MD5)" % formatHex(d2) + print + + +# The real comparison function. + +def compareImp(message): + """Compare two MD5 implementations, C vs. pure Python module. + + For equal digests this returns None, otherwise it returns + a tuple of both digests. + """ + + # Use Python's standard library MD5 compiled C module. + m1 = md5.md5() + m1.update(message) + d1 = m1.digest() + d1h = m1.hexdigest() + + # Use MD5 module in pure Python. + m2 = pymd5.md5() + m2.update(message) + d2 = m2.digest() + d2h = m2.hexdigest() + + # Return None if equal or the different digests if not equal. + if d1 == d2 and d1h == d2h: + return + else: + return d1, d2 + + +class TestMD5Compare: + "Compare pure Python MD5 against Python's std. lib. version." + + def test1(self): + "Test cases with known digest result." + + cases = ( + ("", + "d41d8cd98f00b204e9800998ecf8427e"), + ("a", + "0cc175b9c0f1b6a831c399e269772661"), + ("abc", + "900150983cd24fb0d6963f7d28e17f72"), + ("message digest", + "f96b697d7cb7938d525a2f31aaf161d0"), + ("abcdefghijklmnopqrstuvwxyz", + "c3fcd3d76192e4007dfb496cca67e13b"), + ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + "d174ab98d277d9f5a5611c2c9f419d9f"), + ("1234567890"*8, + "57edf4a22be3c955ac49da2e2107b67a"), + ) + + for i in xrange(len(cases)): + res = compareImp(cases[i][0]) + if res is not None: + d1, d2 = res + message, expectedResult = cases[i][0], None + if len(cases[i]) == 2: + expectedResult = cases[i][1] + printDiff(message, d1, d2, expectedResult) + assert res is None + + + def test2(self): + "Test cases without known digest result." + + cases = ( + "123", + "1234", + "12345", + "123456", + "1234567", + "12345678", + "123456789 123456789 123456789 ", + "123456789 123456789 ", + "123456789 123456789 1", + "123456789 123456789 12", + "123456789 123456789 123", + "123456789 123456789 1234", + "123456789 123456789 123456789 1", + "123456789 123456789 123456789 12", + "123456789 123456789 123456789 123", + "123456789 123456789 123456789 1234", + "123456789 123456789 123456789 12345", + "123456789 123456789 123456789 123456", + "123456789 123456789 123456789 1234567", + "123456789 123456789 123456789 12345678", + ) + + for i in xrange(len(cases)): + res = compareImp(cases[i][0]) + if res is not None: + d1, d2 = res + message = cases[i][0] + printDiff(message, d1, d2) + assert res is None + + + def test3(self): + "Test cases with long messages (can take a while)." + + cases = ( + (2**10*'a',), + (2**10*'abcd',), +## (2**20*'a',), ## 1 MB, takes about 160 sec. on a 233 Mhz Pentium. + ) + + for i in xrange(len(cases)): + res = compareImp(cases[i][0]) + if res is not None: + d1, d2 = res + message = cases[i][0] + printDiff(message, d1, d2) + assert res is None + + + def test4(self): + "Test cases with increasingly growing message lengths." + + i = 0 + while i < 2**5: + message = i * 'a' + res = compareImp(message) + if res is not None: + d1, d2 = res + printDiff(message, d1, d2) + assert res is None + i = i + 1 + + + def test5(self): + "Test updating cloned objects." + + cases = ( + "123", + "1234", + "12345", + "123456", + "1234567", + "12345678", + "123456789 123456789 123456789 ", + "123456789 123456789 ", + "123456789 123456789 1", + "123456789 123456789 12", + "123456789 123456789 123", + "123456789 123456789 1234", + "123456789 123456789 123456789 1", + "123456789 123456789 123456789 12", + "123456789 123456789 123456789 123", + "123456789 123456789 123456789 1234", + "123456789 123456789 123456789 12345", + "123456789 123456789 123456789 123456", + "123456789 123456789 123456789 1234567", + "123456789 123456789 123456789 12345678", + ) + + # Load both with same prefix. + prefix1 = 2**10 * 'a' + + m1 = md5.md5() + m1.update(prefix1) + m1c = m1.copy() + + m2 = pymd5.md5() + m2.update(prefix1) + m2c = m2.copy() + + # Update and compare... + for i in xrange(len(cases)): + message = cases[i][0] + + m1c.update(message) + d1 = m1c.hexdigest() + + m2c.update(message) + d2 = m2c.hexdigest() + + assert d1 == d2 + + +def test_attributes(): + assert pymd5.digest_size == 16 + assert pymd5.digestsize == 16 + assert pymd5.blocksize == 1 + assert pymd5.md5().digest_size == 16 + assert pymd5.md5().digestsize == 16 Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_pickle_extra.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_pickle_extra.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,15 @@ +import pickle + +def test_pickle_moduledict(): + if "_pickle_moduledict" not in pickle.Pickler.__dict__: + import py + py.test.skip("test the _pickle_moduledict() addition to pickle.py") + # + s1 = pickle.dumps(pickle.__dict__) + import gc; gc.collect() + s2 = pickle.dumps(pickle.__dict__) + # + d1 = pickle.loads(s1) + assert d1 is pickle.__dict__ + d2 = pickle.loads(s2) + assert d2 is pickle.__dict__ Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_pyexpat.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_pyexpat.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,665 @@ +# XXX TypeErrors on calling handlers, or on bad return values from a +# handler, are obscure and unhelpful. + +from __future__ import absolute_import +import StringIO, sys +import unittest, py + +from ..ctypes_config_cache import rebuild +rebuild.rebuild_one('pyexpat.ctc.py') + +from .. import pyexpat +#from xml.parsers import expat +expat = pyexpat + +from test.test_support import sortdict, run_unittest + + +class TestSetAttribute: + def setup_method(self, meth): + self.parser = expat.ParserCreate(namespace_separator='!') + self.set_get_pairs = [ + [0, 0], + [1, 1], + [2, 1], + [0, 0], + ] + + def test_returns_unicode(self): + for x, y in self.set_get_pairs: + self.parser.returns_unicode = x + assert self.parser.returns_unicode == y + + def test_ordered_attributes(self): + for x, y in self.set_get_pairs: + self.parser.ordered_attributes = x + assert self.parser.ordered_attributes == y + + def test_specified_attributes(self): + for x, y in self.set_get_pairs: + self.parser.specified_attributes = x + assert self.parser.specified_attributes == y + + +data = '''\ + + + + + + + + + +%unparsed_entity; +]> + + + + Contents of subelements + + +&external_entity; +&skipped_entity; + +''' + + +# Produce UTF-8 output +class TestParse: + class Outputter: + def __init__(self): + self.out = [] + + def StartElementHandler(self, name, attrs): + self.out.append('Start element: ' + repr(name) + ' ' + + sortdict(attrs)) + + def EndElementHandler(self, name): + self.out.append('End element: ' + repr(name)) + + def CharacterDataHandler(self, data): + data = data.strip() + if data: + self.out.append('Character data: ' + repr(data)) + + def ProcessingInstructionHandler(self, target, data): + self.out.append('PI: ' + repr(target) + ' ' + repr(data)) + + def StartNamespaceDeclHandler(self, prefix, uri): + self.out.append('NS decl: ' + repr(prefix) + ' ' + repr(uri)) + + def EndNamespaceDeclHandler(self, prefix): + self.out.append('End of NS decl: ' + repr(prefix)) + + def StartCdataSectionHandler(self): + self.out.append('Start of CDATA section') + + def EndCdataSectionHandler(self): + self.out.append('End of CDATA section') + + def CommentHandler(self, text): + self.out.append('Comment: ' + repr(text)) + + def NotationDeclHandler(self, *args): + name, base, sysid, pubid = args + self.out.append('Notation declared: %s' %(args,)) + + def UnparsedEntityDeclHandler(self, *args): + entityName, base, systemId, publicId, notationName = args + self.out.append('Unparsed entity decl: %s' %(args,)) + + def NotStandaloneHandler(self): + self.out.append('Not standalone') + return 1 + + def ExternalEntityRefHandler(self, *args): + context, base, sysId, pubId = args + self.out.append('External entity ref: %s' %(args[1:],)) + return 1 + + def StartDoctypeDeclHandler(self, *args): + self.out.append(('Start doctype', args)) + return 1 + + def EndDoctypeDeclHandler(self): + self.out.append("End doctype") + return 1 + + def EntityDeclHandler(self, *args): + self.out.append(('Entity declaration', args)) + return 1 + + def XmlDeclHandler(self, *args): + self.out.append(('XML declaration', args)) + return 1 + + def ElementDeclHandler(self, *args): + self.out.append(('Element declaration', args)) + return 1 + + def AttlistDeclHandler(self, *args): + self.out.append(('Attribute list declaration', args)) + return 1 + + def SkippedEntityHandler(self, *args): + self.out.append(("Skipped entity", args)) + return 1 + + def DefaultHandler(self, userData): + pass + + def DefaultHandlerExpand(self, userData): + pass + + handler_names = [ + 'StartElementHandler', 'EndElementHandler', 'CharacterDataHandler', + 'ProcessingInstructionHandler', 'UnparsedEntityDeclHandler', + 'NotationDeclHandler', 'StartNamespaceDeclHandler', + 'EndNamespaceDeclHandler', 'CommentHandler', + 'StartCdataSectionHandler', 'EndCdataSectionHandler', 'DefaultHandler', + 'DefaultHandlerExpand', 'NotStandaloneHandler', + 'ExternalEntityRefHandler', 'StartDoctypeDeclHandler', + 'EndDoctypeDeclHandler', 'EntityDeclHandler', 'XmlDeclHandler', + 'ElementDeclHandler', 'AttlistDeclHandler', 'SkippedEntityHandler', + ] + + def test_utf8(self): + + out = self.Outputter() + parser = expat.ParserCreate(namespace_separator='!') + for name in self.handler_names: + setattr(parser, name, getattr(out, name)) + parser.returns_unicode = 0 + parser.Parse(data, 1) + + # Verify output + operations = out.out + expected_operations = [ + ('XML declaration', (u'1.0', u'iso-8859-1', 0)), + 'PI: \'xml-stylesheet\' \'href="stylesheet.css"\'', + "Comment: ' comment data '", + "Not standalone", + ("Start doctype", ('quotations', 'quotations.dtd', None, 1)), + ('Element declaration', (u'root', (2, 0, None, ()))), + ('Attribute list declaration', ('root', 'attr1', 'CDATA', None, + 1)), + ('Attribute list declaration', ('root', 'attr2', 'CDATA', None, + 0)), + "Notation declared: ('notation', None, 'notation.jpeg', None)", + ('Entity declaration', ('acirc', 0, '\xc3\xa2', None, None, None, None)), + ('Entity declaration', ('external_entity', 0, None, None, + 'entity.file', None, None)), + "Unparsed entity decl: ('unparsed_entity', None, 'entity.file', None, 'notation')", + "Not standalone", + "End doctype", + "Start element: 'root' {'attr1': 'value1', 'attr2': 'value2\\xe1\\xbd\\x80'}", + "NS decl: 'myns' 'http://www.python.org/namespace'", + "Start element: 'http://www.python.org/namespace!subelement' {}", + "Character data: 'Contents of subelements'", + "End element: 'http://www.python.org/namespace!subelement'", + "End of NS decl: 'myns'", + "Start element: 'sub2' {}", + 'Start of CDATA section', + "Character data: 'contents of CDATA section'", + 'End of CDATA section', + "End element: 'sub2'", + "External entity ref: (None, 'entity.file', None)", + ('Skipped entity', ('skipped_entity', 0)), + "End element: 'root'", + ] + for operation, expected_operation in zip(operations, expected_operations): + assert operation == expected_operation + + def test_unicode(self): + # Try the parse again, this time producing Unicode output + out = self.Outputter() + parser = expat.ParserCreate(namespace_separator='!') + parser.returns_unicode = 1 + for name in self.handler_names: + setattr(parser, name, getattr(out, name)) + + parser.Parse(data, 1) + + operations = out.out + expected_operations = [ + ('XML declaration', (u'1.0', u'iso-8859-1', 0)), + 'PI: u\'xml-stylesheet\' u\'href="stylesheet.css"\'', + "Comment: u' comment data '", + "Not standalone", + ("Start doctype", ('quotations', 'quotations.dtd', None, 1)), + ('Element declaration', (u'root', (2, 0, None, ()))), + ('Attribute list declaration', ('root', 'attr1', 'CDATA', None, + 1)), + ('Attribute list declaration', ('root', 'attr2', 'CDATA', None, + 0)), + "Notation declared: (u'notation', None, u'notation.jpeg', None)", + ('Entity declaration', (u'acirc', 0, u'\xe2', None, None, None, + None)), + ('Entity declaration', (u'external_entity', 0, None, None, + u'entity.file', None, None)), + "Unparsed entity decl: (u'unparsed_entity', None, u'entity.file', None, u'notation')", + "Not standalone", + "End doctype", + "Start element: u'root' {u'attr1': u'value1', u'attr2': u'value2\\u1f40'}", + "NS decl: u'myns' u'http://www.python.org/namespace'", + "Start element: u'http://www.python.org/namespace!subelement' {}", + "Character data: u'Contents of subelements'", + "End element: u'http://www.python.org/namespace!subelement'", + "End of NS decl: u'myns'", + "Start element: u'sub2' {}", + 'Start of CDATA section', + "Character data: u'contents of CDATA section'", + 'End of CDATA section', + "End element: u'sub2'", + "External entity ref: (None, u'entity.file', None)", + ('Skipped entity', ('skipped_entity', 0)), + "End element: u'root'", + ] + for operation, expected_operation in zip(operations, expected_operations): + assert operation == expected_operation + + def test_parse_file(self): + # Try parsing a file + out = self.Outputter() + parser = expat.ParserCreate(namespace_separator='!') + parser.returns_unicode = 1 + for name in self.handler_names: + setattr(parser, name, getattr(out, name)) + file = StringIO.StringIO(data) + + parser.ParseFile(file) + + operations = out.out + expected_operations = [ + ('XML declaration', (u'1.0', u'iso-8859-1', 0)), + 'PI: u\'xml-stylesheet\' u\'href="stylesheet.css"\'', + "Comment: u' comment data '", + "Not standalone", + ("Start doctype", ('quotations', 'quotations.dtd', None, 1)), + ('Element declaration', (u'root', (2, 0, None, ()))), + ('Attribute list declaration', ('root', 'attr1', 'CDATA', None, + 1)), + ('Attribute list declaration', ('root', 'attr2', 'CDATA', None, + 0)), + "Notation declared: (u'notation', None, u'notation.jpeg', None)", + ('Entity declaration', ('acirc', 0, u'\xe2', None, None, None, None)), + ('Entity declaration', (u'external_entity', 0, None, None, u'entity.file', None, None)), + "Unparsed entity decl: (u'unparsed_entity', None, u'entity.file', None, u'notation')", + "Not standalone", + "End doctype", + "Start element: u'root' {u'attr1': u'value1', u'attr2': u'value2\\u1f40'}", + "NS decl: u'myns' u'http://www.python.org/namespace'", + "Start element: u'http://www.python.org/namespace!subelement' {}", + "Character data: u'Contents of subelements'", + "End element: u'http://www.python.org/namespace!subelement'", + "End of NS decl: u'myns'", + "Start element: u'sub2' {}", + 'Start of CDATA section', + "Character data: u'contents of CDATA section'", + 'End of CDATA section', + "End element: u'sub2'", + "External entity ref: (None, u'entity.file', None)", + ('Skipped entity', ('skipped_entity', 0)), + "End element: u'root'", + ] + for operation, expected_operation in zip(operations, expected_operations): + assert operation == expected_operation + + +class TestNamespaceSeparator: + def test_legal(self): + # Tests that make sure we get errors when the namespace_separator value + # is illegal, and that we don't for good values: + expat.ParserCreate() + expat.ParserCreate(namespace_separator=None) + expat.ParserCreate(namespace_separator=' ') + + def test_illegal(self): + try: + expat.ParserCreate(namespace_separator=42) + raise AssertionError + except TypeError, e: + assert str(e) == ( + 'ParserCreate() argument 2 must be string or None, not int') + + try: + expat.ParserCreate(namespace_separator='too long') + raise AssertionError + except ValueError, e: + assert str(e) == ( + 'namespace_separator must be at most one character, omitted, or None') + + def test_zero_length(self): + # ParserCreate() needs to accept a namespace_separator of zero length + # to satisfy the requirements of RDF applications that are required + # to simply glue together the namespace URI and the localname. Though + # considered a wart of the RDF specifications, it needs to be supported. + # + # See XML-SIG mailing list thread starting with + # http://mail.python.org/pipermail/xml-sig/2001-April/005202.html + # + expat.ParserCreate(namespace_separator='') # too short + + +class TestInterning: + def test(self): + py.test.skip("Not working") + # Test the interning machinery. + p = expat.ParserCreate() + L = [] + def collector(name, *args): + L.append(name) + p.StartElementHandler = collector + p.EndElementHandler = collector + p.Parse(" ", 1) + tag = L[0] + assert len(L) == 6 + for entry in L: + # L should have the same string repeated over and over. + assert tag is entry + + +class TestBufferText: + def setup_method(self, meth): + self.stuff = [] + self.parser = expat.ParserCreate() + self.parser.buffer_text = 1 + self.parser.CharacterDataHandler = self.CharacterDataHandler + + def check(self, expected, label): + assert self.stuff == expected, ( + "%s\nstuff = %r\nexpected = %r" + % (label, self.stuff, map(unicode, expected))) + + def CharacterDataHandler(self, text): + self.stuff.append(text) + + def StartElementHandler(self, name, attrs): + self.stuff.append("<%s>" % name) + bt = attrs.get("buffer-text") + if bt == "yes": + self.parser.buffer_text = 1 + elif bt == "no": + self.parser.buffer_text = 0 + + def EndElementHandler(self, name): + self.stuff.append("" % name) + + def CommentHandler(self, data): + self.stuff.append("" % data) + + def setHandlers(self, handlers=[]): + for name in handlers: + setattr(self.parser, name, getattr(self, name)) + + def test_default_to_disabled(self): + parser = expat.ParserCreate() + assert not parser.buffer_text + + def test_buffering_enabled(self): + # Make sure buffering is turned on + assert self.parser.buffer_text + self.parser.Parse("123", 1) + assert self.stuff == ['123'], ( + "buffered text not properly collapsed") + + def test1(self): + # XXX This test exposes more detail of Expat's text chunking than we + # XXX like, but it tests what we need to concisely. + self.setHandlers(["StartElementHandler"]) + self.parser.Parse("12\n34\n5", 1) + assert self.stuff == ( + ["", "1", "", "2", "\n", "3", "", "4\n5"]), ( + "buffering control not reacting as expected") + + def test2(self): + self.parser.Parse("1<2> \n 3", 1) + assert self.stuff == ["1<2> \n 3"], ( + "buffered text not properly collapsed") + + def test3(self): + self.setHandlers(["StartElementHandler"]) + self.parser.Parse("123", 1) + assert self.stuff == ["", "1", "", "2", "", "3"], ( + "buffered text not properly split") + + def test4(self): + self.setHandlers(["StartElementHandler", "EndElementHandler"]) + self.parser.CharacterDataHandler = None + self.parser.Parse("123", 1) + assert self.stuff == ( + ["", "", "", "", "", ""]) + + def test5(self): + self.setHandlers(["StartElementHandler", "EndElementHandler"]) + self.parser.Parse("123", 1) + assert self.stuff == ( + ["", "1", "", "", "2", "", "", "3", ""]) + + def test6(self): + self.setHandlers(["CommentHandler", "EndElementHandler", + "StartElementHandler"]) + self.parser.Parse("12345 ", 1) + assert self.stuff == ( + ["", "1", "", "", "2", "", "", "345", ""]), ( + "buffered text not properly split") + + def test7(self): + self.setHandlers(["CommentHandler", "EndElementHandler", + "StartElementHandler"]) + self.parser.Parse("12345 ", 1) + assert self.stuff == ( + ["", "1", "", "", "2", "", "", "3", + "", "4", "", "5", ""]), ( + "buffered text not properly split") + + +# Test handling of exception from callback: +class TestHandlerException: + def StartElementHandler(self, name, attrs): + raise RuntimeError(name) + + def test(self): + parser = expat.ParserCreate() + parser.StartElementHandler = self.StartElementHandler + try: + parser.Parse("", 1) + raise AssertionError + except RuntimeError, e: + assert e.args[0] == 'a', ( + "Expected RuntimeError for element 'a', but" + \ + " found %r" % e.args[0]) + + +# Test Current* members: +class TestPosition: + def StartElementHandler(self, name, attrs): + self.check_pos('s') + + def EndElementHandler(self, name): + self.check_pos('e') + + def check_pos(self, event): + pos = (event, + self.parser.CurrentByteIndex, + self.parser.CurrentLineNumber, + self.parser.CurrentColumnNumber) + assert self.upto < len(self.expected_list) + expected = self.expected_list[self.upto] + assert pos == expected, ( + 'Expected position %s, got position %s' %(pos, expected)) + self.upto += 1 + + def test(self): + self.parser = expat.ParserCreate() + self.parser.StartElementHandler = self.StartElementHandler + self.parser.EndElementHandler = self.EndElementHandler + self.upto = 0 + self.expected_list = [('s', 0, 1, 0), ('s', 5, 2, 1), ('s', 11, 3, 2), + ('e', 15, 3, 6), ('e', 17, 4, 1), ('e', 22, 5, 0)] + + xml = '\n \n \n \n' + self.parser.Parse(xml, 1) + + +class Testsf1296433: + def test_parse_only_xml_data(self): + # http://python.org/sf/1296433 + # + xml = "%s" % ('a' * 1025) + # this one doesn't crash + #xml = "%s" % ('a' * 10000) + + class SpecificException(Exception): + pass + + def handler(text): + raise SpecificException + + parser = expat.ParserCreate() + parser.CharacterDataHandler = handler + + py.test.raises(Exception, parser.Parse, xml) + +class TestChardataBuffer: + """ + test setting of chardata buffer size + """ + + def test_1025_bytes(self): + assert self.small_buffer_test(1025) == 2 + + def test_1000_bytes(self): + assert self.small_buffer_test(1000) == 1 + + def test_wrong_size(self): + parser = expat.ParserCreate() + parser.buffer_text = 1 + def f(size): + parser.buffer_size = size + + py.test.raises(TypeError, f, sys.maxint+1) + py.test.raises(ValueError, f, -1) + py.test.raises(ValueError, f, 0) + + def test_unchanged_size(self): + xml1 = ("%s" % ('a' * 512)) + xml2 = 'a'*512 + '' + parser = expat.ParserCreate() + parser.CharacterDataHandler = self.counting_handler + parser.buffer_size = 512 + parser.buffer_text = 1 + + # Feed 512 bytes of character data: the handler should be called + # once. + self.n = 0 + parser.Parse(xml1) + assert self.n == 1 + + # Reassign to buffer_size, but assign the same size. + parser.buffer_size = parser.buffer_size + assert self.n == 1 + + # Try parsing rest of the document + parser.Parse(xml2) + assert self.n == 2 + + + def test_disabling_buffer(self): + xml1 = "%s" % ('a' * 512) + xml2 = ('b' * 1024) + xml3 = "%s" % ('c' * 1024) + parser = expat.ParserCreate() + parser.CharacterDataHandler = self.counting_handler + parser.buffer_text = 1 + parser.buffer_size = 1024 + assert parser.buffer_size == 1024 + + # Parse one chunk of XML + self.n = 0 + parser.Parse(xml1, 0) + assert parser.buffer_size == 1024 + assert self.n == 1 + + # Turn off buffering and parse the next chunk. + parser.buffer_text = 0 + assert not parser.buffer_text + assert parser.buffer_size == 1024 + for i in range(10): + parser.Parse(xml2, 0) + assert self.n == 11 + + parser.buffer_text = 1 + assert parser.buffer_text + assert parser.buffer_size == 1024 + parser.Parse(xml3, 1) + assert self.n == 12 + + + + def make_document(self, bytes): + return ("" + bytes * 'a' + '') + + def counting_handler(self, text): + self.n += 1 + + def small_buffer_test(self, buffer_len): + xml = "%s" % ('a' * buffer_len) + parser = expat.ParserCreate() + parser.CharacterDataHandler = self.counting_handler + parser.buffer_size = 1024 + parser.buffer_text = 1 + + self.n = 0 + parser.Parse(xml) + return self.n + + def test_change_size_1(self): + xml1 = "%s" % ('a' * 1024) + xml2 = "aaa%s" % ('a' * 1025) + parser = expat.ParserCreate() + parser.CharacterDataHandler = self.counting_handler + parser.buffer_text = 1 + parser.buffer_size = 1024 + assert parser.buffer_size == 1024 + + self.n = 0 + parser.Parse(xml1, 0) + parser.buffer_size *= 2 + assert parser.buffer_size == 2048 + parser.Parse(xml2, 1) + assert self.n == 2 + + def test_change_size_2(self): + xml1 = "a%s" % ('a' * 1023) + xml2 = "aaa%s" % ('a' * 1025) + parser = expat.ParserCreate() + parser.CharacterDataHandler = self.counting_handler + parser.buffer_text = 1 + parser.buffer_size = 2048 + assert parser.buffer_size == 2048 + + self.n=0 + parser.Parse(xml1, 0) + parser.buffer_size /= 2 + assert parser.buffer_size == 1024 + parser.Parse(xml2, 1) + assert self.n == 4 + + def test_segfault(self): + py.test.raises(TypeError, expat.ParserCreate, 1234123123) + +def test_invalid_data(): + parser = expat.ParserCreate() + parser.Parse('invalid.xml', 0) + try: + parser.Parse("", 1) + except expat.ExpatError, e: + assert e.code == 2 # XXX is this reliable? + assert e.lineno == 1 + assert e.message.startswith('syntax error') + else: + py.test.fail("Did not raise") + Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_resource.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_resource.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,31 @@ +from __future__ import absolute_import +from ..ctypes_config_cache import rebuild +rebuild.rebuild_one('resource.ctc.py') + +from .. import resource + +def test_resource(): + x = resource.getrusage(resource.RUSAGE_SELF) + assert len(x) == 16 + assert x[0] == x[-16] == x.ru_utime + assert x[1] == x[-15] == x.ru_stime + assert x[2] == x[-14] == x.ru_maxrss + assert x[3] == x[-13] == x.ru_ixrss + assert x[4] == x[-12] == x.ru_idrss + assert x[5] == x[-11] == x.ru_isrss + assert x[6] == x[-10] == x.ru_minflt + assert x[7] == x[-9] == x.ru_majflt + assert x[8] == x[-8] == x.ru_nswap + assert x[9] == x[-7] == x.ru_inblock + assert x[10] == x[-6] == x.ru_oublock + assert x[11] == x[-5] == x.ru_msgsnd + assert x[12] == x[-4] == x.ru_msgrcv + assert x[13] == x[-3] == x.ru_nsignals + assert x[14] == x[-2] == x.ru_nvcsw + assert x[15] == x[-1] == x.ru_nivcsw + for i in range(16): + if i < 2: + expected_type = float + else: + expected_type = (int, long) + assert isinstance(x[i], expected_type) Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_runpy.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_runpy.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,162 @@ +# Test the runpy module +from __future__ import absolute_import +import py +import unittest +import os +import os.path +import sys +import tempfile +from .. import runpy +from ..runpy import _run_module_code, run_module + +sys.modules['my_runpy'] = runpy # so that code in exec can freely import it + # without messing with __name__ and/or sys.path + +verbose = 0 + +# Set up the test code and expected results +class TestRunModuleCode: + + expected_result = ["Top level assignment", "Lower level reference"] + test_source = py.code.Source(""" + # Check basic code execution + result = ['Top level assignment'] + def f(): + result.append('Lower level reference') + f() + # Check the sys module + import sys + run_argv0 = sys.argv[0] + if __name__ in sys.modules: + run_name = sys.modules[__name__].__name__ + # Check nested operation + import my_runpy # this is imported as ..runpy as manually saved into sys.modules, see above + nested = my_runpy._run_module_code('x=1\', mod_name='', + alter_sys=True) + """).compile() + + + def test_run_module_code(self): + initial = object() + name = "" + file = "Some other nonsense" + loader = "Now you're just being silly" + d1 = dict(initial=initial) + saved_argv0 = sys.argv[0] + d2 = _run_module_code(self.test_source, + d1, + name, + file, + loader, + True) + assert "result" not in d1 + assert d2["initial"] is initial + assert d2["result"] == self.expected_result + assert d2["nested"]["x"] == 1 + assert d2["__name__"] is name + assert d2["run_name"] is name + assert d2["__file__"] is file + assert d2["run_argv0"] is file + assert d2["__loader__"] is loader + assert sys.argv[0] is saved_argv0 + assert name not in sys.modules + + def test_run_module_code_defaults(self): + saved_argv0 = sys.argv[0] + d = _run_module_code(self.test_source) + assert d["result"] == self.expected_result + assert d["__name__"] is None + assert d["__file__"] is None + assert d["__loader__"] is None + assert d["run_argv0"] is saved_argv0 + assert "run_name" not in d + assert sys.argv[0] is saved_argv0 + +class TestRunModule: + + def expect_import_error(self, mod_name): + try: + run_module(mod_name) + except ImportError: + pass + else: + assert false, "Expected import error for " + mod_name + + def test_invalid_names(self): + self.expect_import_error("sys") + self.expect_import_error("sys.imp.eric") + self.expect_import_error("os.path.half") + self.expect_import_error("a.bee") + self.expect_import_error(".howard") + self.expect_import_error("..eaten") + + def test_library_module(self): + run_module("runpy") + + def _make_pkg(self, source, depth): + pkg_name = "__runpy_pkg__" + init_fname = "__init__"+os.extsep+"py" + test_fname = "runpy_test"+os.extsep+"py" + pkg_dir = sub_dir = tempfile.mkdtemp() + if verbose: print " Package tree in:", sub_dir + sys.path.insert(0, pkg_dir) + if verbose: print " Updated sys.path:", sys.path[0] + for i in range(depth): + sub_dir = os.path.join(sub_dir, pkg_name) + os.mkdir(sub_dir) + if verbose: print " Next level in:", sub_dir + pkg_fname = os.path.join(sub_dir, init_fname) + pkg_file = open(pkg_fname, "w") + pkg_file.write("__path__ = [%r]\n" % sub_dir) + pkg_file.close() + if verbose: print " Created:", pkg_fname + mod_fname = os.path.join(sub_dir, test_fname) + mod_file = open(mod_fname, "w") + mod_file.write(source) + mod_file.close() + if verbose: print " Created:", mod_fname + mod_name = (pkg_name+".")*depth + "runpy_test" + return pkg_dir, mod_fname, mod_name + + def _del_pkg(self, top, depth, mod_name): + for root, dirs, files in os.walk(top, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + os.rmdir(top) + if verbose: print " Removed package tree" + for i in range(depth+1): # Don't forget the module itself + parts = mod_name.rsplit(".", i) + entry = parts[0] + del sys.modules[entry] + if verbose: print " Removed sys.modules entries" + del sys.path[0] + if verbose: print " Removed sys.path entry" + + def _check_module(self, depth): + pkg_dir, mod_fname, mod_name = ( + self._make_pkg("x=1\n", depth)) + try: + if verbose: print "Running from source:", mod_name + d1 = run_module(mod_name) # Read from source + __import__(mod_name) + os.remove(mod_fname) + + #--- the block below is to check that "imp.find_module" + #--- manages to import the .pyc file alone. We don't + #--- support it in PyPy in the default configuration. + return + + if verbose: print "Running from compiled:", mod_name + d2 = run_module(mod_name) # Read from bytecode + finally: + self._del_pkg(pkg_dir, depth, mod_name) + assert d1["x"] == d2["x"] == 1 + if verbose: print "Module executed successfully" + + def test_run_module(self): + for depth in range(4): + if verbose: print "Testing package depth:", depth + self._check_module(depth) + Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_sha_extra.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_sha_extra.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,32 @@ +# Testing sha module (NIST's Secure Hash Algorithm) + +# use the three examples from Federal Information Processing Standards +# Publication 180-1, Secure Hash Standard, 1995 April 17 +# http://www.itl.nist.gov/div897/pubs/fip180-1.htm +from __future__ import absolute_import +from .. import sha + +class TestSHA: + def check(self, data, digest): + computed = sha.new(data).hexdigest() + assert computed == digest + + def test_case_1(self): + self.check("abc", + "a9993e364706816aba3e25717850c26c9cd0d89d") + + def test_case_2(self): + self.check("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "84983e441c3bd26ebaae4aa1f95129e5e54670f1") + + def disabled_too_slow_test_case_3(self): + self.check("a" * 1000000, + "34aa973cd4c4daa4f61eeb2bdbad27316534016f") + + +def test_attributes(): + assert sha.digest_size == 20 + assert sha.digestsize == 20 + assert sha.blocksize == 1 + assert sha.sha().digest_size == 20 + assert sha.sha().digestsize == 20 Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_stackless.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_stackless.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,603 @@ +""" +These tests are supposed to run on the following platforms: +1. CStackless +2. CPython (with the stackless_new module in the path +3. pypy-c +""" +from __future__ import absolute_import +from py.test import skip +try: + import stackless +except ImportError: + try: + from .. import stackless + except ImportError, e: + skip('cannot import stackless: %s' % (e,)) + +SHOW_STRANGE = False + +def dprint(txt): + if SHOW_STRANGE: + print txt + +class Test_Stackless: + + def test_simple(self): + rlist = [] + + def f(): + rlist.append('f') + + def g(): + rlist.append('g') + stackless.schedule() + + def main(): + rlist.append('m') + cg = stackless.tasklet(g)() + cf = stackless.tasklet(f)() + stackless.run() + rlist.append('m') + + main() + + assert stackless.getcurrent() is stackless.getmain() + assert rlist == 'm g f m'.split() + + def test_with_channel(self): + pref = {} + pref[-1] = ['s0', 'r0', 's1', 'r1', 's2', 'r2', + 's3', 'r3', 's4', 'r4', 's5', 'r5', + 's6', 'r6', 's7', 'r7', 's8', 'r8', + 's9', 'r9'] + pref[0] = ['s0', 'r0', 's1', 's2', 'r1', 'r2', + 's3', 's4', 'r3', 'r4', 's5', 's6', + 'r5', 'r6', 's7', 's8', 'r7', 'r8', + 's9', 'r9'] + pref[1] = ['s0', 's1', 'r0', 's2', 'r1', 's3', + 'r2', 's4', 'r3', 's5', 'r4', 's6', + 'r5', 's7', 'r6', 's8', 'r7', 's9', + 'r8', 'r9'] + rlist = [] + + def f(outchan): + for i in range(10): + rlist.append('s%s' % i) + outchan.send(i) + outchan.send(-1) + + def g(inchan): + while 1: + val = inchan.receive() + if val == -1: + break + rlist.append('r%s' % val) + + for preference in [-1, 0, 1]: + rlist = [] + ch = stackless.channel() + ch.preference = preference + t1 = stackless.tasklet(f)(ch) + t2 = stackless.tasklet(g)(ch) + + stackless.run() + + assert len(rlist) == 20 + assert rlist == pref[preference] + + def test_send_counter(self): + import random + + numbers = range(20) + random.shuffle(numbers) + + def counter(n, ch): + for i in xrange(n): + stackless.schedule() + ch.send(n) + + ch = stackless.channel() + for each in numbers: + stackless.tasklet(counter)(each, ch) + + stackless.run() + + rlist = [] + while ch.balance: + rlist.append(ch.receive()) + + numbers.sort() + assert rlist == numbers + + def test_receive_counter(self): + import random + + numbers = range(20) + random.shuffle(numbers) + + rlist = [] + def counter(n, ch): + for i in xrange(n): + stackless.schedule() + ch.receive() + rlist.append(n) + + ch = stackless.channel() + for each in numbers: + stackless.tasklet(counter)(each, ch) + + stackless.run() + + while ch.balance: + ch.send(None) + + numbers.sort() + assert rlist == numbers + + def test_scheduling_cleanup(self): + rlist = [] + def f(): + rlist.append('fb') + stackless.schedule() + rlist.append('fa') + + def g(): + rlist.append('gb') + stackless.schedule() + rlist.append('ga') + + def h(): + rlist.append('hb') + stackless.schedule() + rlist.append('ha') + + tf = stackless.tasklet(f)() + tg = stackless.tasklet(g)() + th = stackless.tasklet(h)() + + rlist.append('mb') + stackless.run() + rlist.append('ma') + + assert rlist == 'mb fb gb hb fa ga ha ma'.split() + + def test_except(self): + rlist = [] + def f(): + rlist.append('f') + return 1/0 + + def g(): + rlist.append('bg') + stackless.schedule() + rlist.append('ag') + + def h(): + rlist.append('bh') + stackless.schedule() + rlist.append('ah') + + tg = stackless.tasklet(g)() + tf = stackless.tasklet(f)() + th = stackless.tasklet(h)() + + try: + stackless.run() + # cheating, can't test for ZeroDivisionError + except Exception, e: + rlist.append('E') + stackless.schedule() + stackless.schedule() + + assert rlist == "bg f E bh ag ah".split() + + def test_except_full(self): + rlist = [] + def f(): + rlist.append('f') + return 1/0 + + def g(): + rlist.append('bg') + stackless.schedule() + rlist.append('ag') + + def h(): + rlist.append('bh') + stackless.schedule() + rlist.append('ah') + + tg = stackless.tasklet(g)() + tf = stackless.tasklet(f)() + th = stackless.tasklet(h)() + + try: + stackless.run() + except ZeroDivisionError: + rlist.append('E') + stackless.schedule() + stackless.schedule() + + assert rlist == "bg f E bh ag ah".split() + + def test_kill(self): + def f():pass + t = stackless.tasklet(f)() + t.kill() + assert not t.alive + + def test_catch_taskletexit(self): + # Tests if TaskletExit can be caught in the tasklet being killed. + global taskletexit + taskletexit = False + + def f(): + try: + stackless.schedule() + except TaskletExit: + global TaskletExit + taskletexit = True + raise + + t = stackless.tasklet(f)() + t.run() + assert t.alive + t.kill() + assert not t.alive + assert taskletexit + + def test_autocatch_taskletexit(self): + # Tests if TaskletExit is caught correctly in stackless.tasklet.setup(). + def f(): + stackless.schedule() + + t = stackless.tasklet(f)() + t.run() + t.kill() + + + # tests inspired from simple stackless.com examples + + def test_construction(self): + output = [] + def print_(*args): + output.append(args) + + def aCallable(value): + print_("aCallable:", value) + + task = stackless.tasklet(aCallable) + task.setup('Inline using setup') + + stackless.run() + assert output == [("aCallable:", 'Inline using setup')] + + + del output[:] + task = stackless.tasklet(aCallable) + task('Inline using ()') + + stackless.run() + assert output == [("aCallable:", 'Inline using ()')] + + del output[:] + task = stackless.tasklet() + task.bind(aCallable) + task('Bind using ()') + + stackless.run() + assert output == [("aCallable:", 'Bind using ()')] + + def test_simple_channel(self): + output = [] + def print_(*args): + output.append(args) + + def Sending(channel): + print_("sending") + channel.send("foo") + + def Receiving(channel): + print_("receiving") + print_(channel.receive()) + + ch=stackless.channel() + + task=stackless.tasklet(Sending)(ch) + + # Note: the argument, schedule is taking is the value, + # schedule returns, not the task that runs next + + #stackless.schedule(task) + stackless.schedule() + task2=stackless.tasklet(Receiving)(ch) + #stackless.schedule(task2) + stackless.schedule() + + stackless.run() + + assert output == [('sending',), ('receiving',), ('foo',)] + + def test_balance_zero(self): + ch=stackless.channel() + assert ch.balance == 0 + + def test_balance_send(self): + def Sending(channel): + channel.send("foo") + + ch=stackless.channel() + + task=stackless.tasklet(Sending)(ch) + stackless.run() + + assert ch.balance == 1 + + def test_balance_recv(self): + def Receiving(channel): + channel.receive() + + ch=stackless.channel() + + task=stackless.tasklet(Receiving)(ch) + stackless.run() + + assert ch.balance == -1 + + def test_run(self): + output = [] + def print_(*args): + output.append(args) + + def f(i): + print_(i) + + stackless.tasklet(f)(1) + stackless.tasklet(f)(2) + stackless.run() + + assert output == [(1,), (2,)] + + def test_schedule(self): + output = [] + def print_(*args): + output.append(args) + + def f(i): + print_(i) + + stackless.tasklet(f)(1) + stackless.tasklet(f)(2) + stackless.schedule() + + assert output == [(1,), (2,)] + + + def test_cooperative(self): + output = [] + def print_(*args): + output.append(args) + + def Loop(i): + for x in range(3): + stackless.schedule() + print_("schedule", i) + + stackless.tasklet(Loop)(1) + stackless.tasklet(Loop)(2) + stackless.run() + + assert output == [('schedule', 1), ('schedule', 2), + ('schedule', 1), ('schedule', 2), + ('schedule', 1), ('schedule', 2),] + + def test_channel_callback(self): + res = [] + cb = [] + def callback_function(chan, task, sending, willblock): + cb.append((chan, task, sending, willblock)) + stackless.set_channel_callback(callback_function) + def f(chan): + chan.send('hello') + val = chan.receive() + res.append(val) + + chan = stackless.channel() + task = stackless.tasklet(f)(chan) + val = chan.receive() + res.append(val) + chan.send('world') + assert res == ['hello','world'] + maintask = stackless.getmain() + assert cb == [ + (chan, maintask, 0, 1), + (chan, task, 1, 0), + (chan, maintask, 1, 1), + (chan, task, 0, 0) + ] + + def test_schedule_callback(self): + res = [] + cb = [] + def schedule_cb(prev, next): + cb.append((prev, next)) + + stackless.set_schedule_callback(schedule_cb) + def f(i): + res.append('A_%s' % i) + stackless.schedule() + res.append('B_%s' % i) + + t1 = stackless.tasklet(f)(1) + t2 = stackless.tasklet(f)(2) + maintask = stackless.getmain() + stackless.run() + assert res == ['A_1', 'A_2', 'B_1', 'B_2'] + assert len(cb) == 5 + assert cb[0] == (maintask, t1) + assert cb[1] == (t1, t2) + assert cb[2] == (t2, t1) + assert cb[3] == (t1, t2) + assert cb[4] == (t2, maintask) + + def test_bomb(self): + try: + 1/0 + except: + import sys + b = stackless.bomb(*sys.exc_info()) + assert b.type is ZeroDivisionError + assert str(b.value).startswith('integer division') + assert b.traceback is not None + + def test_send_exception(self): + def exp_sender(chan): + chan.send_exception(Exception, 'test') + + def exp_recv(chan): + try: + val = chan.receive() + except Exception, exp: + assert exp.__class__ is Exception + assert str(exp) == 'test' + + chan = stackless.channel() + t1 = stackless.tasklet(exp_recv)(chan) + t2 = stackless.tasklet(exp_sender)(chan) + stackless.run() + + def test_send_sequence(self): + res = [] + lst = [1,2,3,4,5,6,None] + iterable = iter(lst) + chan = stackless.channel() + def f(chan): + r = chan.receive() + while r: + res.append(r) + r = chan.receive() + + t = stackless.tasklet(f)(chan) + chan.send_sequence(iterable) + assert res == [1,2,3,4,5,6] + + def test_getruncount(self): + assert stackless.getruncount() == 1 + def with_schedule(): + assert stackless.getruncount() == 2 + + t1 = stackless.tasklet(with_schedule)() + assert stackless.getruncount() == 2 + stackless.schedule() + def with_run(): + assert stackless.getruncount() == 1 + + t2 = stackless.tasklet(with_run)() + stackless.run() + + def test_schedule_return(self): + def f():pass + t1= stackless.tasklet(f)() + r = stackless.schedule() + assert r is stackless.getmain() + t2 = stackless.tasklet(f)() + r = stackless.schedule('test') + assert r == 'test' + + def test_simple_pipe(self): + def pipe(X_in, X_out): + foo = X_in.receive() + X_out.send(foo) + + X, Y = stackless.channel(), stackless.channel() + t = stackless.tasklet(pipe)(X, Y) + stackless.run() + X.send(42) + assert Y.receive() == 42 + + def test_nested_pipe(self): + dprint('tnp ==== 1') + def pipe(X, Y): + dprint('tnp_P ==== 1') + foo = X.receive() + dprint('tnp_P ==== 2') + Y.send(foo) + dprint('tnp_P ==== 3') + + def nest(X, Y): + X2, Y2 = stackless.channel(), stackless.channel() + t = stackless.tasklet(pipe)(X2, Y2) + dprint('tnp_N ==== 1') + X_Val = X.receive() + dprint('tnp_N ==== 2') + X2.send(X_Val) + dprint('tnp_N ==== 3') + Y2_Val = Y2.receive() + dprint('tnp_N ==== 4') + Y.send(Y2_Val) + dprint('tnp_N ==== 5') + + X, Y = stackless.channel(), stackless.channel() + t1 = stackless.tasklet(nest)(X, Y) + X.send(13) + dprint('tnp ==== 2') + res = Y.receive() + dprint('tnp ==== 3') + assert res == 13 + if SHOW_STRANGE: + raise Exception('force prints') + + def test_wait_two(self): + """ + A tasklets/channels adaptation of the test_wait_two from the + logic object space + """ + def sleep(X, Y): + dprint('twt_S ==== 1') + value = X.receive() + dprint('twt_S ==== 2') + Y.send((X, value)) + dprint('twt_S ==== 3') + + def wait_two(X, Y, Ret_chan): + Barrier = stackless.channel() + stackless.tasklet(sleep)(X, Barrier) + stackless.tasklet(sleep)(Y, Barrier) + dprint('twt_W ==== 1') + ret = Barrier.receive() + dprint('twt_W ==== 2') + if ret[0] == X: + Ret_chan.send((1, ret[1])) + else: + Ret_chan.send((2, ret[1])) + dprint('twt_W ==== 3') + + X = stackless.channel() + Y = stackless.channel() + Ret_chan = stackless.channel() + + stackless.tasklet(wait_two)(X, Y, Ret_chan) + + dprint('twt ==== 1') + Y.send(42) + + dprint('twt ==== 2') + X.send(42) + dprint('twt ==== 3') + value = Ret_chan.receive() + dprint('twt ==== 4') + assert value == (2, 42) + + + def test_schedule_return_value(self): + + def task(val): + value = stackless.schedule(val) + assert value == val + + stackless.tasklet(task)(10) + stackless.tasklet(task)(5) + + stackless.run() + + Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_stackless_pickling.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_stackless_pickling.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,66 @@ +""" +this test should probably not run from CPython or py.py. +I'm not entirely sure, how to do that. +""" +from __future__ import absolute_import +from py.test import skip +try: + import stackless +except ImportError: + try: + from .. import stackless as stackless + except ImportError, e: + skip('cannot import stackless: %s' % (e,)) + + + +class Test_StacklessPickling: + + def test_basic_tasklet_pickling(self): + try: + import stackless + except ImportError: + skip("can't load stackless and don't know why!!!") + from stackless import run, schedule, tasklet + import pickle + + output = [] + + import new + + mod = new.module('mod') + mod.output = output + + exec """from stackless import schedule + +def aCallable(name): + output.append(('b', name)) + schedule() + output.append(('a', name)) +""" in mod.__dict__ + import sys + sys.modules['mod'] = mod + aCallable = mod.aCallable + + + tasks = [] + for name in "ABCDE": + tasks.append(tasklet(aCallable)(name)) + + schedule() + + assert output == [('b', x) for x in "ABCDE"] + del output[:] + pickledTasks = pickle.dumps(tasks) + + schedule() + assert output == [('a', x) for x in "ABCDE"] + del output[:] + + unpickledTasks = pickle.loads(pickledTasks) + for task in unpickledTasks: + task.insert() + + schedule() + assert output == [('a', x) for x in "ABCDE"] + Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_struct_extra.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_struct_extra.py Tue Jul 6 00:00:33 2010 @@ -0,0 +1,25 @@ +from __future__ import absolute_import +from .. import struct + +def test_simple(): + morezeros = '\x00' * (struct.calcsize('l')-4) + assert struct.pack('')) nofile = type(_pypy_interact)('nofile', 'foo') assert repr(nofile) == "" Modified: pypy/branch/fast-forward/pypy/interpreter/unicodehelper.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/unicodehelper.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/unicodehelper.py Tue Jul 6 00:00:33 2010 @@ -1,30 +1,10 @@ -from pypy.interpreter import gateway +from pypy.module._codecs import interp_codecs -app = gateway.applevel(r''' - def PyUnicode_DecodeUnicodeEscape(data): - import _codecs - return _codecs.unicode_escape_decode(data)[0] +def PyUnicode_AsEncodedString(space, w_data, w_encoding): + return interp_codecs.encode(space, w_data, w_encoding) - def PyUnicode_DecodeRawUnicodeEscape(data): - import _codecs - return _codecs.raw_unicode_escape_decode(data)[0] - - def PyUnicode_DecodeUTF8(data): - import _codecs - return _codecs.utf_8_decode(data)[0] - - def PyUnicode_AsEncodedString(data, encoding): - import _codecs - return _codecs.encode(data, encoding) - - def PyUnicode_EncodeUTF8(data): - import _codecs - return _codecs.utf_8_encode(data)[0] - -''') - -PyUnicode_DecodeUnicodeEscape = app.interphook('PyUnicode_DecodeUnicodeEscape') -PyUnicode_DecodeRawUnicodeEscape = app.interphook('PyUnicode_DecodeRawUnicodeEscape') -PyUnicode_DecodeUTF8 = app.interphook('PyUnicode_DecodeUTF8') -PyUnicode_AsEncodedString = app.interphook('PyUnicode_AsEncodedString') -PyUnicode_EncodeUTF8 = app.interphook('PyUnicode_EncodeUTF8') +# These functions take and return unwrapped rpython strings and unicodes +PyUnicode_DecodeUnicodeEscape = interp_codecs.make_raw_decoder('unicode_escape') +PyUnicode_DecodeRawUnicodeEscape = interp_codecs.make_raw_decoder('raw_unicode_escape') +PyUnicode_DecodeUTF8 = interp_codecs.make_raw_decoder('utf_8') +PyUnicode_EncodeUTF8 = interp_codecs.make_raw_encoder('utf_8') Modified: pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py Tue Jul 6 00:00:33 2010 @@ -832,13 +832,16 @@ raise Exception("Nonsense type %s" % TYPE) failindex = self.cpu._execute_token(loop_token) + jd = loop_token.outermost_jitdriver_sd + assert jd is not None, ("call_assembler(): the loop_token needs " + "to have 'outermost_jitdriver_sd'") + if jd.index_of_virtualizable != -1: + vable = args[jd.index_of_virtualizable] + else: + vable = lltype.nullptr(llmemory.GCREF.TO) + assembler_helper_ptr = jd.assembler_helper_adr.ptr # fish try: - if self.cpu.index_of_virtualizable != -1: - return self.cpu.assembler_helper_ptr(failindex, - args[self.cpu.index_of_virtualizable]) - else: - return self.cpu.assembler_helper_ptr(failindex, - lltype.nullptr(llmemory.GCREF.TO)) + return assembler_helper_ptr(failindex, vable) except LLException, lle: assert _last_exception is None, "exception left behind" _last_exception = lle @@ -1196,10 +1199,18 @@ array = array._obj.container return cast_to_int(array.getitem(index)) +def do_getarrayitem_raw_int(array, index): + array = array.adr.ptr._obj + return cast_to_int(array.getitem(index)) + def do_getarrayitem_gc_float(array, index): array = array._obj.container return cast_to_float(array.getitem(index)) +def do_getarrayitem_raw_float(array, index): + array = array.adr.ptr._obj + return cast_to_float(array.getitem(index)) + def do_getarrayitem_gc_ptr(array, index): array = array._obj.container return cast_to_ptr(array.getitem(index)) @@ -1248,12 +1259,24 @@ newvalue = cast_from_int(ITEMTYPE, newvalue) array.setitem(index, newvalue) +def do_setarrayitem_raw_int(array, index, newvalue): + array = array.adr.ptr + ITEMTYPE = lltype.typeOf(array).TO.OF + newvalue = cast_from_int(ITEMTYPE, newvalue) + array._obj.setitem(index, newvalue) + def do_setarrayitem_gc_float(array, index, newvalue): array = array._obj.container ITEMTYPE = lltype.typeOf(array).OF newvalue = cast_from_float(ITEMTYPE, newvalue) array.setitem(index, newvalue) +def do_setarrayitem_raw_float(array, index, newvalue): + array = array.adr.ptr + ITEMTYPE = lltype.typeOf(array).TO.OF + newvalue = cast_from_int(ITEMTYPE, newvalue) + array._obj.setitem(index, newvalue) + def do_setarrayitem_gc_ptr(array, index, newvalue): array = array._obj.container ITEMTYPE = lltype.typeOf(array).OF Modified: pypy/branch/fast-forward/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llgraph/runner.py Tue Jul 6 00:00:33 2010 @@ -294,7 +294,6 @@ return llimpl.grab_exc_value() def arraydescrof(self, A): - assert isinstance(A, lltype.GcArray) assert A.OF != lltype.Void size = symbolic.get_size(A) token = history.getkind(A.OF) @@ -317,12 +316,18 @@ def bh_getarrayitem_gc_i(self, arraydescr, array, index): assert isinstance(arraydescr, Descr) return llimpl.do_getarrayitem_gc_int(array, index) + def bh_getarrayitem_raw_i(self, arraydescr, array, index): + assert isinstance(arraydescr, Descr) + return llimpl.do_getarrayitem_raw_int(array, index) def bh_getarrayitem_gc_r(self, arraydescr, array, index): assert isinstance(arraydescr, Descr) return llimpl.do_getarrayitem_gc_ptr(array, index) def bh_getarrayitem_gc_f(self, arraydescr, array, index): assert isinstance(arraydescr, Descr) return llimpl.do_getarrayitem_gc_float(array, index) + def bh_getarrayitem_raw_f(self, arraydescr, array, index): + assert isinstance(arraydescr, Descr) + return llimpl.do_getarrayitem_raw_float(array, index) def bh_getfield_gc_i(self, struct, fielddescr): assert isinstance(fielddescr, Descr) @@ -372,6 +377,10 @@ assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_int(array, index, newvalue) + def bh_setarrayitem_raw_i(self, arraydescr, array, index, newvalue): + assert isinstance(arraydescr, Descr) + llimpl.do_setarrayitem_raw_int(array, index, newvalue) + def bh_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_ptr(array, index, newvalue) @@ -380,6 +389,10 @@ assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_float(array, index, newvalue) + def bh_setarrayitem_raw_f(self, arraydescr, array, index, newvalue): + assert isinstance(arraydescr, Descr) + llimpl.do_setarrayitem_raw_float(array, index, newvalue) + def bh_setfield_gc_i(self, struct, fielddescr, newvalue): assert isinstance(fielddescr, Descr) llimpl.do_setfield_gc_int(struct, fielddescr.ofs, newvalue) Modified: pypy/branch/fast-forward/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/model.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/model.py Tue Jul 6 00:00:33 2010 @@ -3,10 +3,6 @@ class AbstractCPU(object): supports_floats = False - _got_exception = None - # assembler_helper_ptr - a pointer to helper to call after a direct - # assembler call - portal_calldescr = None done_with_this_frame_void_v = -1 done_with_this_frame_int_v = -1 done_with_this_frame_ref_v = -1 Modified: pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py Tue Jul 6 00:00:33 2010 @@ -1698,16 +1698,22 @@ assert self.cpu.get_latest_value_int(0) == 10 called.append(failindex) return 4 + 9 - self.cpu.index_of_virtualizable = -1 - self.cpu.assembler_helper_ptr = llhelper(lltype.Ptr(lltype.FuncType - ([lltype.Signed, llmemory.GCREF], lltype.Signed)), assembler_helper) - + + FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF], + lltype.Signed)) + class FakeJitDriverSD: + index_of_virtualizable = -1 + _assembler_helper_ptr = llhelper(FUNCPTR, assembler_helper) + assembler_helper_adr = llmemory.cast_ptr_to_adr( + _assembler_helper_ptr) + ops = ''' [i0, i1] i2 = int_add(i0, i1) finish(i2)''' loop = parse(ops) looptoken = LoopToken() + looptoken.outermost_jitdriver_sd = FakeJitDriverSD() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) ARGS = [lltype.Signed, lltype.Signed] RES = lltype.Signed @@ -1739,9 +1745,15 @@ assert self.cpu.get_latest_value_float(0) == 1.2 + 3.2 called.append(failindex) return 13.5 - self.cpu.index_of_virtualizable = -1 - self.cpu.assembler_helper_ptr = llhelper(lltype.Ptr(lltype.FuncType - ([lltype.Signed, llmemory.GCREF], lltype.Float)), assembler_helper) + + FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF], + lltype.Float)) + class FakeJitDriverSD: + index_of_virtualizable = -1 + _assembler_helper_ptr = llhelper(FUNCPTR, assembler_helper) + assembler_helper_adr = llmemory.cast_ptr_to_adr( + _assembler_helper_ptr) + ARGS = [lltype.Float, lltype.Float] RES = lltype.Float self.cpu.portal_calldescr = self.cpu.calldescrof( @@ -1753,6 +1765,7 @@ finish(f2)''' loop = parse(ops) looptoken = LoopToken() + looptoken.outermost_jitdriver_sd = FakeJitDriverSD() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) self.cpu.set_future_value_float(0, 1.2) self.cpu.set_future_value_float(1, 2.3) Modified: pypy/branch/fast-forward/pypy/jit/backend/test/support.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/test/support.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/test/support.py Tue Jul 6 00:00:33 2010 @@ -62,11 +62,12 @@ warmrunnerdesc = WarmRunnerDesc(t, translate_support_code=True, CPUClass=self.CPUClass, **kwds) - warmrunnerdesc.state.set_param_threshold(3) # for tests - warmrunnerdesc.state.set_param_trace_eagerness(2) # for tests - warmrunnerdesc.state.set_param_trace_limit(trace_limit) - warmrunnerdesc.state.set_param_inlining(inline) - warmrunnerdesc.state.set_param_optimizer(OPTIMIZER_FULL) + for jd in warmrunnerdesc.jitdrivers_sd: + jd.warmstate.set_param_threshold(3) # for tests + jd.warmstate.set_param_trace_eagerness(2) # for tests + jd.warmstate.set_param_trace_limit(trace_limit) + jd.warmstate.set_param_inlining(inline) + jd.warmstate.set_param_optimizer(OPTIMIZER_FULL) mixlevelann = warmrunnerdesc.annhelper entry_point_graph = mixlevelann.getgraph(entry_point, [s_list_of_strings], annmodel.SomeInteger()) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py Tue Jul 6 00:00:33 2010 @@ -109,7 +109,6 @@ self.malloc_array_func_addr = 0 self.malloc_str_func_addr = 0 self.malloc_unicode_func_addr = 0 - self.assembler_helper_adr = 0 self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) @@ -144,14 +143,6 @@ ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, ll_new_unicode) - if we_are_translated(): - self.assembler_helper_adr = self.cpu.cast_ptr_to_int( - self.cpu.assembler_helper_ptr) - else: - if getattr(self.cpu, 'assembler_helper_ptr', None): - self.assembler_helper_adr = self.cpu.cast_ptr_to_int( - self.cpu.assembler_helper_ptr) - # done # we generate the loop body in 'mc' # 'mc2' is for guard recovery code @@ -1389,7 +1380,10 @@ je_location = mc.get_relative_pos() # # Path A: use assembler_helper_adr - self._emit_call(rel32(self.assembler_helper_adr), [eax, arglocs[1]], 0, + jd = descr.outermost_jitdriver_sd + assert jd is not None + asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) + self._emit_call(rel32(asm_helper_adr), [eax, arglocs[1]], 0, tmp=ecx, force_mc=True, mc=mc) if isinstance(result_loc, MODRM64): mc.FSTP(result_loc) @@ -1403,9 +1397,9 @@ mc.overwrite(je_location - 1, [chr(offset)]) # # Reset the vable token --- XXX really too much special logic here:-( - if self.cpu.index_of_virtualizable >= 0: + if jd.index_of_virtualizable >= 0: from pypy.jit.backend.llsupport.descr import BaseFieldDescr - fielddescr = self.cpu.vable_token_descr + fielddescr = jd.vable_token_descr assert isinstance(fielddescr, BaseFieldDescr) ofs = fielddescr.offset mc.MOV(eax, arglocs[1]) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py Tue Jul 6 00:00:33 2010 @@ -636,10 +636,14 @@ self._consider_call(op, guard_op) def consider_call_assembler(self, op, guard_op): - descr = op.descr portal_calldescr = self.assembler.cpu.portal_calldescr size = portal_calldescr.get_result_size(self.translate_support_code) - vable_index = self.assembler.cpu.index_of_virtualizable + # + descr = op.descr + assert isinstance(descr, LoopToken) + jd = descr.outermost_jitdriver_sd + assert jd is not None + vable_index = jd.index_of_virtualizable if vable_index >= 0: self.rm._sync_var(op.args[vable_index]) vable = self.fm.loc(op.args[vable_index], 1) @@ -779,12 +783,14 @@ arglocs.append(self.loc(op.args[0])) return self._call(op, arglocs) # boehm GC (XXX kill the following code at some point) - scale_of_field, basesize, _ = self._unpack_arraydescr(op.descr) - return self._malloc_varsize(basesize, 0, scale_of_field, op.args[0], - op.result) + scale_of_field, basesize, ofs_length, _ = ( + self._unpack_arraydescr(op.descr)) + return self._malloc_varsize(basesize, ofs_length, scale_of_field, + op.args[0], op.result) def _unpack_arraydescr(self, arraydescr): assert isinstance(arraydescr, BaseArrayDescr) + ofs_length = arraydescr.get_ofs_length(self.translate_support_code) ofs = arraydescr.get_base_size(self.translate_support_code) size = arraydescr.get_item_size(self.translate_support_code) ptr = arraydescr.is_array_of_pointers() @@ -792,7 +798,7 @@ while (1 << scale) < size: scale += 1 assert (1 << scale) == size - return scale, ofs, ptr + return scale, ofs, ofs_length, ptr def _unpack_fielddescr(self, fielddescr): assert isinstance(fielddescr, BaseFieldDescr) @@ -827,7 +833,7 @@ consider_unicodesetitem = consider_strsetitem def consider_setarrayitem_gc(self, op): - scale, ofs, ptr = self._unpack_arraydescr(op.descr) + scale, ofs, _, ptr = self._unpack_arraydescr(op.descr) base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) if scale == 0: need_lower_byte = True @@ -854,7 +860,7 @@ consider_getfield_gc_pure = consider_getfield_gc def consider_getarrayitem_gc(self, op): - scale, ofs, _ = self._unpack_arraydescr(op.descr) + scale, ofs, _, _ = self._unpack_arraydescr(op.descr) base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) ofs_loc = self.rm.make_sure_var_in_reg(op.args[1], op.args) self.rm.possibly_free_vars(op.args) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_zrpy_gc.py Tue Jul 6 00:00:33 2010 @@ -105,6 +105,12 @@ def test_compile_boehm(): myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) + @dont_look_inside + def see(lst, n): + assert len(lst) == 3 + assert lst[0] == n+10 + assert lst[1] == n+20 + assert lst[2] == n+30 def main(n, x): while n > 0: myjitdriver.can_enter_jit(n=n, x=x) @@ -112,6 +118,7 @@ y = X() y.foo = x.foo n -= y.foo + see([n+10, n+20, n+30], n) res = compile_and_run(get_entry(get_g(main)), "boehm", jit=True) assert int(res) >= 16 Modified: pypy/branch/fast-forward/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/assembler.py Tue Jul 6 00:00:33 2010 @@ -60,37 +60,38 @@ self.code.append(chr(reg.index)) def emit_const(self, const, kind, allow_short=False): - if const not in self.constants_dict: - value = const.value - TYPE = lltype.typeOf(value) - if kind == 'int': - if isinstance(TYPE, lltype.Ptr): - assert TYPE.TO._gckind == 'raw' - self.see_raw_object(value) - value = llmemory.cast_ptr_to_adr(value) - TYPE = llmemory.Address - if TYPE == llmemory.Address: - value = heaptracker.adr2int(value) - elif not isinstance(value, ComputedIntSymbolic): - value = lltype.cast_primitive(lltype.Signed, value) - if allow_short and -128 <= value <= 127: - # emit the constant as a small integer - self.code.append(chr(value & 0xFF)) - return True - constants = self.constants_i - elif kind == 'ref': - value = lltype.cast_opaque_ptr(llmemory.GCREF, value) - constants = self.constants_r - elif kind == 'float': - assert TYPE == lltype.Float - constants = self.constants_f - else: - raise NotImplementedError(const) + value = const.value + TYPE = lltype.typeOf(value) + if kind == 'int': + if isinstance(TYPE, lltype.Ptr): + assert TYPE.TO._gckind == 'raw' + self.see_raw_object(value) + value = llmemory.cast_ptr_to_adr(value) + TYPE = llmemory.Address + if TYPE == llmemory.Address: + value = heaptracker.adr2int(value) + elif not isinstance(value, ComputedIntSymbolic): + value = lltype.cast_primitive(lltype.Signed, value) + if allow_short and -128 <= value <= 127: + # emit the constant as a small integer + self.code.append(chr(value & 0xFF)) + return True + constants = self.constants_i + elif kind == 'ref': + value = lltype.cast_opaque_ptr(llmemory.GCREF, value) + constants = self.constants_r + elif kind == 'float': + assert TYPE == lltype.Float + constants = self.constants_f + else: + raise NotImplementedError(const) + key = (kind, Constant(value)) + if key not in self.constants_dict: constants.append(value) - self.constants_dict[const] = 256 - len(constants) + self.constants_dict[key] = 256 - len(constants) # emit the constant normally, as one byte that is an index in the # list of constants - self.code.append(chr(self.constants_dict[const])) + self.code.append(chr(self.constants_dict[key])) return False def write_insn(self, insn): Modified: pypy/branch/fast-forward/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/call.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/call.py Tue Jul 6 00:00:33 2010 @@ -16,21 +16,22 @@ class CallControl(object): virtualref_info = None # optionally set from outside - virtualizable_info = None # optionally set from outside - portal_runner_ptr = None # optionally set from outside - def __init__(self, cpu=None, portal_graph=None): + def __init__(self, cpu=None, jitdrivers_sd=[]): + assert isinstance(jitdrivers_sd, list) # debugging self.cpu = cpu - self.portal_graph = portal_graph + self.jitdrivers_sd = jitdrivers_sd self.jitcodes = {} # map {graph: jitcode} self.unfinished_graphs = [] # list of graphs with pending jitcodes - self.jitdriver = None if hasattr(cpu, 'rtyper'): # for tests self.rtyper = cpu.rtyper translator = self.rtyper.annotator.translator self.raise_analyzer = RaiseAnalyzer(translator) self.readwrite_analyzer = ReadWriteAnalyzer(translator) self.virtualizable_analyzer = VirtualizableAnalyzer(translator) + # + for index, jd in enumerate(jitdrivers_sd): + jd.index = index def find_all_graphs(self, policy): try: @@ -41,8 +42,8 @@ def is_candidate(graph): return policy.look_inside_graph(graph) - assert self.portal_graph is not None - todo = [self.portal_graph] + assert len(self.jitdrivers_sd) > 0 + todo = [jd.portal_graph for jd in self.jitdrivers_sd] if hasattr(self, 'rtyper'): for oopspec_name, ll_args, ll_res in support.inline_calls_to: c_func, _ = support.builtin_func_for_spec(self.rtyper, @@ -122,7 +123,7 @@ def guess_call_kind(self, op, is_candidate=None): if op.opname == 'direct_call': funcptr = op.args[0].value - if funcptr is self.portal_runner_ptr: + if self.jitdriver_sd_from_portal_runner_ptr(funcptr) is not None: return 'recursive' funcobj = get_funcobj(funcptr) if getattr(funcobj, 'graph', None) is None: @@ -143,6 +144,11 @@ # used only after find_all_graphs() return graph in self.candidate_graphs + def grab_initial_jitcodes(self): + for jd in self.jitdrivers_sd: + jd.mainjitcode = self.get_jitcode(jd.portal_graph) + jd.mainjitcode.is_portal = True + def enum_pending_graphs(self): while self.unfinished_graphs: graph = self.unfinished_graphs.pop() @@ -241,12 +247,32 @@ return (effectinfo is None or effectinfo.extraeffect >= EffectInfo.EF_CAN_RAISE) - def found_jitdriver(self, jitdriver): - if self.jitdriver is None: - self.jitdriver = jitdriver - else: - assert self.jitdriver is jitdriver + def jitdriver_sd_from_portal_graph(self, graph): + for jd in self.jitdrivers_sd: + if jd.portal_graph is graph: + return jd + return None - def getjitdriver(self): - assert self.jitdriver is not None, "order dependency issue?" - return self.jitdriver + def jitdriver_sd_from_portal_runner_ptr(self, funcptr): + for jd in self.jitdrivers_sd: + if funcptr is jd.portal_runner_ptr: + return jd + return None + + def jitdriver_sd_from_jitdriver(self, jitdriver): + for jd in self.jitdrivers_sd: + if jd.jitdriver is jitdriver: + return jd + return None + + def get_vinfo(self, VTYPEPTR): + seen = set() + for jd in self.jitdrivers_sd: + if jd.virtualizable_info is not None: + if jd.virtualizable_info.is_vtypeptr(VTYPEPTR): + seen.add(jd.virtualizable_info) + if seen: + assert len(seen) == 1 + return seen.pop() + else: + return None Modified: pypy/branch/fast-forward/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/codewriter.py Tue Jul 6 00:00:33 2010 @@ -14,29 +14,30 @@ class CodeWriter(object): callcontrol = None # for tests - def __init__(self, cpu=None, maingraph=None): + def __init__(self, cpu=None, jitdrivers_sd=[]): self.cpu = cpu self.assembler = Assembler() - self.portal_graph = maingraph - self.callcontrol = CallControl(cpu, maingraph) + self.callcontrol = CallControl(cpu, jitdrivers_sd) + self._seen_files = set() def transform_func_to_jitcode(self, func, values, type_system='lltype'): """For testing.""" rtyper = support.annotate(func, values, type_system=type_system) graph = rtyper.annotator.translator.graphs[0] jitcode = JitCode("test") - self.transform_graph_to_jitcode(graph, jitcode, True, True) + self.transform_graph_to_jitcode(graph, jitcode, True) return jitcode - def transform_graph_to_jitcode(self, graph, jitcode, portal, verbose): + def transform_graph_to_jitcode(self, graph, jitcode, verbose): """Transform a graph into a JitCode containing the same bytecode in a different format. """ + portal_jd = self.callcontrol.jitdriver_sd_from_portal_graph(graph) graph = copygraph(graph, shallowvars=True) # # step 1: mangle the graph so that it contains the final instructions # that we want in the JitCode, but still as a control flow graph - transform_graph(graph, self.cpu, self.callcontrol, portal) + transform_graph(graph, self.cpu, self.callcontrol, portal_jd) # # step 2: perform register allocation on it regallocs = {} @@ -59,16 +60,14 @@ self.assembler.assemble(ssarepr, jitcode) # # print the resulting assembler - self.print_ssa_repr(ssarepr, portal, verbose) + self.print_ssa_repr(ssarepr, portal_jd, verbose) def make_jitcodes(self, verbose=False): log.info("making JitCodes...") - maingraph = self.portal_graph - self.mainjitcode = self.callcontrol.get_jitcode(maingraph) + self.callcontrol.grab_initial_jitcodes() count = 0 for graph, jitcode in self.callcontrol.enum_pending_graphs(): - self.transform_graph_to_jitcode(graph, jitcode, - graph is maingraph, verbose) + self.transform_graph_to_jitcode(graph, jitcode, verbose) count += 1 if not count % 500: log.info("Produced %d jitcodes" % count) @@ -76,33 +75,35 @@ log.info("there are %d JitCode instances." % count) def setup_vrefinfo(self, vrefinfo): + # must be called at most once + assert self.callcontrol.virtualref_info is None self.callcontrol.virtualref_info = vrefinfo - def setup_virtualizable_info(self, vinfo): - self.callcontrol.virtualizable_info = vinfo - - def setup_portal_runner_ptr(self, portal_runner_ptr): - self.callcontrol.portal_runner_ptr = portal_runner_ptr + def setup_jitdriver(self, jitdriver_sd): + # Must be called once per jitdriver. Usually jitdriver_sd is an + # instance of pypy.jit.metainterp.jitdriver.JitDriverStaticData. + self.callcontrol.jitdrivers_sd.append(jitdriver_sd) def find_all_graphs(self, policy): return self.callcontrol.find_all_graphs(policy) - def print_ssa_repr(self, ssarepr, portal, verbose): + def print_ssa_repr(self, ssarepr, portal_jitdriver, verbose): if verbose: print '%s:' % (ssarepr.name,) print format_assembler(ssarepr) else: dir = udir.ensure("jitcodes", dir=1) - if portal: - name = "00_portal_runner" + if portal_jitdriver: + name = "%02d_portal_runner" % (portal_jitdriver.index,) elif ssarepr.name and ssarepr.name != '?': name = ssarepr.name else: name = 'unnamed' % id(ssarepr) i = 1 extra = '' - while dir.join(name+extra).check(exists=1): + while name+extra in self._seen_files: i += 1 extra = '.%d' % i + self._seen_files.add(name+extra) dir.join(name+extra).write(format_assembler(ssarepr)) log.dot() Modified: pypy/branch/fast-forward/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/jitcode.py Tue Jul 6 00:00:33 2010 @@ -13,6 +13,7 @@ self.name = name self.fnaddr = fnaddr self.calldescr = calldescr + self.is_portal = False self._called_from = called_from # debugging self._ssarepr = None # debugging @@ -24,11 +25,11 @@ self.constants_i = constants_i or self._empty_i self.constants_r = constants_r or self._empty_r self.constants_f = constants_f or self._empty_f - # encode the three num_regs into a single integer + # encode the three num_regs into a single char each assert num_regs_i < 256 and num_regs_r < 256 and num_regs_f < 256 - self.num_regs_encoded = ((num_regs_i << 16) | - (num_regs_f << 8) | - (num_regs_r << 0)) + self.c_num_regs_i = chr(num_regs_i) + self.c_num_regs_r = chr(num_regs_r) + self.c_num_regs_f = chr(num_regs_f) self.liveness = make_liveness_cache(liveness) self._startpoints = startpoints # debugging self._alllabels = alllabels # debugging @@ -37,13 +38,13 @@ return heaptracker.adr2int(self.fnaddr) def num_regs_i(self): - return self.num_regs_encoded >> 16 - - def num_regs_f(self): - return (self.num_regs_encoded >> 8) & 0xFF + return ord(self.c_num_regs_i) def num_regs_r(self): - return self.num_regs_encoded & 0xFF + return ord(self.c_num_regs_r) + + def num_regs_f(self): + return ord(self.c_num_regs_f) def has_liveness_info(self, pc): return pc in self.liveness Modified: pypy/branch/fast-forward/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/jtransform.py Tue Jul 6 00:00:33 2010 @@ -13,23 +13,23 @@ from pypy.translator.simplify import get_funcobj -def transform_graph(graph, cpu=None, callcontrol=None, portal=True): +def transform_graph(graph, cpu=None, callcontrol=None, portal_jd=None): """Transform a control flow graph to make it suitable for being flattened in a JitCode. """ - t = Transformer(cpu, callcontrol) - t.transform(graph, portal) + t = Transformer(cpu, callcontrol, portal_jd) + t.transform(graph) class Transformer(object): - def __init__(self, cpu=None, callcontrol=None): + def __init__(self, cpu=None, callcontrol=None, portal_jd=None): self.cpu = cpu self.callcontrol = callcontrol + self.portal_jd = portal_jd # non-None only for the portal graph(s) - def transform(self, graph, portal): + def transform(self, graph): self.graph = graph - self.portal = portal for block in list(graph.iterblocks()): self.optimize_block(block) @@ -325,10 +325,13 @@ return op1 def handle_recursive_call(self, op): - ops = self.promote_greens(op.args[1:]) - targetgraph = self.callcontrol.portal_graph - num_green_args = len(self.callcontrol.getjitdriver().greens) - args = (self.make_three_lists(op.args[1:1+num_green_args]) + + jitdriver_sd = self.callcontrol.jitdriver_sd_from_portal_runner_ptr( + op.args[0].value) + assert jitdriver_sd is not None + ops = self.promote_greens(op.args[1:], jitdriver_sd.jitdriver) + num_green_args = len(jitdriver_sd.jitdriver.greens) + args = ([Constant(jitdriver_sd.index, lltype.Signed)] + + self.make_three_lists(op.args[1:1+num_green_args]) + self.make_three_lists(op.args[1+num_green_args:])) kind = getkind(op.result.concretetype)[0] op0 = SpaceOperation('recursive_call_%s' % kind, args, op.result) @@ -400,7 +403,12 @@ log.WARNING('ignoring hint %r at %r' % (hints, self.graph)) def rewrite_op_malloc_varsize(self, op): - assert op.args[1].value == {'flavor': 'gc'} + if op.args[1].value['flavor'] == 'raw': + ARRAY = op.args[0].value + return self._do_builtin_call(op, 'raw_malloc', + [op.args[2]], + extra = (ARRAY,), + extrakey = ARRAY) if op.args[0].value == rstr.STR: return SpaceOperation('newstr', [op.args[2]], op.result) elif op.args[0].value == rstr.UNICODE: @@ -412,9 +420,14 @@ return SpaceOperation('new_array', [arraydescr, op.args[2]], op.result) + def rewrite_op_free(self, op): + assert op.args[1].value == 'raw' + ARRAY = op.args[0].concretetype.TO + return self._do_builtin_call(op, 'raw_free', [op.args[0]], + extra = (ARRAY,), extrakey = ARRAY) + def rewrite_op_getarrayitem(self, op): ARRAY = op.args[0].concretetype.TO - assert ARRAY._gckind == 'gc' if self._array_of_voids(ARRAY): return [] if op.args[0] in self.vable_array_vars: # for virtualizables @@ -428,13 +441,12 @@ # normal case follows arraydescr = self.cpu.arraydescrof(ARRAY) kind = getkind(op.result.concretetype) - return SpaceOperation('getarrayitem_gc_%s' % kind[0], + return SpaceOperation('getarrayitem_%s_%s' % (ARRAY._gckind, kind[0]), [op.args[0], arraydescr, op.args[1]], op.result) def rewrite_op_setarrayitem(self, op): ARRAY = op.args[0].concretetype.TO - assert ARRAY._gckind == 'gc' if self._array_of_voids(ARRAY): return [] if op.args[0] in self.vable_array_vars: # for virtualizables @@ -447,7 +459,7 @@ op.args[1], op.args[2]], None)] arraydescr = self.cpu.arraydescrof(ARRAY) kind = getkind(op.args[2].concretetype) - return SpaceOperation('setarrayitem_gc_%s' % kind[0], + return SpaceOperation('setarrayitem_%s_%s' % (ARRAY._gckind, kind[0]), [op.args[0], arraydescr, op.args[1], op.args[2]], None) @@ -483,14 +495,14 @@ # check for virtualizable try: if self.is_virtualizable_getset(op): - descr = self.get_virtualizable_field_descr(op.args[1].value) + descr = self.get_virtualizable_field_descr(op) kind = getkind(RESULT)[0] return [SpaceOperation('-live-', [], None), SpaceOperation('getfield_vable_%s' % kind, [v_inst, descr], op.result)] - except VirtualizableArrayField: + except VirtualizableArrayField, e: # xxx hack hack hack - vinfo = self.callcontrol.virtualizable_info + vinfo = e.args[1] arrayindex = vinfo.array_field_counter[op.args[1].value] arrayfielddescr = vinfo.array_field_descrs[arrayindex] arraydescr = vinfo.array_descrs[arrayindex] @@ -527,7 +539,7 @@ return # check for virtualizable if self.is_virtualizable_getset(op): - descr = self.get_virtualizable_field_descr(op.args[1].value) + descr = self.get_virtualizable_field_descr(op) kind = getkind(RESULT)[0] return [SpaceOperation('-live-', [], None), SpaceOperation('setfield_vable_%s' % kind, @@ -544,21 +556,23 @@ return (op.args[1].value == 'typeptr' and op.args[0].concretetype.TO._hints.get('typeptr')) + def get_vinfo(self, v_virtualizable): + if self.callcontrol is None: # for tests + return None + return self.callcontrol.get_vinfo(v_virtualizable.concretetype) + def is_virtualizable_getset(self, op): # every access of an object of exactly the type VTYPEPTR is # likely to be a virtualizable access, but we still have to # check it in pyjitpl.py. - try: - vinfo = self.callcontrol.virtualizable_info - except AttributeError: - return False - if vinfo is None or not vinfo.is_vtypeptr(op.args[0].concretetype): + vinfo = self.get_vinfo(op.args[0]) + if vinfo is None: return False res = False if op.args[1].value in vinfo.static_field_to_extra_box: res = True if op.args[1].value in vinfo.array_fields: - res = VirtualizableArrayField(self.graph) + res = VirtualizableArrayField(self.graph, vinfo) if res: flags = self.vable_flags[op.args[0]] @@ -568,8 +582,9 @@ raise res return res - def get_virtualizable_field_descr(self, fieldname): - vinfo = self.callcontrol.virtualizable_info + def get_virtualizable_field_descr(self, op): + fieldname = op.args[1].value + vinfo = self.get_vinfo(op.args[0]) index = vinfo.static_field_to_extra_box[fieldname] return vinfo.static_field_descrs[index] @@ -750,9 +765,10 @@ return Constant(value, lltype.Bool) return op - def promote_greens(self, args): + def promote_greens(self, args, jitdriver): ops = [] - num_green_args = len(self.callcontrol.getjitdriver().greens) + num_green_args = len(jitdriver.greens) + assert len(args) == num_green_args + len(jitdriver.reds) for v in args[:num_green_args]: if isinstance(v, Variable) and v.concretetype is not lltype.Void: kind = getkind(v.concretetype) @@ -762,21 +778,28 @@ return ops def rewrite_op_jit_marker(self, op): - self.callcontrol.found_jitdriver(op.args[1].value) key = op.args[0].value - return getattr(self, 'handle_jit_marker__%s' % key)(op) + jitdriver = op.args[1].value + return getattr(self, 'handle_jit_marker__%s' % key)(op, jitdriver) - def handle_jit_marker__jit_merge_point(self, op): - assert self.portal, "jit_merge_point in non-main graph!" - ops = self.promote_greens(op.args[2:]) - num_green_args = len(self.callcontrol.getjitdriver().greens) - args = (self.make_three_lists(op.args[2:2+num_green_args]) + + def handle_jit_marker__jit_merge_point(self, op, jitdriver): + assert self.portal_jd is not None, ( + "'jit_merge_point' in non-portal graph!") + assert jitdriver is self.portal_jd.jitdriver, ( + "general mix-up of jitdrivers?") + ops = self.promote_greens(op.args[2:], jitdriver) + num_green_args = len(jitdriver.greens) + args = ([Constant(self.portal_jd.index, lltype.Signed)] + + self.make_three_lists(op.args[2:2+num_green_args]) + self.make_three_lists(op.args[2+num_green_args:])) op1 = SpaceOperation('jit_merge_point', args, None) return ops + [op1] - def handle_jit_marker__can_enter_jit(self, op): - return SpaceOperation('can_enter_jit', [], None) + def handle_jit_marker__can_enter_jit(self, op, jitdriver): + jd = self.callcontrol.jitdriver_sd_from_jitdriver(jitdriver) + assert jd is not None + c_index = Constant(jd.index, lltype.Signed) + return SpaceOperation('can_enter_jit', [c_index], None) def rewrite_op_debug_assert(self, op): log.WARNING("found debug_assert in %r; should have be removed" % @@ -974,9 +997,8 @@ def rewrite_op_jit_force_virtualizable(self, op): # this one is for virtualizables - vinfo = self.callcontrol.virtualizable_info + vinfo = self.get_vinfo(op.args[0]) assert vinfo is not None - assert vinfo.is_vtypeptr(op.args[0].concretetype) self.vable_flags[op.args[0]] = op.args[2].value return [] Modified: pypy/branch/fast-forward/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/support.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/support.py Tue Jul 6 00:00:33 2010 @@ -282,6 +282,9 @@ # ---------- malloc with del ---------- + def _ll_2_raw_malloc(TP, size): + return lltype.malloc(TP, size, flavor='raw') + def build_ll_0_alloc_with_del(RESULT, vtable): def _ll_0_alloc_with_del(): p = lltype.malloc(RESULT.TO) @@ -289,6 +292,15 @@ return p return _ll_0_alloc_with_del + def build_ll_1_raw_malloc(ARRAY): + def _ll_1_raw_malloc(n): + return lltype.malloc(ARRAY, n, flavor='raw') + return _ll_1_raw_malloc + + def build_ll_1_raw_free(ARRAY): + def _ll_1_raw_free(p): + lltype.free(p, flavor='raw') + return _ll_1_raw_free class OOtypeHelpers: Modified: pypy/branch/fast-forward/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/test/test_assembler.py Tue Jul 6 00:00:33 2010 @@ -127,6 +127,25 @@ assert assembler.insns == {'foobar/IR': 0} assert jitcode.constants_i == [42] +def test_assemble_list_semibug(): + # the semibug is that after forcing 42 into the dict of constants, + # it would be reused for all future 42's, even ones that can be + # encoded directly. + ssarepr = SSARepr("test") + ssarepr.insns = [ + ('foobar', ListOfKind('int', [Constant(42, lltype.Signed)])), + ('foobar', ListOfKind('int', [Constant(42, lltype.Signed)])), + ('baz', Constant(42, lltype.Signed)), + ] + assembler = Assembler() + jitcode = assembler.assemble(ssarepr) + assert jitcode.code == ("\x00\x01\xFF" + "\x00\x01\xFF" + "\x01\x2A") + assert assembler.insns == {'foobar/I': 0, + 'baz/c': 1} + assert jitcode.constants_i == [42] + def test_assemble_descr(): class FooDescr(AbstractDescr): pass Modified: pypy/branch/fast-forward/pypy/jit/codewriter/test/test_call.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/test/test_call.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/test/test_call.py Tue Jul 6 00:00:33 2010 @@ -52,13 +52,19 @@ # ____________________________________________________________ +class FakeJitDriverSD: + def __init__(self, portal_graph): + self.portal_graph = portal_graph + self.portal_runner_ptr = "???" + def test_find_all_graphs(): def g(x): return x + 2 def f(x): return g(x) + 1 rtyper = support.annotate(f, [7]) - cc = CallControl(portal_graph=rtyper.annotator.translator.graphs[0]) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cc = CallControl(jitdrivers_sd=[jitdriver_sd]) res = cc.find_all_graphs(FakePolicy()) funcs = set([graph.func for graph in res]) assert funcs == set([f, g]) @@ -69,7 +75,8 @@ def f(x): return g(x) + 1 rtyper = support.annotate(f, [7]) - cc = CallControl(portal_graph=rtyper.annotator.translator.graphs[0]) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cc = CallControl(jitdrivers_sd=[jitdriver_sd]) class CustomFakePolicy: def look_inside_graph(self, graph): assert graph.name == 'g' @@ -83,10 +90,11 @@ def test_guess_call_kind_and_calls_from_graphs(): class portal_runner_obj: graph = object() + class FakeJitDriverSD: + portal_runner_ptr = portal_runner_obj g = object() g1 = object() - cc = CallControl() - cc.portal_runner_ptr = portal_runner_obj + cc = CallControl(jitdrivers_sd=[FakeJitDriverSD()]) cc.candidate_graphs = [g, g1] op = SpaceOperation('direct_call', [Constant(portal_runner_obj)], Modified: pypy/branch/fast-forward/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/test/test_codewriter.py Tue Jul 6 00:00:33 2010 @@ -2,7 +2,7 @@ from pypy.jit.codewriter.codewriter import CodeWriter from pypy.jit.codewriter import support from pypy.jit.metainterp.history import AbstractDescr -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rffi class FakeCallDescr(AbstractDescr): def __init__(self, FUNC, ARGS, RESULT, effectinfo=None): @@ -24,17 +24,28 @@ def as_vtable_size_descr(self): return self +class FakeArrayDescr(AbstractDescr): + def __init__(self, ARRAY): + self.ARRAY = ARRAY + class FakeCPU: def __init__(self, rtyper): self.rtyper = rtyper calldescrof = FakeCallDescr fielddescrof = FakeFieldDescr sizeof = FakeSizeDescr + arraydescrof = FakeArrayDescr class FakePolicy: def look_inside_graph(self, graph): return graph.name != 'dont_look' +class FakeJitDriverSD: + def __init__(self, portal_graph): + self.portal_graph = portal_graph + self.portal_runner_ptr = "???" + self.virtualizable_info = None + def test_loop(): def f(a, b): @@ -70,11 +81,11 @@ def fff(a, b): return ggg(b) - ggg(a) rtyper = support.annotate(fff, [35, 42]) - maingraph = rtyper.annotator.translator.graphs[0] - cw = CodeWriter(FakeCPU(rtyper), maingraph) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cw = CodeWriter(FakeCPU(rtyper), [jitdriver_sd]) cw.find_all_graphs(FakePolicy()) cw.make_jitcodes(verbose=True) - jitcode = cw.mainjitcode + jitcode = jitdriver_sd.mainjitcode print jitcode.dump() [jitcode2] = cw.assembler.descrs print jitcode2.dump() @@ -100,7 +111,7 @@ blackholeinterp.setarg_i(0, 6) blackholeinterp.setarg_i(1, 100) blackholeinterp.run() - assert blackholeinterp.final_result_i() == 100+6+5+4+3 + assert blackholeinterp.get_tmpreg_i() == 100+6+5+4+3 def test_instantiate(): class A1: id = 651 @@ -117,7 +128,7 @@ return x().id + y().id + dont_look(n) rtyper = support.annotate(f, [35]) maingraph = rtyper.annotator.translator.graphs[0] - cw = CodeWriter(FakeCPU(rtyper), maingraph) + cw = CodeWriter(FakeCPU(rtyper), [FakeJitDriverSD(maingraph)]) cw.find_all_graphs(FakePolicy()) cw.make_jitcodes(verbose=True) # @@ -144,10 +155,32 @@ def f(n): return abs(n) rtyper = support.annotate(f, [35]) - maingraph = rtyper.annotator.translator.graphs[0] - cw = CodeWriter(FakeCPU(rtyper), maingraph) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cw = CodeWriter(FakeCPU(rtyper), [jitdriver_sd]) cw.find_all_graphs(FakePolicy()) cw.make_jitcodes(verbose=True) # - s = cw.mainjitcode.dump() + s = jitdriver_sd.mainjitcode.dump() assert "inline_call_ir_i " in s + +def test_raw_malloc_and_access(): + TP = rffi.CArray(lltype.Signed) + + def f(n): + a = lltype.malloc(TP, n, flavor='raw') + a[0] = n + res = a[0] + lltype.free(a, flavor='raw') + return res + + rtyper = support.annotate(f, [35]) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cw = CodeWriter(FakeCPU(rtyper), [jitdriver_sd]) + cw.find_all_graphs(FakePolicy()) + cw.make_jitcodes(verbose=True) + # + s = jitdriver_sd.mainjitcode.dump() + assert 'residual_call_ir_i $<* fn _ll_1_raw_malloc__Signed>' in s + assert 'setarrayitem_raw_i' in s + assert 'getarrayitem_raw_i' in s + assert 'residual_call_ir_v $<* fn _ll_1_raw_free__arrayPtr>' in s Modified: pypy/branch/fast-forward/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/test/test_flatten.py Tue Jul 6 00:00:33 2010 @@ -68,11 +68,8 @@ return FakeDescr() def calldescr_canraise(self, calldescr): return calldescr is not self._descr_cannot_raise - def found_jitdriver(self, jitdriver): - assert isinstance(jitdriver, JitDriver) - self.jitdriver = jitdriver - def getjitdriver(self): - return self.jitdriver + def get_vinfo(self, VTYPEPTR): + return None class FakeCallControlWithVRefInfo: class virtualref_info: @@ -118,13 +115,13 @@ return self.rtyper.annotator.translator.graphs def encoding_test(self, func, args, expected, - transform=False, liveness=False, cc=None): + transform=False, liveness=False, cc=None, jd=None): graphs = self.make_graphs(func, args) #graphs[0].show() if transform: from pypy.jit.codewriter.jtransform import transform_graph cc = cc or FakeCallControl() - transform_graph(graphs[0], FakeCPU(self.rtyper), cc) + transform_graph(graphs[0], FakeCPU(self.rtyper), cc, jd) ssarepr = flatten_graph(graphs[0], fake_regallocs(), _include_all_exc_links=not transform) if liveness: @@ -584,13 +581,21 @@ def f(x, y): myjitdriver.jit_merge_point(x=x, y=y) myjitdriver.can_enter_jit(x=y, y=x) + class FakeJitDriverSD: + jitdriver = myjitdriver + index = 27 + jd = FakeJitDriverSD() + class MyFakeCallControl(FakeCallControl): + def jitdriver_sd_from_jitdriver(self, jitdriver): + assert jitdriver == myjitdriver + return jd self.encoding_test(f, [4, 5], """ -live- %i0, %i1 int_guard_value %i0 - jit_merge_point I[%i0], R[], F[], I[%i1], R[], F[] - can_enter_jit + jit_merge_point $27, I[%i0], R[], F[], I[%i1], R[], F[] + can_enter_jit $27 void_return - """, transform=True, liveness=True) + """, transform=True, liveness=True, cc=MyFakeCallControl(), jd=jd) def test_keepalive(self): S = lltype.GcStruct('S') Modified: pypy/branch/fast-forward/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/blackhole.py Tue Jul 6 00:00:33 2010 @@ -315,27 +315,12 @@ def get_tmpreg_f(self): return self.tmpreg_f - def final_result_i(self): - assert self._return_type == 'i' - return self.get_tmpreg_i() - - def final_result_r(self): - assert self._return_type == 'r' - return self.get_tmpreg_r() - - def final_result_f(self): - assert self._return_type == 'f' - return self.get_tmpreg_f() - - def final_result_v(self): - assert self._return_type == 'v' - def _final_result_anytype(self): "NOT_RPYTHON" - if self._return_type == 'i': return self.final_result_i() - if self._return_type == 'r': return self.final_result_r() - if self._return_type == 'f': return self.final_result_f() - if self._return_type == 'v': return self.final_result_v() + if self._return_type == 'i': return self.get_tmpreg_i() + if self._return_type == 'r': return self.get_tmpreg_r() + if self._return_type == 'f': return self.get_tmpreg_f() + if self._return_type == 'v': return None raise ValueError(self._return_type) def cleanup_registers(self): @@ -774,12 +759,12 @@ # ---------- # the main hints and recursive calls - @arguments() - def bhimpl_can_enter_jit(): + @arguments("i") + def bhimpl_can_enter_jit(jdindex): pass - @arguments("self", "I", "R", "F", "I", "R", "F") - def bhimpl_jit_merge_point(self, *args): + @arguments("self", "i", "I", "R", "F", "I", "R", "F") + def bhimpl_jit_merge_point(self, jdindex, *args): if self.nextblackholeinterp is None: # we are the last level CRN = self.builder.metainterp_sd.ContinueRunningNormally raise CRN(*args) @@ -793,55 +778,55 @@ # call the interpreter main loop from here, and just return its # result. sd = self.builder.metainterp_sd - if sd.result_type == 'void': - self.bhimpl_recursive_call_v(*args) + result_type = sd.jitdrivers_sd[jdindex].result_type + if result_type == 'v': + self.bhimpl_recursive_call_v(jdindex, *args) self.bhimpl_void_return() - elif sd.result_type == 'int': - x = self.bhimpl_recursive_call_i(*args) + elif result_type == 'i': + x = self.bhimpl_recursive_call_i(jdindex, *args) self.bhimpl_int_return(x) - elif sd.result_type == 'ref': - x = self.bhimpl_recursive_call_r(*args) + elif result_type == 'r': + x = self.bhimpl_recursive_call_r(jdindex, *args) self.bhimpl_ref_return(x) - elif sd.result_type == 'float': - x = self.bhimpl_recursive_call_f(*args) + elif result_type == 'f': + x = self.bhimpl_recursive_call_f(jdindex, *args) self.bhimpl_float_return(x) assert False - def get_portal_runner(self): - metainterp_sd = self.builder.metainterp_sd - fnptr = llmemory.cast_ptr_to_adr(metainterp_sd._portal_runner_ptr) - fnptr = heaptracker.adr2int(fnptr) - calldescr = metainterp_sd.portal_code.calldescr + def get_portal_runner(self, jdindex): + jitdriver_sd = self.builder.metainterp_sd.jitdrivers_sd[jdindex] + fnptr = heaptracker.adr2int(jitdriver_sd.portal_runner_adr) + calldescr = jitdriver_sd.mainjitcode.calldescr return fnptr, calldescr - @arguments("self", "I", "R", "F", "I", "R", "F", returns="i") - def bhimpl_recursive_call_i(self, greens_i, greens_r, greens_f, - reds_i, reds_r, reds_f): - fnptr, calldescr = self.get_portal_runner() + @arguments("self", "i", "I", "R", "F", "I", "R", "F", returns="i") + def bhimpl_recursive_call_i(self, jdindex, greens_i, greens_r, greens_f, + reds_i, reds_r, reds_f): + fnptr, calldescr = self.get_portal_runner(jdindex) return self.cpu.bh_call_i(fnptr, calldescr, greens_i + reds_i, greens_r + reds_r, greens_f + reds_f) - @arguments("self", "I", "R", "F", "I", "R", "F", returns="r") - def bhimpl_recursive_call_r(self, greens_i, greens_r, greens_f, - reds_i, reds_r, reds_f): - fnptr, calldescr = self.get_portal_runner() + @arguments("self", "i", "I", "R", "F", "I", "R", "F", returns="r") + def bhimpl_recursive_call_r(self, jdindex, greens_i, greens_r, greens_f, + reds_i, reds_r, reds_f): + fnptr, calldescr = self.get_portal_runner(jdindex) return self.cpu.bh_call_r(fnptr, calldescr, greens_i + reds_i, greens_r + reds_r, greens_f + reds_f) - @arguments("self", "I", "R", "F", "I", "R", "F", returns="f") - def bhimpl_recursive_call_f(self, greens_i, greens_r, greens_f, - reds_i, reds_r, reds_f): - fnptr, calldescr = self.get_portal_runner() + @arguments("self", "i", "I", "R", "F", "I", "R", "F", returns="f") + def bhimpl_recursive_call_f(self, jdindex, greens_i, greens_r, greens_f, + reds_i, reds_r, reds_f): + fnptr, calldescr = self.get_portal_runner(jdindex) return self.cpu.bh_call_f(fnptr, calldescr, greens_i + reds_i, greens_r + reds_r, greens_f + reds_f) - @arguments("self", "I", "R", "F", "I", "R", "F") - def bhimpl_recursive_call_v(self, greens_i, greens_r, greens_f, - reds_i, reds_r, reds_f): - fnptr, calldescr = self.get_portal_runner() + @arguments("self", "i", "I", "R", "F", "I", "R", "F") + def bhimpl_recursive_call_v(self, jdindex, greens_i, greens_r, greens_f, + reds_i, reds_r, reds_f): + fnptr, calldescr = self.get_portal_runner(jdindex) return self.cpu.bh_call_v(fnptr, calldescr, greens_i + reds_i, greens_r + reds_r, @@ -1005,6 +990,13 @@ bhimpl_getarrayitem_gc_pure_r = bhimpl_getarrayitem_gc_r bhimpl_getarrayitem_gc_pure_f = bhimpl_getarrayitem_gc_f + @arguments("cpu", "i", "d", "i", returns="i") + def bhimpl_getarrayitem_raw_i(cpu, array, arraydescr, index): + return cpu.bh_getarrayitem_raw_i(arraydescr, array, index) + @arguments("cpu", "i", "d", "i", returns="f") + def bhimpl_getarrayitem_raw_f(cpu, array, arraydescr, index): + return cpu.bh_getarrayitem_raw_f(arraydescr, array, index) + @arguments("cpu", "r", "d", "i", "i") def bhimpl_setarrayitem_gc_i(cpu, array, arraydescr, index, newvalue): cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue) @@ -1015,6 +1007,15 @@ def bhimpl_setarrayitem_gc_f(cpu, array, arraydescr, index, newvalue): cpu.bh_setarrayitem_gc_f(arraydescr, array, index, newvalue) + @arguments("cpu", "i", "d", "i", "i") + def bhimpl_setarrayitem_raw_i(cpu, array, arraydescr, index, newvalue): + cpu.bh_setarrayitem_raw_i(arraydescr, array, index, newvalue) + @arguments("cpu", "i", "d", "i", "f") + def bhimpl_setarrayitem_raw_f(cpu, array, arraydescr, index, newvalue): + cpu.bh_setarrayitem_raw_f(arraydescr, array, index, newvalue) + + # note, there is no 'r' here, since it can't happen + @arguments("cpu", "r", "d", returns="i") def bhimpl_arraylen_gc(cpu, array, arraydescr): return cpu.bh_arraylen_gc(arraydescr, array) @@ -1178,11 +1179,11 @@ self._done_with_this_frame() kind = self._return_type if kind == 'i': - caller._setup_return_value_i(self.final_result_i()) + caller._setup_return_value_i(self.get_tmpreg_i()) elif kind == 'r': - caller._setup_return_value_r(self.final_result_r()) + caller._setup_return_value_r(self.get_tmpreg_r()) elif kind == 'f': - caller._setup_return_value_f(self.final_result_f()) + caller._setup_return_value_f(self.get_tmpreg_f()) else: assert kind == 'v' return lltype.nullptr(rclass.OBJECTPTR.TO) @@ -1248,15 +1249,15 @@ # rare case: we only get there if the blackhole interps all returned # normally (in general we get a ContinueRunningNormally exception). sd = self.builder.metainterp_sd - if sd.result_type == 'void': - self.final_result_v() + kind = self._return_type + if kind == 'v': raise sd.DoneWithThisFrameVoid() - elif sd.result_type == 'int': - raise sd.DoneWithThisFrameInt(self.final_result_i()) - elif sd.result_type == 'ref': - raise sd.DoneWithThisFrameRef(self.cpu, self.final_result_r()) - elif sd.result_type == 'float': - raise sd.DoneWithThisFrameFloat(self.final_result_f()) + elif kind == 'i': + raise sd.DoneWithThisFrameInt(self.get_tmpreg_i()) + elif kind == 'r': + raise sd.DoneWithThisFrameRef(self.cpu, self.get_tmpreg_r()) + elif kind == 'f': + raise sd.DoneWithThisFrameFloat(self.get_tmpreg_f()) else: assert False @@ -1290,12 +1291,14 @@ blackholeinterp.builder.release_interp(blackholeinterp) blackholeinterp = blackholeinterp.nextblackholeinterp -def resume_in_blackhole(metainterp_sd, resumedescr, all_virtuals=None): +def resume_in_blackhole(metainterp_sd, jitdriver_sd, resumedescr, + all_virtuals=None): from pypy.jit.metainterp.resume import blackhole_from_resumedata debug_start('jit-blackhole') metainterp_sd.profiler.start_blackhole() blackholeinterp = blackhole_from_resumedata( metainterp_sd.blackholeinterpbuilder, + jitdriver_sd, resumedescr, all_virtuals) current_exc = blackholeinterp._prepare_resume_from_failure( Modified: pypy/branch/fast-forward/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/compile.py Tue Jul 6 00:00:33 2010 @@ -39,9 +39,10 @@ name = metainterp.staticdata.stats.name_for_new_loop() return TreeLoop(name) -def make_loop_token(nb_args): +def make_loop_token(nb_args, jitdriver_sd): loop_token = LoopToken() loop_token.specnodes = [prebuiltNotSpecNode] * nb_args + loop_token.outermost_jitdriver_sd = jitdriver_sd return loop_token # ____________________________________________________________ @@ -63,11 +64,12 @@ # make a copy, because optimize_loop can mutate the ops and descrs loop.operations = [op.clone() for op in ops] metainterp_sd = metainterp.staticdata - loop_token = make_loop_token(len(loop.inputargs)) + jitdriver_sd = metainterp.jitdriver_sd + loop_token = make_loop_token(len(loop.inputargs), jitdriver_sd) loop.token = loop_token loop.operations[-1].descr = loop_token # patch the target of the JUMP try: - old_loop_token = metainterp_sd.state.optimize_loop( + old_loop_token = jitdriver_sd.warmstate.optimize_loop( metainterp_sd, old_loop_tokens, loop) except InvalidLoop: return None @@ -141,32 +143,32 @@ pass class DoneWithThisFrameDescrVoid(_DoneWithThisFrameDescr): - def handle_fail(self, metainterp_sd): - assert metainterp_sd.result_type == 'void' + def handle_fail(self, metainterp_sd, jitdriver_sd): + assert jitdriver_sd.result_type == history.VOID raise metainterp_sd.DoneWithThisFrameVoid() class DoneWithThisFrameDescrInt(_DoneWithThisFrameDescr): - def handle_fail(self, metainterp_sd): - assert metainterp_sd.result_type == 'int' + def handle_fail(self, metainterp_sd, jitdriver_sd): + assert jitdriver_sd.result_type == history.INT result = metainterp_sd.cpu.get_latest_value_int(0) raise metainterp_sd.DoneWithThisFrameInt(result) class DoneWithThisFrameDescrRef(_DoneWithThisFrameDescr): - def handle_fail(self, metainterp_sd): - assert metainterp_sd.result_type == 'ref' + def handle_fail(self, metainterp_sd, jitdriver_sd): + assert jitdriver_sd.result_type == history.REF cpu = metainterp_sd.cpu result = cpu.get_latest_value_ref(0) cpu.clear_latest_values(1) raise metainterp_sd.DoneWithThisFrameRef(cpu, result) class DoneWithThisFrameDescrFloat(_DoneWithThisFrameDescr): - def handle_fail(self, metainterp_sd): - assert metainterp_sd.result_type == 'float' + def handle_fail(self, metainterp_sd, jitdriver_sd): + assert jitdriver_sd.result_type == history.FLOAT result = metainterp_sd.cpu.get_latest_value_float(0) raise metainterp_sd.DoneWithThisFrameFloat(result) class ExitFrameWithExceptionDescrRef(_DoneWithThisFrameDescr): - def handle_fail(self, metainterp_sd): + def handle_fail(self, metainterp_sd, jitdriver_sd): cpu = metainterp_sd.cpu value = cpu.get_latest_value_ref(0) cpu.clear_latest_values(1) @@ -210,8 +212,6 @@ class ResumeDescr(AbstractFailDescr): def __init__(self, original_greenkey): self.original_greenkey = original_greenkey - def _clone_if_mutable(self): - raise NotImplementedError class ResumeGuardDescr(ResumeDescr): _counter = 0 # if < 0, there is one counter per value; @@ -258,22 +258,27 @@ # a negative value self._counter = cnt | i - def handle_fail(self, metainterp_sd): - if self.must_compile(metainterp_sd): - return self._trace_and_compile_from_bridge(metainterp_sd) + def handle_fail(self, metainterp_sd, jitdriver_sd): + if self.must_compile(metainterp_sd, jitdriver_sd): + return self._trace_and_compile_from_bridge(metainterp_sd, + jitdriver_sd) else: from pypy.jit.metainterp.blackhole import resume_in_blackhole - resume_in_blackhole(metainterp_sd, self) + resume_in_blackhole(metainterp_sd, jitdriver_sd, self) assert 0, "unreachable" - def _trace_and_compile_from_bridge(self, metainterp_sd): + def _trace_and_compile_from_bridge(self, metainterp_sd, jitdriver_sd): + # 'jitdriver_sd' corresponds to the outermost one, i.e. the one + # of the jit_merge_point where we started the loop, even if the + # loop itself may contain temporarily recursion into other + # jitdrivers. from pypy.jit.metainterp.pyjitpl import MetaInterp - metainterp = MetaInterp(metainterp_sd) + metainterp = MetaInterp(metainterp_sd, jitdriver_sd) return metainterp.handle_guard_failure(self) _trace_and_compile_from_bridge._dont_inline_ = True - def must_compile(self, metainterp_sd): - trace_eagerness = metainterp_sd.state.trace_eagerness + def must_compile(self, metainterp_sd, jitdriver_sd): + trace_eagerness = jitdriver_sd.warmstate.trace_eagerness if self._counter >= 0: self._counter += 1 return self._counter >= trace_eagerness @@ -320,8 +325,7 @@ send_bridge_to_backend(metainterp.staticdata, self, inputargs, new_loop.operations) - def _clone_if_mutable(self): - res = self.__class__(self.metainterp_sd, self.original_greenkey) + def copy_all_attrbutes_into(self, res): # XXX a bit ugly to have to list them all here res.rd_snapshot = self.rd_snapshot res.rd_frame_info_list = self.rd_frame_info_list @@ -329,11 +333,19 @@ res.rd_consts = self.rd_consts res.rd_virtuals = self.rd_virtuals res.rd_pendingfields = self.rd_pendingfields + + def _clone_if_mutable(self): + res = ResumeGuardDescr(self.metainterp_sd, self.original_greenkey) + self.copy_all_attrbutes_into(res) return res class ResumeGuardForcedDescr(ResumeGuardDescr): - def handle_fail(self, metainterp_sd): + def __init__(self, metainterp_sd, original_greenkey, jitdriver_sd): + ResumeGuardDescr.__init__(self, metainterp_sd, original_greenkey) + self.jitdriver_sd = jitdriver_sd + + def handle_fail(self, metainterp_sd, jitdriver_sd): # Failures of a GUARD_NOT_FORCED are never compiled, but # always just blackholed. First fish for the data saved when # the virtualrefs and virtualizable have been forced by @@ -343,7 +355,8 @@ all_virtuals = self.fetch_data(token) if all_virtuals is None: all_virtuals = [] - resume_in_blackhole(metainterp_sd, self, all_virtuals) + assert jitdriver_sd is self.jitdriver_sd + resume_in_blackhole(metainterp_sd, jitdriver_sd, self, all_virtuals) assert 0, "unreachable" @staticmethod @@ -358,7 +371,8 @@ def handle_async_forcing(self, force_token): from pypy.jit.metainterp.resume import force_from_resumedata metainterp_sd = self.metainterp_sd - all_virtuals = force_from_resumedata(metainterp_sd, self) + vinfo = self.jitdriver_sd.virtualizable_info + all_virtuals = force_from_resumedata(metainterp_sd, self, vinfo) # The virtualizable data was stored on the real virtualizable above. # Handle all_virtuals: keep them for later blackholing from the # future failure of the GUARD_NOT_FORCED @@ -392,6 +406,13 @@ assert 0, "not found: %r" % (key,) return data + def _clone_if_mutable(self): + res = ResumeGuardForcedDescr(self.metainterp_sd, + self.original_greenkey, + self.jitdriver_sd) + self.copy_all_attrbutes_into(res) + return res + class AbstractResumeGuardCounters(object): # Completely custom algorithm for now: keep 5 pairs (value, counter), @@ -464,19 +485,19 @@ # a loop at all but ends in a jump to the target loop. It starts # with completely unoptimized arguments, as in the interpreter. metainterp_sd = metainterp.staticdata + jitdriver_sd = metainterp.jitdriver_sd metainterp.history.inputargs = self.redkey - new_loop_token = make_loop_token(len(self.redkey)) + new_loop_token = make_loop_token(len(self.redkey), jitdriver_sd) new_loop.greenkey = self.original_greenkey new_loop.inputargs = self.redkey new_loop.token = new_loop_token send_loop_to_backend(metainterp_sd, new_loop, "entry bridge") # send the new_loop to warmspot.py, to be called directly the next time - metainterp_sd.state.attach_unoptimized_bridge_from_interp( + jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp( self.original_greenkey, new_loop_token) # store the new loop in compiled_merge_points too - glob = metainterp_sd.globaldata - old_loop_tokens = glob.get_compiled_merge_points( + old_loop_tokens = metainterp.get_compiled_merge_points( self.original_greenkey) # it always goes at the end of the list, as it is the most # general loop token @@ -500,10 +521,11 @@ # clone ops, as optimize_bridge can mutate the ops new_loop.operations = [op.clone() for op in metainterp.history.operations] metainterp_sd = metainterp.staticdata + state = metainterp.jitdriver_sd.warmstate try: - target_loop_token = metainterp_sd.state.optimize_bridge(metainterp_sd, - old_loop_tokens, - new_loop) + target_loop_token = state.optimize_bridge(metainterp_sd, + old_loop_tokens, + new_loop) except InvalidLoop: # XXX I am fairly convinced that optimize_bridge cannot actually raise # InvalidLoop Modified: pypy/branch/fast-forward/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/executor.py Tue Jul 6 00:00:33 2010 @@ -90,6 +90,15 @@ else: return BoxInt(cpu.bh_getarrayitem_gc_i(arraydescr, array, index)) +def do_getarrayitem_raw(cpu, _, arraybox, indexbox, arraydescr): + array = arraybox.getint() + index = indexbox.getint() + assert not arraydescr.is_array_of_pointers() + if arraydescr.is_array_of_floats(): + return BoxFloat(cpu.bh_getarrayitem_raw_f(arraydescr, array, index)) + else: + return BoxInt(cpu.bh_getarrayitem_raw_i(arraydescr, array, index)) + def do_setarrayitem_gc(cpu, _, arraybox, indexbox, itembox, arraydescr): array = arraybox.getref_base() index = indexbox.getint() @@ -101,6 +110,15 @@ else: cpu.bh_setarrayitem_gc_i(arraydescr, array, index, itembox.getint()) +def do_setarrayitem_raw(cpu, _, arraybox, indexbox, itembox, arraydescr): + array = arraybox.getint() + index = indexbox.getint() + assert not arraydescr.is_array_of_pointers() + if arraydescr.is_array_of_floats(): + cpu.bh_setarrayitem_raw_f(arraydescr, array, index, itembox.getfloat()) + else: + cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint()) + def do_getfield_gc(cpu, _, structbox, fielddescr): struct = structbox.getref_base() if fielddescr.is_pointer_field(): Modified: pypy/branch/fast-forward/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/history.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/history.py Tue Jul 6 00:00:33 2010 @@ -125,9 +125,6 @@ def repr_of_descr(self): return '%r' % (self,) - def _clone_if_mutable(self): - return self - def get_arg_types(self): """ Implement in call descr. Must return a string of INT, REF and FLOAT ('i', 'r', 'f'). @@ -171,10 +168,18 @@ """ raise NotImplementedError + def _clone_if_mutable(self): + return self + def clone_if_mutable(self): + clone = self._clone_if_mutable() + if not we_are_translated(): + assert clone.__class__ is self.__class__ + return clone + class AbstractFailDescr(AbstractDescr): index = -1 - def handle_fail(self, metainterp_sd): + def handle_fail(self, metainterp_sd, jitdriver_sd): raise NotImplementedError def compile_and_attach(self, metainterp, new_loop): raise NotImplementedError @@ -694,6 +699,7 @@ generated assembler. """ terminating = False # see TerminatingLoopToken in compile.py + outermost_jitdriver_sd = None # specnodes = ... # and more data specified by the backend when the loop is compiled number = 0 Modified: pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py Tue Jul 6 00:00:33 2010 @@ -56,6 +56,7 @@ self.parent_resumedata_snapshot = None self.parent_resumedata_frame_info_list = None + @specialize.arg(3) def copy_constants(self, registers, constants, ConstClass): """Copy jitcode.constants[0] to registers[255], jitcode.constants[1] to registers[254], @@ -68,7 +69,6 @@ assert j >= 0 registers[j] = ConstClass(constants[i]) i -= 1 - copy_constants._annspecialcase_ = 'specialize:arg(3)' def cleanup_registers(self): # To avoid keeping references alive, this cleans up the registers_r. @@ -80,6 +80,7 @@ # ------------------------------ # Decoding of the JitCode + @specialize.arg(4) def prepare_list_of_boxes(self, outvalue, startindex, position, argcode): assert argcode in 'IRF' code = self.bytecode @@ -92,7 +93,6 @@ elif argcode == 'F': reg = self.registers_f[index] else: raise AssertionError(argcode) outvalue[startindex+i] = reg - prepare_list_of_boxes._annspecialcase_ = 'specialize:arg(4)' def get_current_position_info(self): return self.jitcode.get_live_vars_info(self.pc) @@ -378,6 +378,14 @@ opimpl_getarrayitem_gc_f = _opimpl_getarrayitem_gc_any @arguments("box", "descr", "box") + def _opimpl_getarrayitem_raw_any(self, arraybox, arraydescr, indexbox): + return self.execute_with_descr(rop.GETARRAYITEM_RAW, + arraydescr, arraybox, indexbox) + + opimpl_getarrayitem_raw_i = _opimpl_getarrayitem_raw_any + opimpl_getarrayitem_raw_f = _opimpl_getarrayitem_raw_any + + @arguments("box", "descr", "box") def _opimpl_getarrayitem_gc_pure_any(self, arraybox, arraydescr, indexbox): return self.execute_with_descr(rop.GETARRAYITEM_GC_PURE, arraydescr, arraybox, indexbox) @@ -396,6 +404,15 @@ opimpl_setarrayitem_gc_r = _opimpl_setarrayitem_gc_any opimpl_setarrayitem_gc_f = _opimpl_setarrayitem_gc_any + @arguments("box", "descr", "box", "box") + def _opimpl_setarrayitem_raw_any(self, arraybox, arraydescr, + indexbox, itembox): + self.execute_with_descr(rop.SETARRAYITEM_RAW, arraydescr, arraybox, + indexbox, itembox) + + opimpl_setarrayitem_raw_i = _opimpl_setarrayitem_raw_any + opimpl_setarrayitem_raw_f = _opimpl_setarrayitem_raw_any + @arguments("box", "descr") def opimpl_arraylen_gc(self, arraybox, arraydescr): return self.execute_with_descr(rop.ARRAYLEN_GC, arraydescr, arraybox) @@ -516,6 +533,8 @@ def _nonstandard_virtualizable(self, pc, box): # returns True if 'box' is actually not the "standard" virtualizable # that is stored in metainterp.virtualizable_boxes[-1] + if self.metainterp.jitdriver_sd.virtualizable_info is None: + return True # can occur in case of multiple JITs standard_box = self.metainterp.virtualizable_boxes[-1] if standard_box is box: return False @@ -528,17 +547,11 @@ return not isstandard def _get_virtualizable_field_index(self, fielddescr): - vinfo = self.metainterp.staticdata.virtualizable_info + # Get the index of a fielddescr. Must only be called for + # the "standard" virtualizable. + vinfo = self.metainterp.jitdriver_sd.virtualizable_info return vinfo.static_field_by_descrs[fielddescr] - def _get_virtualizable_array_field_descr(self, index): - vinfo = self.metainterp.staticdata.virtualizable_info - return vinfo.array_field_descrs[index] - - def _get_virtualizable_array_descr(self, index): - vinfo = self.metainterp.staticdata.virtualizable_info - return vinfo.array_descrs[index] - @arguments("orgpc", "box", "descr") def _opimpl_getfield_vable(self, pc, box, fielddescr): if self._nonstandard_virtualizable(pc, box): @@ -566,8 +579,11 @@ opimpl_setfield_vable_f = _opimpl_setfield_vable def _get_arrayitem_vable_index(self, pc, arrayfielddescr, indexbox): + # Get the index of an array item: the index'th of the array + # described by arrayfielddescr. Must only be called for + # the "standard" virtualizable. indexbox = self.implement_guard_value(pc, indexbox) - vinfo = self.metainterp.staticdata.virtualizable_info + vinfo = self.metainterp.jitdriver_sd.virtualizable_info virtualizable_box = self.metainterp.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) arrayindex = vinfo.array_field_by_descrs[arrayfielddescr] @@ -616,7 +632,7 @@ arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, fdescr, box) return self.execute_with_descr(rop.ARRAYLEN_GC, adescr, arraybox) - vinfo = self.metainterp.staticdata.virtualizable_info + vinfo = self.metainterp.jitdriver_sd.virtualizable_info virtualizable_box = self.metainterp.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) arrayindex = vinfo.array_field_by_descrs[fdescr] @@ -665,12 +681,12 @@ opimpl_residual_call_irf_f = _opimpl_residual_call3 opimpl_residual_call_irf_v = _opimpl_residual_call3 - @arguments("boxes3", "boxes3") - def _opimpl_recursive_call(self, greenboxes, redboxes): + @arguments("int", "boxes3", "boxes3") + def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes): + targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex] allboxes = greenboxes + redboxes - metainterp_sd = self.metainterp.staticdata - portal_code = metainterp_sd.portal_code - warmrunnerstate = metainterp_sd.state + portal_code = targetjitdriver_sd.mainjitcode + warmrunnerstate = targetjitdriver_sd.warmstate token = None if warmrunnerstate.inlining: if warmrunnerstate.can_inline_callable(greenboxes): @@ -679,12 +695,13 @@ token = warmrunnerstate.get_assembler_token(greenboxes) # verify that we have all green args, needed to make sure # that assembler that we call is still correct - self.verify_green_args(greenboxes) + self.verify_green_args(targetjitdriver_sd, greenboxes) # - k = llmemory.cast_ptr_to_adr(metainterp_sd._portal_runner_ptr) + k = targetjitdriver_sd.portal_runner_adr funcbox = ConstInt(heaptracker.adr2int(k)) return self.do_residual_call(funcbox, portal_code.calldescr, - allboxes, assembler_call_token=token) + allboxes, assembler_call_token=token, + assembler_call_jd=targetjitdriver_sd) opimpl_recursive_call_i = _opimpl_recursive_call opimpl_recursive_call_r = _opimpl_recursive_call @@ -768,24 +785,27 @@ self.generate_guard(rop.GUARD_CLASS, box, [clsbox], resumepc=orgpc) return clsbox - @arguments() - def opimpl_can_enter_jit(self): + @arguments("int") + def opimpl_can_enter_jit(self, jdindex): if self.metainterp.in_recursion: from pypy.jit.metainterp.warmspot import CannotInlineCanEnterJit raise CannotInlineCanEnterJit() + assert jdindex == self.metainterp.jitdriver_sd.index, ( + "found a can_enter_jit that does not match the current jitdriver") self.metainterp.seen_can_enter_jit = True - def verify_green_args(self, varargs): - num_green_args = self.metainterp.staticdata.num_green_args + def verify_green_args(self, jitdriver_sd, varargs): + num_green_args = jitdriver_sd.num_green_args assert len(varargs) == num_green_args for i in range(num_green_args): assert isinstance(varargs[i], Const) - @arguments("orgpc", "boxes3", "boxes3") - def opimpl_jit_merge_point(self, orgpc, greenboxes, redboxes): - self.verify_green_args(greenboxes) + @arguments("orgpc", "int", "boxes3", "boxes3") + def opimpl_jit_merge_point(self, orgpc, jdindex, greenboxes, redboxes): + jitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex] + self.verify_green_args(jitdriver_sd, greenboxes) # xxx we may disable the following line in some context later - self.debug_merge_point(greenboxes) + self.debug_merge_point(jitdriver_sd, greenboxes) if self.metainterp.seen_can_enter_jit: self.metainterp.seen_can_enter_jit = False # Assert that it's impossible to arrive here with in_recursion @@ -793,6 +813,7 @@ # to True by opimpl_can_enter_jit, which should be executed # just before opimpl_jit_merge_point (no recursion inbetween). assert not self.metainterp.in_recursion + assert jitdriver_sd is self.metainterp.jitdriver_sd # Set self.pc to point to jit_merge_point instead of just after: # if reached_can_enter_jit() raises SwitchToBlackhole, then the # pc is still at the jit_merge_point, which is a point that is @@ -802,10 +823,9 @@ self.metainterp.reached_can_enter_jit(greenboxes, redboxes) self.pc = saved_pc - def debug_merge_point(self, greenkey): + def debug_merge_point(self, jitdriver_sd, greenkey): # debugging: produce a DEBUG_MERGE_POINT operation - sd = self.metainterp.staticdata - loc = sd.state.get_location_str(greenkey) + loc = jitdriver_sd.warmstate.get_location_str(greenkey) debug_print(loc) constloc = self.metainterp.cpu.ts.conststr(loc) self.metainterp.history.record(rop.DEBUG_MERGE_POINT, @@ -948,14 +968,15 @@ original_greenkey = metainterp.resumekey.original_greenkey if opnum == rop.GUARD_NOT_FORCED: resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd, - original_greenkey) + original_greenkey, + metainterp.jitdriver_sd) else: resumedescr = compile.ResumeGuardDescr(metainterp_sd, original_greenkey) guard_op = metainterp.history.record(opnum, moreargs, None, descr=resumedescr) virtualizable_boxes = None - if metainterp.staticdata.virtualizable_info is not None: + if metainterp.jitdriver_sd.virtualizable_info is not None: virtualizable_boxes = metainterp.virtualizable_boxes saved_pc = self.pc if resumepc >= 0: @@ -1007,7 +1028,8 @@ return resbox def do_residual_call(self, funcbox, descr, argboxes, - assembler_call_token=None): + assembler_call_token=None, + assembler_call_jd=None): # First build allboxes: it may need some reordering from the # list provided in argboxes, depending on the order in which # the arguments are expected by the function @@ -1052,7 +1074,8 @@ rop.CALL_MAY_FORCE, allboxes, descr=descr) self.metainterp.vrefs_after_residual_call() if assembler_call_token is not None: - self.metainterp.direct_assembler_call(assembler_call_token) + self.metainterp.direct_assembler_call(assembler_call_token, + assembler_call_jd) if resbox is not None: self.make_result_of_lastop(resbox) self.metainterp.vable_after_residual_call() @@ -1137,6 +1160,11 @@ self._addr2name_keys = [key for key, value in list_of_addr2name] self._addr2name_values = [value for key, value in list_of_addr2name] + def setup_jitdrivers_sd(self, optimizer): + if optimizer is not None: + for jd in self.jitdrivers_sd: + jd.warmstate.set_param_optimizer(optimizer) + def finish_setup(self, codewriter, optimizer=None): from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder self.blackholeinterpbuilder = BlackholeInterpBuilder(codewriter, self) @@ -1147,28 +1175,21 @@ self.setup_indirectcalltargets(asm.indirectcalltargets) self.setup_list_of_addr2name(asm.list_of_addr2name) # - self.portal_code = codewriter.mainjitcode - self._portal_runner_ptr = codewriter.callcontrol.portal_runner_ptr + self.jitdrivers_sd = codewriter.callcontrol.jitdrivers_sd self.virtualref_info = codewriter.callcontrol.virtualref_info - self.virtualizable_info = codewriter.callcontrol.virtualizable_info - RESULT = codewriter.portal_graph.getreturnvar().concretetype - self.result_type = history.getkind(RESULT) + self.setup_jitdrivers_sd(optimizer) # # store this information for fastpath of call_assembler - name = self.result_type - tokens = getattr(self, 'loop_tokens_done_with_this_frame_%s' % name) - num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) - setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num) + # (only the paths that can actually be taken) + for jd in self.jitdrivers_sd: + name = {history.INT: 'int', + history.REF: 'ref', + history.FLOAT: 'float', + history.VOID: 'void'}[jd.result_type] + tokens = getattr(self, 'loop_tokens_done_with_this_frame_%s' % name) + num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) + setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num) # - warmrunnerdesc = self.warmrunnerdesc - if warmrunnerdesc is not None: - self.num_green_args = warmrunnerdesc.num_green_args - self.state = warmrunnerdesc.state - if optimizer is not None: - self.state.set_param_optimizer(optimizer) - else: - self.num_green_args = 0 - self.state = None self.globaldata = MetaInterpGlobalData(self) def _setup_once(self): @@ -1189,8 +1210,7 @@ # Build the dictionary at run-time. This is needed # because the keys are function/class addresses, so they # can change from run to run. - k = llmemory.cast_ptr_to_adr(self._portal_runner_ptr) - d = {k: 'recursive call'} + d = {} keys = self._addr2name_keys values = self._addr2name_values for i in range(len(keys)): @@ -1248,47 +1268,31 @@ self.loopnumbering = 0 self.resume_virtuals = {} self.resume_virtuals_not_translated = [] - # - state = staticdata.state - if state is not None: - self.jit_cell_at_key = state.jit_cell_at_key - else: - # for tests only; not RPython - class JitCell: - compiled_merge_points = None - _jitcell_dict = {} - def jit_cell_at_key(greenkey): - greenkey = tuple(greenkey) - return _jitcell_dict.setdefault(greenkey, JitCell()) - self.jit_cell_at_key = jit_cell_at_key - - def get_compiled_merge_points(self, greenkey): - cell = self.jit_cell_at_key(greenkey) - if cell.compiled_merge_points is None: - cell.compiled_merge_points = [] - return cell.compiled_merge_points # ____________________________________________________________ class MetaInterp(object): in_recursion = 0 - def __init__(self, staticdata): + def __init__(self, staticdata, jitdriver_sd): self.staticdata = staticdata self.cpu = staticdata.cpu + self.jitdriver_sd = jitdriver_sd + # Note: self.jitdriver_sd is the JitDriverStaticData that corresponds + # to the current loop -- the outermost one. Be careful, because + # during recursion we can also see other jitdrivers. self.portal_trace_positions = [] self.free_frames_list = [] self.last_exc_value_box = None def perform_call(self, jitcode, boxes, greenkey=None): # causes the metainterp to enter the given subfunction - # with a special case for recursive portal calls f = self.newframe(jitcode, greenkey) f.setup_call(boxes) raise ChangeFrame def newframe(self, jitcode, greenkey=None): - if jitcode is self.staticdata.portal_code: + if jitcode.is_portal: self.in_recursion += 1 if greenkey is not None: self.portal_trace_positions.append( @@ -1303,7 +1307,7 @@ def popframe(self): frame = self.framestack.pop() - if frame.jitcode is self.staticdata.portal_code: + if frame.jitcode.is_portal: self.in_recursion -= 1 if frame.greenkey is not None: self.portal_trace_positions.append( @@ -1327,14 +1331,15 @@ except SwitchToBlackhole, stb: self.aborted_tracing(stb.reason) sd = self.staticdata - if sd.result_type == 'void': + result_type = self.jitdriver_sd.result_type + if result_type == history.VOID: assert resultbox is None raise sd.DoneWithThisFrameVoid() - elif sd.result_type == 'int': + elif result_type == history.INT: raise sd.DoneWithThisFrameInt(resultbox.getint()) - elif sd.result_type == 'ref': + elif result_type == history.REF: raise sd.DoneWithThisFrameRef(self.cpu, resultbox.getref_base()) - elif sd.result_type == 'float': + elif result_type == history.FLOAT: raise sd.DoneWithThisFrameFloat(resultbox.getfloat()) else: assert False @@ -1364,14 +1369,17 @@ in_recursion = -1 for frame in self.framestack: jitcode = frame.jitcode - if jitcode is self.staticdata.portal_code: + assert jitcode.is_portal == len([ + jd for jd in self.staticdata.jitdrivers_sd + if jd.mainjitcode is jitcode]) + if jitcode.is_portal: in_recursion += 1 if in_recursion != self.in_recursion: print "in_recursion problem!!!" print in_recursion, self.in_recursion for frame in self.framestack: jitcode = frame.jitcode - if jitcode is self.staticdata.portal_code: + if jitcode.is_portal: print "P", else: print " ", @@ -1379,7 +1387,6 @@ raise AssertionError def create_empty_history(self): - warmrunnerstate = self.staticdata.state self.history = history.History() self.staticdata.stats.set_history(self.history) @@ -1489,12 +1496,11 @@ self.resumekey.reset_counter_from_failure() def blackhole_if_trace_too_long(self): - warmrunnerstate = self.staticdata.state + warmrunnerstate = self.jitdriver_sd.warmstate if len(self.history.operations) > warmrunnerstate.trace_limit: greenkey_of_huge_function = self.find_biggest_function() self.portal_trace_positions = None if greenkey_of_huge_function is not None: - warmrunnerstate = self.staticdata.state warmrunnerstate.disable_noninlinable_function( greenkey_of_huge_function) raise SwitchToBlackhole(ABORT_TOO_LONG) @@ -1521,21 +1527,27 @@ self.staticdata.log(sys.exc_info()[0].__name__) raise - def compile_and_run_once(self, *args): + @specialize.arg(1) + def compile_and_run_once(self, jitdriver_sd, *args): + # NB. we pass explicity 'jitdriver_sd' around here, even though it + # is also available as 'self.jitdriver_sd', because we need to + # specialize this function and a few other ones for the '*args'. debug_start('jit-tracing') self.staticdata._setup_once() self.staticdata.profiler.start_tracing() + assert jitdriver_sd is self.jitdriver_sd self.create_empty_history() try: - return self._compile_and_run_once(*args) + original_boxes = self.initialize_original_boxes(jitdriver_sd,*args) + return self._compile_and_run_once(original_boxes) finally: self.staticdata.profiler.end_tracing() debug_stop('jit-tracing') - def _compile_and_run_once(self, *args): - original_boxes = self.initialize_state_from_start(*args) + def _compile_and_run_once(self, original_boxes): + self.initialize_state_from_start(original_boxes) self.current_merge_points = [(original_boxes, 0)] - num_green_args = self.staticdata.num_green_args + num_green_args = self.jitdriver_sd.num_green_args original_greenkey = original_boxes[:num_green_args] redkey = original_boxes[num_green_args:] self.resumekey = compile.ResumeFromInterpDescr(original_greenkey, @@ -1602,7 +1614,7 @@ self.remove_consts_and_duplicates(redboxes, len(redboxes), duplicates) live_arg_boxes = greenboxes + redboxes - if self.staticdata.virtualizable_info is not None: + if self.jitdriver_sd.virtualizable_info is not None: # we use pop() to remove the last item, which is the virtualizable # itself self.remove_consts_and_duplicates(self.virtualizable_boxes, @@ -1624,11 +1636,12 @@ # Search in current_merge_points for original_boxes with compatible # green keys, representing the beginning of the same loop as the one # we end now. - + + num_green_args = self.jitdriver_sd.num_green_args for j in range(len(self.current_merge_points)-1, -1, -1): original_boxes, start = self.current_merge_points[j] assert len(original_boxes) == len(live_arg_boxes) or start < 0 - for i in range(self.staticdata.num_green_args): + for i in range(num_green_args): box1 = original_boxes[i] box2 = live_arg_boxes[i] assert isinstance(box1, Const) @@ -1651,7 +1664,7 @@ def designate_target_loop(self, gmp): loop_token = gmp.target_loop_token - num_green_args = self.staticdata.num_green_args + num_green_args = self.jitdriver_sd.num_green_args residual_args = self.get_residual_args(loop_token.specnodes, gmp.argboxes[num_green_args:]) history.set_future_values(self.cpu, residual_args) @@ -1692,12 +1705,17 @@ from pypy.jit.metainterp.resoperation import opname raise NotImplementedError(opname[opnum]) + def get_compiled_merge_points(self, greenkey): + cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey) + if cell.compiled_merge_points is None: + cell.compiled_merge_points = [] + return cell.compiled_merge_points + def compile(self, original_boxes, live_arg_boxes, start): - num_green_args = self.staticdata.num_green_args + num_green_args = self.jitdriver_sd.num_green_args self.history.inputargs = original_boxes[num_green_args:] greenkey = original_boxes[:num_green_args] - glob = self.staticdata.globaldata - old_loop_tokens = glob.get_compiled_merge_points(greenkey) + old_loop_tokens = self.get_compiled_merge_points(greenkey) self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) loop_token = compile.compile_new_loop(self, old_loop_tokens, greenkey, start) @@ -1706,10 +1724,9 @@ self.history.operations.pop() # remove the JUMP def compile_bridge(self, live_arg_boxes): - num_green_args = self.staticdata.num_green_args + num_green_args = self.jitdriver_sd.num_green_args greenkey = live_arg_boxes[:num_green_args] - glob = self.staticdata.globaldata - old_loop_tokens = glob.get_compiled_merge_points(greenkey) + old_loop_tokens = self.get_compiled_merge_points(greenkey) if len(old_loop_tokens) == 0: return self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) @@ -1723,17 +1740,18 @@ self.gen_store_back_in_virtualizable() # temporarily put a JUMP to a pseudo-loop sd = self.staticdata - if sd.result_type == 'void': + result_type = self.jitdriver_sd.result_type + if result_type == history.VOID: assert exitbox is None exits = [] loop_tokens = sd.loop_tokens_done_with_this_frame_void - elif sd.result_type == 'int': + elif result_type == history.INT: exits = [exitbox] loop_tokens = sd.loop_tokens_done_with_this_frame_int - elif sd.result_type == 'ref': + elif result_type == history.REF: exits = [exitbox] loop_tokens = sd.loop_tokens_done_with_this_frame_ref - elif sd.result_type == 'float': + elif result_type == history.FLOAT: exits = [exitbox] loop_tokens = sd.loop_tokens_done_with_this_frame_float else: @@ -1765,26 +1783,32 @@ specnode.extract_runtime_data(self.cpu, args[i], expanded_args) return expanded_args - def _initialize_from_start(self, original_boxes, num_green_args, *args): + @specialize.arg(1) + def initialize_original_boxes(self, jitdriver_sd, *args): + original_boxes = [] + self._fill_original_boxes(jitdriver_sd, original_boxes, + jitdriver_sd.num_green_args, *args) + return original_boxes + + @specialize.arg(1) + def _fill_original_boxes(self, jitdriver_sd, original_boxes, + num_green_args, *args): if args: from pypy.jit.metainterp.warmstate import wrap box = wrap(self.cpu, args[0], num_green_args > 0) original_boxes.append(box) - self._initialize_from_start(original_boxes, num_green_args-1, - *args[1:]) + self._fill_original_boxes(jitdriver_sd, original_boxes, + num_green_args-1, *args[1:]) - def initialize_state_from_start(self, *args): - self.in_recursion = -1 # always one portal around - num_green_args = self.staticdata.num_green_args - original_boxes = [] - self._initialize_from_start(original_boxes, num_green_args, *args) + def initialize_state_from_start(self, original_boxes): # ----- make a new frame ----- + self.in_recursion = -1 # always one portal around self.framestack = [] - f = self.newframe(self.staticdata.portal_code) + f = self.newframe(self.jitdriver_sd.mainjitcode) f.setup_call(original_boxes) + assert self.in_recursion == 0 self.virtualref_boxes = [] self.initialize_virtualizable(original_boxes) - return original_boxes def initialize_state_from_guard_failure(self, resumedescr): # guard failure: rebuild a complete MIFrame stack @@ -1794,9 +1818,11 @@ self.history.inputargs = [box for box in inputargs_and_holes if box] def initialize_virtualizable(self, original_boxes): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info if vinfo is not None: - virtualizable_box = original_boxes[vinfo.index_of_virtualizable] + index = (self.jitdriver_sd.num_green_args + + self.jitdriver_sd.index_of_virtualizable) + virtualizable_box = original_boxes[index] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) # The field 'virtualizable_boxes' is not even present # if 'virtualizable_info' is None. Check for that first. @@ -1807,7 +1833,7 @@ self.initialize_virtualizable_enter() def initialize_virtualizable_enter(self): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.clear_vable_token(virtualizable) @@ -1821,7 +1847,7 @@ # the FORCE_TOKEN is already set at runtime in each vref when # it is created, by optimizeopt.py. # - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) @@ -1847,7 +1873,7 @@ self.stop_tracking_virtualref(i) def vable_after_residual_call(self): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) @@ -1898,7 +1924,7 @@ assert self.last_exc_value_box is None def rebuild_state_after_failure(self, resumedescr): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info self.framestack = [] boxlists = resume.rebuild_from_resumedata(self, resumedescr, vinfo) inputargs_and_holes, virtualizable_boxes, virtualref_boxes = boxlists @@ -1923,24 +1949,19 @@ virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) assert not virtualizable.vable_token - if 0: ## self._already_allocated_resume_virtuals is not None: - # resuming from a ResumeGuardForcedDescr: load the new values - # currently stored on the virtualizable fields - self.load_fields_from_virtualizable() - else: - # normal case: fill the virtualizable with the local boxes - self.synchronize_virtualizable() + # fill the virtualizable with the local boxes + self.synchronize_virtualizable() return inputargs_and_holes def check_synchronized_virtualizable(self): if not we_are_translated(): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.check_boxes(virtualizable, self.virtualizable_boxes) def synchronize_virtualizable(self): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.write_boxes(virtualizable, self.virtualizable_boxes) @@ -1949,7 +1970,7 @@ # Force a reload of the virtualizable fields into the local # boxes (called only in escaping cases). Only call this function # just before SwitchToBlackhole. - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) @@ -1958,7 +1979,7 @@ self.virtualizable_boxes.append(virtualizable_box) def gen_store_back_in_virtualizable(self): - vinfo = self.staticdata.virtualizable_info + vinfo = self.jitdriver_sd.virtualizable_info if vinfo is not None: # xxx only write back the fields really modified vbox = self.virtualizable_boxes[-1] @@ -1979,8 +2000,7 @@ abox, ConstInt(j), itembox) assert i + 1 == len(self.virtualizable_boxes) - def gen_load_from_other_virtualizable(self, vbox): - vinfo = self.staticdata.virtualizable_info + def gen_load_from_other_virtualizable(self, vinfo, vbox): boxes = [] assert vinfo is not None for i in range(vinfo.num_static_extra_boxes): @@ -2004,7 +2024,7 @@ for i in range(len(boxes)): if boxes[i] is oldbox: boxes[i] = newbox - if self.staticdata.virtualizable_info is not None: + if self.jitdriver_sd.virtualizable_info is not None: boxes = self.virtualizable_boxes for i in range(len(boxes)): if boxes[i] is oldbox: @@ -2052,18 +2072,19 @@ op.args = [resbox_as_const] + op.args return resbox - def direct_assembler_call(self, token): + def direct_assembler_call(self, token, targetjitdriver_sd): """ Generate a direct call to assembler for portal entry point, patching the CALL_MAY_FORCE that occurred just now. """ op = self.history.operations.pop() assert op.opnum == rop.CALL_MAY_FORCE - num_green_args = self.staticdata.num_green_args + num_green_args = targetjitdriver_sd.num_green_args args = op.args[num_green_args + 1:] - if self.staticdata.virtualizable_info is not None: - vindex = self.staticdata.virtualizable_info.index_of_virtualizable - vbox = args[vindex - num_green_args] - args = args + self.gen_load_from_other_virtualizable(vbox) + vinfo = targetjitdriver_sd.virtualizable_info + if vinfo is not None: + index = targetjitdriver_sd.index_of_virtualizable + vbox = args[index] + args = args + self.gen_load_from_other_virtualizable(vinfo, vbox) # ^^^ and not "+=", which makes 'args' a resizable list op.opnum = rop.CALL_ASSEMBLER op.args = args @@ -2166,6 +2187,16 @@ position = position3 + 1 + length3 elif argtype == "orgpc": value = orgpc + elif argtype == "int": + argcode = argcodes[next_argcode] + next_argcode = next_argcode + 1 + if argcode == 'i': + value = self.registers_i[ord(code[position])].getint() + elif argcode == 'c': + value = signedord(code[position]) + else: + raise AssertionError("bad argcode") + position += 1 else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) Modified: pypy/branch/fast-forward/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/resoperation.py Tue Jul 6 00:00:33 2010 @@ -34,7 +34,7 @@ def clone(self): descr = self.descr if descr is not None: - descr = descr._clone_if_mutable() + descr = descr.clone_if_mutable() op = ResOperation(self.opnum, self.args, self.result, descr) op.fail_args = self.fail_args if not we_are_translated(): @@ -199,6 +199,7 @@ '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- 'GETARRAYITEM_GC/2d', + 'GETARRAYITEM_RAW/2d', 'GETFIELD_GC/1d', 'GETFIELD_RAW/1d', 'NEW/0d', @@ -209,7 +210,7 @@ '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'SETARRAYITEM_GC/3d', - 'SETARRAYITEM_RAW/3d',#only added by backend.llsupport.gc.rewrite_assembler + 'SETARRAYITEM_RAW/3d', 'SETFIELD_GC/2d', 'SETFIELD_RAW/2d', 'ARRAYCOPY/7d', # removed before it's passed to the backend Modified: pypy/branch/fast-forward/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/resume.py Tue Jul 6 00:00:33 2010 @@ -709,11 +709,11 @@ # ---------- when resuming for blackholing, get direct values ---------- -def blackhole_from_resumedata(blackholeinterpbuilder, storage, +def blackhole_from_resumedata(blackholeinterpbuilder, jitdriver_sd, storage, all_virtuals=None): resumereader = ResumeDataDirectReader(blackholeinterpbuilder.cpu, storage, all_virtuals) - vinfo = blackholeinterpbuilder.metainterp_sd.virtualizable_info + vinfo = jitdriver_sd.virtualizable_info vrefinfo = blackholeinterpbuilder.metainterp_sd.virtualref_info resumereader.consume_vref_and_vable(vrefinfo, vinfo) # @@ -745,10 +745,9 @@ resumereader.done() return firstbh -def force_from_resumedata(metainterp_sd, storage): +def force_from_resumedata(metainterp_sd, storage, vinfo=None): resumereader = ResumeDataDirectReader(metainterp_sd.cpu, storage) resumereader.handling_async_forcing() - vinfo = metainterp_sd.virtualizable_info vrefinfo = metainterp_sd.virtualref_info resumereader.consume_vref_and_vable(vrefinfo, vinfo) return resumereader.virtuals Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_basic.py Tue Jul 6 00:00:33 2010 @@ -16,10 +16,18 @@ from pypy.jit.codewriter import support, codewriter from pypy.jit.metainterp import simple_optimize + class FakeJitCell: + compiled_merge_points = None + class FakeWarmRunnerState: def attach_unoptimized_bridge_from_interp(self, greenkey, newloop): pass + def jit_cell_at_key(self, greenkey): + assert greenkey == [] + return self._cell + _cell = FakeJitCell() + # pick the optimizer this way optimize_loop = staticmethod(simple_optimize.optimize_loop) optimize_bridge = staticmethod(simple_optimize.optimize_bridge) @@ -30,14 +38,24 @@ func._jit_unroll_safe_ = True rtyper = support.annotate(func, values, type_system=type_system) graphs = rtyper.annotator.translator.graphs + result_kind = history.getkind(graphs[0].getreturnvar().concretetype)[0] + + class FakeJitDriverSD: + num_green_args = 0 + portal_graph = graphs[0] + virtualizable_info = None + result_type = result_kind + portal_runner_ptr = "???" + stats = history.Stats() cpu = CPUClass(rtyper, stats, None, False) - cw = codewriter.CodeWriter(cpu, graphs[0]) + cw = codewriter.CodeWriter(cpu, [FakeJitDriverSD()]) testself.cw = cw cw.find_all_graphs(JitPolicy()) # testself.warmrunnerstate = FakeWarmRunnerState() testself.warmrunnerstate.cpu = cpu + FakeJitDriverSD.warmstate = testself.warmrunnerstate if hasattr(testself, 'finish_setup_for_interp_operations'): testself.finish_setup_for_interp_operations() # @@ -62,7 +80,8 @@ count_f += 1 else: raise TypeError(T) - blackholeinterp.setposition(cw.mainjitcode, 0) + [jitdriver_sd] = cw.callcontrol.jitdrivers_sd + blackholeinterp.setposition(jitdriver_sd.mainjitcode, 0) blackholeinterp.run() return blackholeinterp._final_result_anytype() @@ -78,16 +97,15 @@ cw = testself.cw opt = history.Options(listops=True) metainterp_sd = pyjitpl.MetaInterpStaticData(cw.cpu, opt) - metainterp_sd.finish_setup(cw, optimizer="bogus") - metainterp_sd.state = testself.warmrunnerstate - metainterp_sd.state.cpu = metainterp_sd.cpu - metainterp = pyjitpl.MetaInterp(metainterp_sd) + metainterp_sd.finish_setup(cw) + [jitdriver_sd] = metainterp_sd.jitdrivers_sd + metainterp = pyjitpl.MetaInterp(metainterp_sd, jitdriver_sd) metainterp_sd.DoneWithThisFrameInt = DoneWithThisFrame metainterp_sd.DoneWithThisFrameRef = DoneWithThisFrameRef metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrame testself.metainterp = metainterp try: - metainterp.compile_and_run_once(*args) + metainterp.compile_and_run_once(jitdriver_sd, *args) except DoneWithThisFrame, e: #if conftest.option.view: # metainterp.stats.view() @@ -864,8 +882,9 @@ translator.config.translation.gc = "boehm" warmrunnerdesc = WarmRunnerDesc(translator, CPUClass=self.CPUClass) - warmrunnerdesc.state.set_param_threshold(3) # for tests - warmrunnerdesc.state.set_param_trace_eagerness(0) # for tests + state = warmrunnerdesc.jitdrivers_sd[0].warmstate + state.set_param_threshold(3) # for tests + state.set_param_trace_eagerness(0) # for tests warmrunnerdesc.finish() for n, k in [(20, 0), (20, 1)]: interp.eval_graph(graph, [n, k]) @@ -1543,6 +1562,35 @@ res = self.interp_operations(f, [3, 2]) assert res == 1 + def test_raw_malloc_and_access(self): + from pypy.rpython.lltypesystem import rffi + + TP = rffi.CArray(lltype.Signed) + + def f(n): + a = lltype.malloc(TP, n, flavor='raw') + a[0] = n + res = a[0] + lltype.free(a, flavor='raw') + return res + + res = self.interp_operations(f, [10]) + assert res == 10 + + def test_raw_malloc_and_access_float(self): + from pypy.rpython.lltypesystem import rffi + + TP = rffi.CArray(lltype.Float) + + def f(n, f): + a = lltype.malloc(TP, n, flavor='raw') + a[0] = f + res = a[0] + lltype.free(a, flavor='raw') + return res + + res = self.interp_operations(f, [10, 3.5]) + assert res == 3.5 class TestOOtype(BasicTests, OOJitMixin): Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_blackhole.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_blackhole.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_blackhole.py Tue Jul 6 00:00:33 2010 @@ -42,7 +42,7 @@ blackholeinterp.setarg_i(0, 40) blackholeinterp.setarg_i(1, 2) blackholeinterp.run() - assert blackholeinterp.final_result_i() == 42 + assert blackholeinterp._final_result_anytype() == 42 def test_simple_const(): jitcode = JitCode("test") @@ -54,7 +54,7 @@ blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(1, 6) blackholeinterp.run() - assert blackholeinterp.final_result_i() == 42 + assert blackholeinterp._final_result_anytype() == 42 def test_simple_bigconst(): jitcode = JitCode("test") @@ -66,7 +66,7 @@ blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(1, 10000) blackholeinterp.run() - assert blackholeinterp.final_result_i() == 42 + assert blackholeinterp._final_result_anytype() == 42 def test_simple_loop(): jitcode = JitCode("test") @@ -85,7 +85,7 @@ blackholeinterp.setarg_i(0x16, 6) # %i0 blackholeinterp.setarg_i(0x17, 100) # %i1 blackholeinterp.run() - assert blackholeinterp.final_result_i() == 100+6+5+4+3 + assert blackholeinterp._final_result_anytype() == 100+6+5+4+3 def test_simple_exception(): jitcode = JitCode("test") @@ -104,12 +104,12 @@ blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(0x9, 100) blackholeinterp.run() - assert blackholeinterp.final_result_i() == 200 + assert blackholeinterp._final_result_anytype() == 200 # blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(0x9, -100) blackholeinterp.run() - assert blackholeinterp.final_result_i() == 42 + assert blackholeinterp._final_result_anytype() == 42 def test_convert_and_run_from_pyjitpl(): class MyMIFrame: Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_compile.py Tue Jul 6 00:00:33 2010 @@ -51,14 +51,14 @@ logger_noopt = FakeLogger() logger_ops = FakeLogger() - state = FakeState() stats = Stats() profiler = jitprof.EmptyProfiler() def log(self, msg, event_kind=None): pass class FakeMetaInterp: - pass + class jitdriver_sd: + warmstate = FakeState() def test_compile_new_loop(): cpu = FakeCPU() Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_list.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_list.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_list.py Tue Jul 6 00:00:33 2010 @@ -143,6 +143,51 @@ assert res == 7 self.check_loops(call=0) + def test_fold_getitem_1(self): + jitdriver = JitDriver(greens = ['pc', 'n', 'l'], reds = ['total']) + def f(n): + l = [100, n, 300, n, 500] + total = 0 + pc = n + while True: + jitdriver.can_enter_jit(l=l, pc=pc, n=n, total=total) + jitdriver.jit_merge_point(l=l, pc=pc, n=n, total=total) + total += l[pc] + if total > 10000: + return total + pc -= 1 + if pc < 0: + pc = n + + res = self.meta_interp(f, [4], listops=True) + assert res == f(4) + self.check_loops(call=0) + + def test_fold_getitem_2(self): + jitdriver = JitDriver(greens = ['pc', 'n', 'l'], reds = ['total', 'x']) + class X: + pass + def f(n): + l = [100, n, 300, n, 500] + total = 0 + x = X() + x.pc = n + while True: + pc = x.pc + jitdriver.can_enter_jit(l=l, pc=pc, n=n, total=total, x=x) + jitdriver.jit_merge_point(l=l, pc=pc, n=n, total=total, x=x) + x.pc = pc + total += l[x.pc] + if total > 10000: + return total + x.pc -= 1 + if x.pc < 0: + x.pc = n + + res = self.meta_interp(f, [4], listops=True) + assert res == f(4) + self.check_loops(call=0, getfield_gc=0) + class TestOOtype(ListTests, OOJitMixin): pass Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_pyjitpl.py Tue Jul 6 00:00:33 2010 @@ -17,9 +17,8 @@ portal.setup(None) class FakeStaticData: cpu = None - portal_code = portal - metainterp = pyjitpl.MetaInterp(FakeStaticData()) + metainterp = pyjitpl.MetaInterp(FakeStaticData(), None) metainterp.framestack = [] class FakeHistory: operations = [] @@ -59,7 +58,7 @@ assert isinstance(box, referencebox.clonebox().__class__) assert box.value == referencebox.value return True - metainterp = pyjitpl.MetaInterp(FakeStaticData()) + metainterp = pyjitpl.MetaInterp(FakeStaticData(), None) metainterp.history = History() b1 = BoxInt(1) b2 = BoxInt(2) Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmspot.py Tue Jul 6 00:00:33 2010 @@ -128,8 +128,9 @@ from pypy.jit.metainterp import optimize - assert warmrunnerdescr.state.optimize_loop is optimize.optimize_loop - assert warmrunnerdescr.state.optimize_bridge is optimize.optimize_bridge + state = warmrunnerdescr.jitdrivers_sd[0].warmstate + assert state.optimize_loop is optimize.optimize_loop + assert state.optimize_bridge is optimize.optimize_bridge def test_static_debug_level(self, capfd): py.test.skip("debug_level is being deprecated") @@ -294,7 +295,7 @@ def __init__(self, no): self.no = no - def handle_fail(self, metainterp_sd): + def handle_fail(self, metainterp_sd, jitdrivers_sd): if self.no == 0: raise metainterp_sd.warmrunnerdesc.DoneWithThisFrameInt(3) if self.no == 1: @@ -355,12 +356,13 @@ def test_call_helper(self): from pypy.rpython.llinterp import LLException - - assert self.desc.assembler_call_helper(0, 0) == 3 - assert self.desc.assembler_call_helper(1, 0) == 10 - assert self.desc.assembler_call_helper(2, 0) == 10 + + [jd] = self.desc.jitdrivers_sd + assert jd._assembler_call_helper(0, 0) == 3 + assert jd._assembler_call_helper(1, 0) == 10 + assert jd._assembler_call_helper(2, 0) == 10 try: - self.desc.assembler_call_helper(3, 0) + jd._assembler_call_helper(3, 0) except LLException, lle: assert lle[0] == self.exc_vtable else: Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmstate.py Tue Jul 6 00:00:33 2010 @@ -4,10 +4,9 @@ from pypy.rpython.annlowlevel import llhelper from pypy.jit.metainterp.warmstate import wrap, unwrap from pypy.jit.metainterp.warmstate import equal_whatever, hash_whatever -from pypy.jit.metainterp.warmstate import WarmEnterState +from pypy.jit.metainterp.warmstate import WarmEnterState, JitCell from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr from pypy.jit.metainterp.history import ConstInt, ConstFloat, ConstPtr -from pypy.rlib.jit import BaseJitCell def test_unwrap(): @@ -58,14 +57,12 @@ def test_make_jitcell_getter_default(): - class FakeWarmRunnerDesc: - green_args_spec = [lltype.Signed, lltype.Float] - class FakeJitCell(BaseJitCell): - pass - state = WarmEnterState(FakeWarmRunnerDesc()) - get_jitcell = state._make_jitcell_getter_default(FakeJitCell) + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + state = WarmEnterState(None, FakeJitDriverSD()) + get_jitcell = state._make_jitcell_getter_default() cell1 = get_jitcell(42, 42.5) - assert isinstance(cell1, FakeJitCell) + assert isinstance(cell1, JitCell) cell2 = get_jitcell(42, 42.5) assert cell1 is cell2 cell3 = get_jitcell(41, 42.5) @@ -73,10 +70,10 @@ assert cell1 is not cell3 is not cell4 is not cell1 def test_make_jitcell_getter(): - class FakeWarmRunnerDesc: - green_args_spec = [lltype.Float] - get_jitcell_at_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc()) + class FakeJitDriverSD: + _green_args_spec = [lltype.Float] + _get_jitcell_at_ptr = None + state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state.make_jitcell_getter() cell1 = get_jitcell(1.75) cell2 = get_jitcell(1.75) @@ -87,8 +84,6 @@ from pypy.rpython.typesystem import LowLevelTypeSystem class FakeRTyper: type_system = LowLevelTypeSystem.instance - class FakeJitCell(BaseJitCell): - pass celldict = {} def getter(x, y): return celldict.get((x, y)) @@ -102,14 +97,14 @@ lltype.Float], lltype.Void)) class FakeWarmRunnerDesc: rtyper = FakeRTyper() - cpu = None - get_jitcell_at_ptr = llhelper(GETTER, getter) - set_jitcell_at_ptr = llhelper(SETTER, setter) + class FakeJitDriverSD: + _get_jitcell_at_ptr = llhelper(GETTER, getter) + _set_jitcell_at_ptr = llhelper(SETTER, setter) # - state = WarmEnterState(FakeWarmRunnerDesc()) - get_jitcell = state._make_jitcell_getter_custom(FakeJitCell) + state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) + get_jitcell = state._make_jitcell_getter_custom() cell1 = get_jitcell(5, 42.5) - assert isinstance(cell1, FakeJitCell) + assert isinstance(cell1, JitCell) assert cell1.x == 5 assert cell1.y == 42.5 cell2 = get_jitcell(5, 42.5) @@ -127,10 +122,11 @@ future_values[j] = "float", value class FakeWarmRunnerDesc: cpu = FakeCPU() - red_args_types = ["int", "float"] + class FakeJitDriverSD: + _red_args_types = ["int", "float"] virtualizable_info = None # - state = WarmEnterState(FakeWarmRunnerDesc()) + state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) set_future_values = state.make_set_future_values() set_future_values(5, 42.5) assert future_values == { @@ -140,19 +136,19 @@ assert set_future_values is state.make_set_future_values() def test_make_unwrap_greenkey(): - class FakeWarmRunnerDesc: - green_args_spec = [lltype.Signed, lltype.Float] - state = WarmEnterState(FakeWarmRunnerDesc()) + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + state = WarmEnterState(None, FakeJitDriverSD()) unwrap_greenkey = state.make_unwrap_greenkey() greenargs = unwrap_greenkey([ConstInt(42), ConstFloat(42.5)]) assert greenargs == (42, 42.5) assert type(greenargs[0]) is int def test_attach_unoptimized_bridge_from_interp(): - class FakeWarmRunnerDesc: - green_args_spec = [lltype.Signed, lltype.Float] - get_jitcell_at_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc()) + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + _get_jitcell_at_ptr = None + state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state.make_jitcell_getter() state.attach_unoptimized_bridge_from_interp([ConstInt(5), ConstFloat(2.25)], @@ -162,14 +158,14 @@ assert cell1.entry_loop_token == "entry loop token" def test_make_jitdriver_callbacks_1(): - class FakeWarmRunnerDesc: - can_inline_ptr = None - get_printable_location_ptr = None - confirm_enter_jit_ptr = None - green_args_spec = [lltype.Signed, lltype.Float] + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + _can_inline_ptr = None + _get_printable_location_ptr = None + _confirm_enter_jit_ptr = None class FakeCell: dont_trace_here = False - state = WarmEnterState(FakeWarmRunnerDesc()) + state = WarmEnterState(None, FakeJitDriverSD()) def jit_getter(*args): return FakeCell() state.jit_getter = jit_getter @@ -190,11 +186,12 @@ dont_trace_here = False class FakeWarmRunnerDesc: rtyper = None - green_args_spec = [lltype.Signed, lltype.Float] - can_inline_ptr = llhelper(CAN_INLINE, can_inline) - get_printable_location_ptr = None - confirm_enter_jit_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc()) + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + _can_inline_ptr = llhelper(CAN_INLINE, can_inline) + _get_printable_location_ptr = None + _confirm_enter_jit_ptr = None + state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) def jit_getter(*args): return FakeCell() state.jit_getter = jit_getter @@ -211,12 +208,13 @@ lltype.Ptr(rstr.STR))) class FakeWarmRunnerDesc: rtyper = None - green_args_spec = [lltype.Signed, lltype.Float] - can_inline_ptr = None - get_printable_location_ptr = llhelper(GET_LOCATION, get_location) - confirm_enter_jit_ptr = None - get_jitcell_at_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc()) + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + _can_inline_ptr = None + _get_printable_location_ptr = llhelper(GET_LOCATION, get_location) + _confirm_enter_jit_ptr = None + _get_jitcell_at_ptr = None + state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) state.make_jitdriver_callbacks() res = state.get_location_str([ConstInt(5), ConstFloat(42.5)]) assert res == "hi there" @@ -231,13 +229,14 @@ lltype.Signed], lltype.Bool)) class FakeWarmRunnerDesc: rtyper = None - green_args_spec = [lltype.Signed, lltype.Float] - can_inline_ptr = None - get_printable_location_ptr = None - confirm_enter_jit_ptr = llhelper(ENTER_JIT, confirm_enter_jit) - get_jitcell_at_ptr = None + class FakeJitDriverSD: + _green_args_spec = [lltype.Signed, lltype.Float] + _can_inline_ptr = None + _get_printable_location_ptr = None + _confirm_enter_jit_ptr = llhelper(ENTER_JIT, confirm_enter_jit) + _get_jitcell_at_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc()) + state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) state.make_jitdriver_callbacks() res = state.confirm_enter_jit(5, 42.5, 3) assert res is True Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_ztranslation.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_ztranslation.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_ztranslation.py Tue Jul 6 00:00:33 2010 @@ -20,6 +20,7 @@ # - profiler # - full optimizer # - jitdriver hooks + # - two JITs class Frame(object): _virtualizable2_ = ['i'] @@ -61,14 +62,28 @@ frame.i -= 2 frame.i -= 1 return total * 10 - res = ll_meta_interp(f, [40], CPUClass=self.CPUClass, + # + myjitdriver2 = JitDriver(greens = ['g'], reds = ['m', 'x'], + can_inline = lambda *args: False) + def f2(g, m, x): + while m > 0: + myjitdriver2.can_enter_jit(g=g, m=m, x=x) + myjitdriver2.jit_merge_point(g=g, m=m, x=x) + m -= 1 + x += 3 + return x + # + def main(i, j): + return f(i) - f2(i+j, i, j) + res = ll_meta_interp(main, [40, 5], CPUClass=self.CPUClass, type_system=self.type_system) - assert res == f(40) - res = rpython_ll_meta_interp(f, [40], loops=2, CPUClass=self.CPUClass, + assert res == main(40, 5) + res = rpython_ll_meta_interp(main, [40, 5], loops=2, + CPUClass=self.CPUClass, type_system=self.type_system, optimizer=OPTIMIZER_FULL, ProfilerClass=Profiler) - assert res == f(40) + assert res == main(40, 5) def test_external_exception_handling_translates(self): jitdriver = JitDriver(greens = [], reds = ['n', 'total']) Modified: pypy/branch/fast-forward/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/virtualizable.py Tue Jul 6 00:00:33 2010 @@ -14,9 +14,8 @@ TOKEN_NONE = 0 # must be 0 -- see also x86.call_assembler TOKEN_TRACING_RESCALL = -1 - def __init__(self, warmrunnerdesc): + def __init__(self, warmrunnerdesc, VTYPEPTR): self.warmrunnerdesc = warmrunnerdesc - jitdriver = warmrunnerdesc.jitdriver cpu = warmrunnerdesc.cpu if cpu.ts.name == 'ootype': import py @@ -24,11 +23,6 @@ self.cpu = cpu self.BoxArray = cpu.ts.BoxRef # - assert len(jitdriver.virtualizables) == 1 # for now - [vname] = jitdriver.virtualizables - index = len(jitdriver.greens) + jitdriver.reds.index(vname) - self.index_of_virtualizable = index - VTYPEPTR = warmrunnerdesc.JIT_ENTER_FUNCTYPE.ARGS[index] while 'virtualizable2_accessor' not in deref(VTYPEPTR)._hints: VTYPEPTR = cpu.ts.get_superclass(VTYPEPTR) self.VTYPEPTR = VTYPEPTR Modified: pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py Tue Jul 6 00:00:33 2010 @@ -21,6 +21,7 @@ from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler from pypy.jit.metainterp.jitexc import JitException +from pypy.jit.metainterp.jitdriver import JitDriverStaticData from pypy.jit.codewriter import support, codewriter from pypy.jit.codewriter.policy import JitPolicy from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE @@ -44,8 +45,9 @@ no_stats = True, ProfilerClass = ProfilerClass, **kwds) - warmrunnerdesc.state.set_param_inlining(inline) - warmrunnerdesc.state.set_param_debug(debug_level) + for jd in warmrunnerdesc.jitdrivers_sd: + jd.warmstate.set_param_inlining(inline) + jd.warmstate.set_param_debug(debug_level) warmrunnerdesc.finish() translator.warmrunnerdesc = warmrunnerdesc # for later debugging @@ -69,11 +71,12 @@ translator.config.translation.gc = "boehm" translator.config.translation.list_comprehension_operations = True warmrunnerdesc = WarmRunnerDesc(translator, backendopt=backendopt, **kwds) - warmrunnerdesc.state.set_param_threshold(3) # for tests - warmrunnerdesc.state.set_param_trace_eagerness(2) # for tests - warmrunnerdesc.state.set_param_trace_limit(trace_limit) - warmrunnerdesc.state.set_param_inlining(inline) - warmrunnerdesc.state.set_param_debug(debug_level) + for jd in warmrunnerdesc.jitdrivers_sd: + jd.warmstate.set_param_threshold(3) # for tests + jd.warmstate.set_param_trace_eagerness(2) # for tests + jd.warmstate.set_param_trace_limit(trace_limit) + jd.warmstate.set_param_inlining(inline) + jd.warmstate.set_param_debug(debug_level) warmrunnerdesc.finish() res = interp.eval_graph(graph, args) if not kwds.get('translate_support_code', False): @@ -110,12 +113,11 @@ raise Exception("no can_enter_jit found!") return results -def find_jit_merge_point(graphs): +def find_jit_merge_points(graphs): results = _find_jit_marker(graphs, 'jit_merge_point') - if len(results) != 1: - raise Exception("found %d jit_merge_points, need exactly one!" % - (len(results),)) - return results[0] + if not results: + raise Exception("no jit_merge_point found!") + return results def find_set_param(graphs): return _find_jit_marker(graphs, 'set_param') @@ -146,8 +148,8 @@ pyjitpl._warmrunnerdesc = self # this is a global for debugging only! self.set_translator(translator) self.build_cpu(CPUClass, **kwds) - self.find_portal() - self.codewriter = codewriter.CodeWriter(self.cpu, self.portal_graph) + self.find_portals() + self.codewriter = codewriter.CodeWriter(self.cpu, self.jitdrivers_sd) if policy is None: policy = JitPolicy() policy.set_supports_floats(self.cpu.supports_floats) @@ -158,35 +160,31 @@ self.prejit_optimizations(policy, graphs) self.build_meta_interp(ProfilerClass) - self.make_args_specification() + self.make_args_specifications() # from pypy.jit.metainterp.virtualref import VirtualRefInfo vrefinfo = VirtualRefInfo(self) self.codewriter.setup_vrefinfo(vrefinfo) - if self.jitdriver.virtualizables: - from pypy.jit.metainterp.virtualizable import VirtualizableInfo - self.virtualizable_info = VirtualizableInfo(self) - self.codewriter.setup_virtualizable_info(self.virtualizable_info) - else: - self.virtualizable_info = None # + self.make_virtualizable_infos() self.make_exception_classes() self.make_driverhook_graphs() - self.make_enter_function() - self.rewrite_jit_merge_point(policy) + self.make_enter_functions() + self.rewrite_jit_merge_points(policy) verbose = not self.cpu.translate_support_code self.codewriter.make_jitcodes(verbose=verbose) - self.rewrite_can_enter_jit() + self.rewrite_can_enter_jits() self.rewrite_set_param() self.rewrite_force_virtual(vrefinfo) self.add_finish() self.metainterp_sd.finish_setup(self.codewriter, optimizer=optimizer) def finish(self): - vinfo = self.virtualizable_info - if vinfo is not None: - vinfo.finish() + vinfos = set([jd.virtualizable_info for jd in self.jitdrivers_sd]) + for vinfo in vinfos: + if vinfo is not None: + vinfo.finish() if self.cpu.translate_support_code: self.annhelper.finish() @@ -198,18 +196,27 @@ self.rtyper = translator.rtyper self.gcdescr = gc.get_description(translator.config) - def find_portal(self): + def find_portals(self): + self.jitdrivers_sd = [] graphs = self.translator.graphs - self.jit_merge_point_pos = find_jit_merge_point(graphs) - graph, block, pos = self.jit_merge_point_pos + for jit_merge_point_pos in find_jit_merge_points(graphs): + self.split_graph_and_record_jitdriver(*jit_merge_point_pos) + # + assert (len(set([jd.jitdriver for jd in self.jitdrivers_sd])) == + len(self.jitdrivers_sd)), \ + "there are multiple jit_merge_points with the same jitdriver" + + def split_graph_and_record_jitdriver(self, graph, block, pos): + jd = JitDriverStaticData() + jd._jit_merge_point_pos = (graph, block, pos) op = block.operations[pos] args = op.args[2:] s_binding = self.translator.annotator.binding - self.portal_args_s = [s_binding(v) for v in args] + jd._portal_args_s = [s_binding(v) for v in args] graph = copygraph(graph) graph.startblock.isstartblock = False - graph.startblock = support.split_before_jit_merge_point( - *find_jit_merge_point([graph])) + [jmpp] = find_jit_merge_points([graph]) + graph.startblock = support.split_before_jit_merge_point(*jmpp) graph.startblock.isstartblock = True # a crash in the following checkgraph() means that you forgot # to list some variable in greens=[] or reds=[] in JitDriver. @@ -218,12 +225,16 @@ assert isinstance(v, Variable) assert len(dict.fromkeys(graph.getargs())) == len(graph.getargs()) self.translator.graphs.append(graph) - self.portal_graph = graph + jd.portal_graph = graph # it's a bit unbelievable to have a portal without func assert hasattr(graph, "func") graph.func._dont_inline_ = True graph.func._jit_unroll_safe_ = True - self.jitdriver = block.operations[pos].args[1].value + jd.jitdriver = block.operations[pos].args[1].value + jd.portal_runner_ptr = "" + jd.result_type = history.getkind(jd.portal_graph.getreturnvar() + .concretetype)[0] + self.jitdrivers_sd.append(jd) def check_access_directly_sanity(self, graphs): from pypy.translator.backendopt.inline import collect_called_graphs @@ -268,6 +279,27 @@ ProfilerClass=ProfilerClass, warmrunnerdesc=self) + def make_virtualizable_infos(self): + vinfos = {} + for jd in self.jitdrivers_sd: + if not jd.jitdriver.virtualizables: + jd.virtualizable_info = None + jd.index_of_virtualizable = -1 + continue + # + jitdriver = jd.jitdriver + assert len(jitdriver.virtualizables) == 1 # for now + [vname] = jitdriver.virtualizables + # XXX skip the Voids here too + jd.index_of_virtualizable = jitdriver.reds.index(vname) + # + index = jd.num_green_args + jd.index_of_virtualizable + VTYPEPTR = jd._JIT_ENTER_FUNCTYPE.ARGS[index] + if VTYPEPTR not in vinfos: + from pypy.jit.metainterp.virtualizable import VirtualizableInfo + vinfos[VTYPEPTR] = VirtualizableInfo(self, VTYPEPTR) + jd.virtualizable_info = vinfos[VTYPEPTR] + def make_exception_classes(self): class DoneWithThisFrameVoid(JitException): @@ -317,6 +349,8 @@ self.green_int, self.green_ref, self.green_float, self.red_int, self.red_ref, self.red_float) + # XXX there is no point any more to not just have the exceptions + # as globals self.DoneWithThisFrameVoid = DoneWithThisFrameVoid self.DoneWithThisFrameInt = DoneWithThisFrameInt self.DoneWithThisFrameRef = DoneWithThisFrameRef @@ -330,11 +364,15 @@ self.metainterp_sd.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef self.metainterp_sd.ContinueRunningNormally = ContinueRunningNormally - def make_enter_function(self): + def make_enter_functions(self): + for jd in self.jitdrivers_sd: + self.make_enter_function(jd) + + def make_enter_function(self, jd): from pypy.jit.metainterp.warmstate import WarmEnterState - state = WarmEnterState(self) + state = WarmEnterState(self, jd) maybe_compile_and_run = state.make_entry_point() - self.state = state + jd.warmstate = state def crash_in_jit(e): if not we_are_translated(): @@ -359,15 +397,16 @@ def maybe_enter_jit(*args): maybe_compile_and_run(*args) maybe_enter_jit._always_inline_ = True - self.maybe_enter_jit_fn = maybe_enter_jit + jd._maybe_enter_jit_fn = maybe_enter_jit - can_inline = self.state.can_inline_greenargs + can_inline = state.can_inline_greenargs + num_green_args = jd.num_green_args def maybe_enter_from_start(*args): - if can_inline is not None and not can_inline(*args[:self.num_green_args]): + if can_inline is not None and not can_inline(*args[:num_green_args]): maybe_compile_and_run(*args) maybe_enter_from_start._always_inline_ = True - self.maybe_enter_from_start_fn = maybe_enter_from_start - + jd._maybe_enter_from_start_fn = maybe_enter_from_start + def make_driverhook_graphs(self): from pypy.rlib.jit import BaseJitCell bk = self.rtyper.annotator.bookkeeper @@ -378,22 +417,23 @@ s_Str = annmodel.SomeString() # annhelper = MixLevelHelperAnnotator(self.translator.rtyper) - self.set_jitcell_at_ptr = self._make_hook_graph( - annhelper, self.jitdriver.set_jitcell_at, annmodel.s_None, - s_BaseJitCell_not_None) - self.get_jitcell_at_ptr = self._make_hook_graph( - annhelper, self.jitdriver.get_jitcell_at, s_BaseJitCell_or_None) - self.can_inline_ptr = self._make_hook_graph( - annhelper, self.jitdriver.can_inline, annmodel.s_Bool) - self.get_printable_location_ptr = self._make_hook_graph( - annhelper, self.jitdriver.get_printable_location, s_Str) - self.confirm_enter_jit_ptr = self._make_hook_graph( - annhelper, self.jitdriver.confirm_enter_jit, annmodel.s_Bool, - onlygreens=False) + for jd in self.jitdrivers_sd: + jd._set_jitcell_at_ptr = self._make_hook_graph(jd, + annhelper, jd.jitdriver.set_jitcell_at, annmodel.s_None, + s_BaseJitCell_not_None) + jd._get_jitcell_at_ptr = self._make_hook_graph(jd, + annhelper, jd.jitdriver.get_jitcell_at, s_BaseJitCell_or_None) + jd._can_inline_ptr = self._make_hook_graph(jd, + annhelper, jd.jitdriver.can_inline, annmodel.s_Bool) + jd._get_printable_location_ptr = self._make_hook_graph(jd, + annhelper, jd.jitdriver.get_printable_location, s_Str) + jd._confirm_enter_jit_ptr = self._make_hook_graph(jd, + annhelper, jd.jitdriver.confirm_enter_jit, annmodel.s_Bool, + onlygreens=False) annhelper.finish() - def _make_hook_graph(self, annhelper, func, s_result, s_first_arg=None, - onlygreens=True): + def _make_hook_graph(self, jitdriver_sd, annhelper, func, + s_result, s_first_arg=None, onlygreens=True): if func is None: return None # @@ -401,38 +441,57 @@ if s_first_arg is not None: extra_args_s.append(s_first_arg) # - args_s = self.portal_args_s + args_s = jitdriver_sd._portal_args_s if onlygreens: - args_s = args_s[:len(self.green_args_spec)] + args_s = args_s[:len(jitdriver_sd._green_args_spec)] graph = annhelper.getgraph(func, extra_args_s + args_s, s_result) funcptr = annhelper.graph2delayed(graph) return funcptr - def make_args_specification(self): - graph, block, index = self.jit_merge_point_pos + def make_args_specifications(self): + for jd in self.jitdrivers_sd: + self.make_args_specification(jd) + + def make_args_specification(self, jd): + graph, block, index = jd._jit_merge_point_pos op = block.operations[index] greens_v, reds_v = support.decode_hp_hint_args(op) ALLARGS = [v.concretetype for v in (greens_v + reds_v)] - self.green_args_spec = [v.concretetype for v in greens_v] - self.red_args_types = [history.getkind(v.concretetype) for v in reds_v] - self.num_green_args = len(self.green_args_spec) + jd._green_args_spec = [v.concretetype for v in greens_v] + jd._red_args_types = [history.getkind(v.concretetype) for v in reds_v] + jd.num_green_args = len(jd._green_args_spec) RESTYPE = graph.getreturnvar().concretetype - (self.JIT_ENTER_FUNCTYPE, - self.PTR_JIT_ENTER_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, lltype.Void) - (self.PORTAL_FUNCTYPE, - self.PTR_PORTAL_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, RESTYPE) - (_, self.PTR_ASSEMBLER_HELPER_FUNCTYPE) = self.cpu.ts.get_FuncType( + (jd._JIT_ENTER_FUNCTYPE, + jd._PTR_JIT_ENTER_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, lltype.Void) + (jd._PORTAL_FUNCTYPE, + jd._PTR_PORTAL_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, RESTYPE) + (_, jd._PTR_ASSEMBLER_HELPER_FUNCTYPE) = self.cpu.ts.get_FuncType( [lltype.Signed, llmemory.GCREF], RESTYPE) - def rewrite_can_enter_jit(self): - FUNC = self.JIT_ENTER_FUNCTYPE - FUNCPTR = self.PTR_JIT_ENTER_FUNCTYPE - jit_enter_fnptr = self.helper_func(FUNCPTR, self.maybe_enter_jit_fn) + def rewrite_can_enter_jits(self): + can_enter_jits = find_can_enter_jit(self.translator.graphs) + sublists = {} + for jd in self.jitdrivers_sd: + sublists[jd.jitdriver] = [] + for graph, block, index in can_enter_jits: + op = block.operations[index] + jitdriver = op.args[1].value + assert jitdriver in sublists, \ + "can_enter_jit with no matching jit_merge_point" + sublists[jitdriver].append((graph, block, index)) + for jd in self.jitdrivers_sd: + sublist = sublists[jd.jitdriver] + assert len(sublist) > 0, \ + "found no can_enter_jit for %r" % (jd.jitdriver,) + self.rewrite_can_enter_jit(jd, sublist) + + def rewrite_can_enter_jit(self, jd, can_enter_jits): + FUNC = jd._JIT_ENTER_FUNCTYPE + FUNCPTR = jd._PTR_JIT_ENTER_FUNCTYPE + jit_enter_fnptr = self.helper_func(FUNCPTR, jd._maybe_enter_jit_fn) - graphs = self.translator.graphs - can_enter_jits = find_can_enter_jit(graphs) for graph, block, index in can_enter_jits: - if graph is self.jit_merge_point_pos[0]: + if graph is jd._jit_merge_point_pos[0]: continue op = block.operations[index] @@ -455,7 +514,11 @@ graph = self.annhelper.getgraph(func, args_s, s_result) return self.annhelper.graph2delayed(graph, FUNC) - def rewrite_jit_merge_point(self, policy): + def rewrite_jit_merge_points(self, policy): + for jd in self.jitdrivers_sd: + self.rewrite_jit_merge_point(jd, policy) + + def rewrite_jit_merge_point(self, jd, policy): # # Mutate the original portal graph from this: # @@ -486,9 +549,9 @@ # while 1: # more stuff # - origportalgraph = self.jit_merge_point_pos[0] - portalgraph = self.portal_graph - PORTALFUNC = self.PORTAL_FUNCTYPE + origportalgraph = jd._jit_merge_point_pos[0] + portalgraph = jd.portal_graph + PORTALFUNC = jd._PORTAL_FUNCTYPE # ____________________________________________________________ # Prepare the portal_runner() helper @@ -496,12 +559,12 @@ from pypy.jit.metainterp.warmstate import specialize_value portal_ptr = self.cpu.ts.functionptr(PORTALFUNC, 'portal', graph = portalgraph) - self.portal_ptr = portal_ptr + jd._portal_ptr = portal_ptr # portalfunc_ARGS = [] nums = {} for i, ARG in enumerate(PORTALFUNC.ARGS): - if i < len(self.jitdriver.greens): + if i < len(jd.jitdriver.greens): color = 'green' else: color = 'red' @@ -519,7 +582,7 @@ def ll_portal_runner(*args): while 1: try: - self.maybe_enter_from_start_fn(*args) + jd._maybe_enter_from_start_fn(*args) return support.maybe_on_top_of_llinterp(rtyper, portal_ptr)(*args) except self.ContinueRunningNormally, e: @@ -548,27 +611,26 @@ value = cast_base_ptr_to_instance(Exception, value) raise Exception, value - self.ll_portal_runner = ll_portal_runner # for debugging - self.portal_runner_ptr = self.helper_func(self.PTR_PORTAL_FUNCTYPE, - ll_portal_runner) + jd._ll_portal_runner = ll_portal_runner # for debugging + jd.portal_runner_ptr = self.helper_func(jd._PTR_PORTAL_FUNCTYPE, + ll_portal_runner) + jd.portal_runner_adr = llmemory.cast_ptr_to_adr(jd.portal_runner_ptr) self.cpu.portal_calldescr = self.cpu.calldescrof( - self.PTR_PORTAL_FUNCTYPE.TO, - self.PTR_PORTAL_FUNCTYPE.TO.ARGS, - self.PTR_PORTAL_FUNCTYPE.TO.RESULT) - self.codewriter.setup_portal_runner_ptr(self.portal_runner_ptr) + jd._PTR_PORTAL_FUNCTYPE.TO, + jd._PTR_PORTAL_FUNCTYPE.TO.ARGS, + jd._PTR_PORTAL_FUNCTYPE.TO.RESULT) - vinfo = self.virtualizable_info + vinfo = jd.virtualizable_info def assembler_call_helper(failindex, virtualizableref): fail_descr = self.cpu.get_fail_descr_from_number(failindex) while True: + if vinfo is not None: + virtualizable = lltype.cast_opaque_ptr( + vinfo.VTYPEPTR, virtualizableref) + vinfo.reset_vable_token(virtualizable) try: - if vinfo is not None: - virtualizable = lltype.cast_opaque_ptr( - vinfo.VTYPEPTR, virtualizableref) - vinfo.reset_vable_token(virtualizable) - loop_token = fail_descr.handle_fail(self.metainterp_sd) - fail_descr = self.cpu.execute_token(loop_token) + loop_token = fail_descr.handle_fail(self.metainterp_sd, jd) except self.ContinueRunningNormally, e: args = () for ARGTYPE, attrname, count in portalfunc_ARGS: @@ -595,29 +657,26 @@ else: value = cast_base_ptr_to_instance(Exception, value) raise Exception, value + fail_descr = self.cpu.execute_token(loop_token) - self.assembler_call_helper = assembler_call_helper # for debugging - self.cpu.assembler_helper_ptr = self.helper_func( - self.PTR_ASSEMBLER_HELPER_FUNCTYPE, + jd._assembler_call_helper = assembler_call_helper # for debugging + jd._assembler_helper_ptr = self.helper_func( + jd._PTR_ASSEMBLER_HELPER_FUNCTYPE, assembler_call_helper) - # XXX a bit ugly sticking + jd.assembler_helper_adr = llmemory.cast_ptr_to_adr( + jd._assembler_helper_ptr) if vinfo is not None: - self.cpu.index_of_virtualizable = (vinfo.index_of_virtualizable - - self.num_green_args) - self.cpu.vable_token_descr = vinfo.vable_token_descr - else: - self.cpu.index_of_virtualizable = -1 - self.cpu.vable_token_descr = None + jd.vable_token_descr = vinfo.vable_token_descr # ____________________________________________________________ # Now mutate origportalgraph to end with a call to portal_runner_ptr # - _, origblock, origindex = self.jit_merge_point_pos + _, origblock, origindex = jd._jit_merge_point_pos op = origblock.operations[origindex] assert op.opname == 'jit_marker' assert op.args[0].value == 'jit_merge_point' greens_v, reds_v = support.decode_hp_hint_args(op) - vlist = [Constant(self.portal_runner_ptr, self.PTR_PORTAL_FUNCTYPE)] + vlist = [Constant(jd.portal_runner_ptr, jd._PTR_PORTAL_FUNCTYPE)] vlist += greens_v vlist += reds_v v_result = Variable() @@ -627,6 +686,18 @@ origblock.operations.append(newop) origblock.exitswitch = None origblock.recloseblock(Link([v_result], origportalgraph.returnblock)) + # + # Also kill any can_enter_jit left behind (example: see + # test_jitdriver.test_simple, which has a can_enter_jit in + # loop1's origportalgraph) + can_enter_jits = _find_jit_marker([origportalgraph], 'can_enter_jit') + for _, block, i in can_enter_jits: + op = block.operations[i] + assert op.opname == 'jit_marker' + block.operations[i] = SpaceOperation('same_as', + [Constant(None, lltype.Void)], + op.result) + # checkgraph(origportalgraph) def add_finish(self): @@ -644,8 +715,8 @@ graphs = self.translator.graphs _, PTR_SET_PARAM_FUNCTYPE = self.cpu.ts.get_FuncType([lltype.Signed], lltype.Void) - def make_closure(fullfuncname): - state = self.state + def make_closure(jd, fullfuncname): + state = jd.warmstate def closure(i): getattr(state, fullfuncname)(i) funcptr = self.helper_func(PTR_SET_PARAM_FUNCTYPE, closure) @@ -653,12 +724,17 @@ # for graph, block, i in find_set_param(graphs): op = block.operations[i] - assert op.args[1].value == self.jitdriver + for jd in self.jitdrivers_sd: + if jd.jitdriver is op.args[1].value: + break + else: + assert 0, "jitdriver of set_param() not found" funcname = op.args[2].value - if funcname not in closures: - closures[funcname] = make_closure('set_param_' + funcname) + key = jd, funcname + if key not in closures: + closures[key] = make_closure(jd, 'set_param_' + funcname) op.opname = 'direct_call' - op.args[:3] = [closures[funcname]] + op.args[:3] = [closures[key]] def rewrite_force_virtual(self, vrefinfo): if self.cpu.ts.name != 'lltype': Modified: pypy/branch/fast-forward/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/warmstate.py Tue Jul 6 00:00:33 2010 @@ -1,7 +1,7 @@ import sys from pypy.rpython.lltypesystem import lltype, llmemory, rstr from pypy.rpython.ootypesystem import ootype -from pypy.rpython.annlowlevel import hlstr, cast_base_ptr_to_instance +from pypy.rpython.annlowlevel import hlstr, llstr, cast_base_ptr_to_instance from pypy.rpython.annlowlevel import cast_object_to_ptr from pypy.rlib.objectmodel import specialize, we_are_translated, r_dict from pypy.rlib.rarithmetic import intmask @@ -120,6 +120,16 @@ else: assert False +class JitCell(BaseJitCell): + # the counter can mean the following things: + # counter >= 0: not yet traced, wait till threshold is reached + # counter == -1: there is an entry bridge for this cell + # counter == -2: tracing is currently going on for this cell + counter = 0 + compiled_merge_points = None + dont_trace_here = False + entry_loop_token = None + # ____________________________________________________________ @@ -127,9 +137,10 @@ THRESHOLD_LIMIT = sys.maxint // 2 default_jitcell_dict = None - def __init__(self, warmrunnerdesc): + def __init__(self, warmrunnerdesc, jitdriver_sd): "NOT_RPYTHON" self.warmrunnerdesc = warmrunnerdesc + self.jitdriver_sd = jitdriver_sd try: self.profiler = warmrunnerdesc.metainterp_sd.profiler except AttributeError: # for tests @@ -176,8 +187,7 @@ cell = self.jit_cell_at_key(greenkey) cell.dont_trace_here = True debug_start("jit-disableinlining") - sd = self.warmrunnerdesc.metainterp_sd - loc = sd.state.get_location_str(greenkey) + loc = self.get_location_str(greenkey) debug_print("disabled inlining", loc) debug_stop("jit-disableinlining") @@ -195,8 +205,10 @@ return self.maybe_compile_and_run metainterp_sd = self.warmrunnerdesc.metainterp_sd - vinfo = self.warmrunnerdesc.virtualizable_info - num_green_args = self.warmrunnerdesc.num_green_args + jitdriver_sd = self.jitdriver_sd + vinfo = jitdriver_sd.virtualizable_info + index_of_virtualizable = jitdriver_sd.index_of_virtualizable + num_green_args = jitdriver_sd.num_green_args get_jitcell = self.make_jitcell_getter() set_future_values = self.make_set_future_values() self.make_jitdriver_callbacks() @@ -206,7 +218,6 @@ """Entry point to the JIT. Called at the point with the can_enter_jit() hint. """ - globaldata = metainterp_sd.globaldata if NonConstant(False): # make sure we always see the saner optimizer from an # annotation point of view, otherwise we get lots of @@ -214,7 +225,7 @@ self.set_param_optimizer(OPTIMIZER_FULL) if vinfo is not None: - virtualizable = args[vinfo.index_of_virtualizable] + virtualizable = args[num_green_args + index_of_virtualizable] virtualizable = vinfo.cast_to_vtype(virtualizable) else: virtualizable = None @@ -234,11 +245,12 @@ return # bound reached; start tracing from pypy.jit.metainterp.pyjitpl import MetaInterp - metainterp = MetaInterp(metainterp_sd) + metainterp = MetaInterp(metainterp_sd, jitdriver_sd) # set counter to -2, to mean "tracing in effect" cell.counter = -2 try: - loop_token = metainterp.compile_and_run_once(*args) + loop_token = metainterp.compile_and_run_once(jitdriver_sd, + *args) finally: if cell.counter == -2: cell.counter = 0 @@ -264,7 +276,8 @@ metainterp_sd.profiler.end_running() if vinfo is not None: vinfo.reset_vable_token(virtualizable) - loop_token = fail_descr.handle_fail(metainterp_sd) + loop_token = fail_descr.handle_fail(metainterp_sd, + jitdriver_sd) maybe_compile_and_run._dont_inline_ = True self.maybe_compile_and_run = maybe_compile_and_run @@ -277,8 +290,8 @@ if hasattr(self, 'unwrap_greenkey'): return self.unwrap_greenkey # - warmrunnerdesc = self.warmrunnerdesc - green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec) + jitdriver_sd = self.jitdriver_sd + green_args_spec = unrolling_iterable(jitdriver_sd._green_args_spec) # def unwrap_greenkey(greenkey): greenargs = () @@ -302,20 +315,10 @@ if hasattr(self, 'jit_getter'): return self.jit_getter # - class JitCell(BaseJitCell): - # the counter can mean the following things: - # counter >= 0: not yet traced, wait till threshold is reached - # counter == -1: there is an entry bridge for this cell - # counter == -2: tracing is currently going on for this cell - counter = 0 - compiled_merge_points = None - dont_trace_here = False - entry_loop_token = None - # - if self.warmrunnerdesc.get_jitcell_at_ptr is None: - jit_getter = self._make_jitcell_getter_default(JitCell) + if self.jitdriver_sd._get_jitcell_at_ptr is None: + jit_getter = self._make_jitcell_getter_default() else: - jit_getter = self._make_jitcell_getter_custom(JitCell) + jit_getter = self._make_jitcell_getter_custom() # unwrap_greenkey = self.make_unwrap_greenkey() # @@ -327,10 +330,10 @@ # return jit_getter - def _make_jitcell_getter_default(self, JitCell): + def _make_jitcell_getter_default(self): "NOT_RPYTHON" - warmrunnerdesc = self.warmrunnerdesc - green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec) + jitdriver_sd = self.jitdriver_sd + green_args_spec = unrolling_iterable(jitdriver_sd._green_args_spec) # def comparekey(greenargs1, greenargs2): i = 0 @@ -361,11 +364,11 @@ return cell return get_jitcell - def _make_jitcell_getter_custom(self, JitCell): + def _make_jitcell_getter_custom(self): "NOT_RPYTHON" rtyper = self.warmrunnerdesc.rtyper - get_jitcell_at_ptr = self.warmrunnerdesc.get_jitcell_at_ptr - set_jitcell_at_ptr = self.warmrunnerdesc.set_jitcell_at_ptr + get_jitcell_at_ptr = self.jitdriver_sd._get_jitcell_at_ptr + set_jitcell_at_ptr = self.jitdriver_sd._set_jitcell_at_ptr lltohlhack = {} # def get_jitcell(*greenargs): @@ -415,9 +418,10 @@ return self.set_future_values warmrunnerdesc = self.warmrunnerdesc + jitdriver_sd = self.jitdriver_sd cpu = warmrunnerdesc.cpu - vinfo = warmrunnerdesc.virtualizable_info - red_args_types = unrolling_iterable(warmrunnerdesc.red_args_types) + vinfo = jitdriver_sd.virtualizable_info + red_args_types = unrolling_iterable(jitdriver_sd._red_args_types) # def set_future_values(*redargs): i = 0 @@ -428,8 +432,9 @@ set_future_values_from_vinfo(*redargs) # if vinfo is not None: - i0 = len(warmrunnerdesc.red_args_types) - num_green_args = warmrunnerdesc.num_green_args + i0 = len(jitdriver_sd._red_args_types) + num_green_args = jitdriver_sd.num_green_args + index_of_virtualizable = jitdriver_sd.index_of_virtualizable vable_static_fields = unrolling_iterable( zip(vinfo.static_extra_types, vinfo.static_fields)) vable_array_fields = unrolling_iterable( @@ -439,8 +444,7 @@ # def set_future_values_from_vinfo(*redargs): i = i0 - virtualizable = redargs[vinfo.index_of_virtualizable - - num_green_args] + virtualizable = redargs[index_of_virtualizable] virtualizable = vinfo.cast_to_vtype(virtualizable) for typecode, fieldname in vable_static_fields: x = getattr(virtualizable, fieldname) @@ -464,8 +468,9 @@ if hasattr(self, 'get_location_str'): return # - can_inline_ptr = self.warmrunnerdesc.can_inline_ptr + can_inline_ptr = self.jitdriver_sd._can_inline_ptr unwrap_greenkey = self.make_unwrap_greenkey() + jit_getter = self.make_jitcell_getter() if can_inline_ptr is None: def can_inline_callable(*greenargs): # XXX shouldn't it be False by default? @@ -477,7 +482,7 @@ fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) return fn(*greenargs) def can_inline(*greenargs): - cell = self.jit_getter(*greenargs) + cell = jit_getter(*greenargs) if cell.dont_trace_here: return False return can_inline_callable(*greenargs) @@ -486,21 +491,25 @@ greenargs = unwrap_greenkey(greenkey) return can_inline(*greenargs) self.can_inline_callable = can_inline_greenkey - - get_jitcell = self.make_jitcell_getter() + def get_assembler_token(greenkey): greenargs = unwrap_greenkey(greenkey) - cell = get_jitcell(*greenargs) + cell = jit_getter(*greenargs) if cell.counter >= 0: return None return cell.entry_loop_token self.get_assembler_token = get_assembler_token # - get_location_ptr = self.warmrunnerdesc.get_printable_location_ptr + get_location_ptr = self.jitdriver_sd._get_printable_location_ptr if get_location_ptr is None: + missing = '(no jitdriver.get_printable_location!)' + missingll = llstr(missing) def get_location_str(greenkey): - return '(no jitdriver.get_printable_location!)' + if we_are_translated(): + return missingll + else: + return missing else: rtyper = self.warmrunnerdesc.rtyper unwrap_greenkey = self.make_unwrap_greenkey() @@ -514,7 +523,7 @@ return res self.get_location_str = get_location_str # - confirm_enter_jit_ptr = self.warmrunnerdesc.confirm_enter_jit_ptr + confirm_enter_jit_ptr = self.jitdriver_sd._confirm_enter_jit_ptr if confirm_enter_jit_ptr is None: def confirm_enter_jit(*args): return True Modified: pypy/branch/fast-forward/pypy/jit/tl/spli/targetspli.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/tl/spli/targetspli.py (original) +++ pypy/branch/fast-forward/pypy/jit/tl/spli/targetspli.py Tue Jul 6 00:00:33 2010 @@ -31,7 +31,7 @@ def jitpolicy(driver): """Returns the JIT policy to use when translating.""" - from pypy.jit.metainterp.policy import JitPolicy + from pypy.jit.codewriter.policy import JitPolicy return JitPolicy() if __name__ == '__main__': Modified: pypy/branch/fast-forward/pypy/jit/tl/targettlc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/tl/targettlc.py (original) +++ pypy/branch/fast-forward/pypy/jit/tl/targettlc.py Tue Jul 6 00:00:33 2010 @@ -2,7 +2,7 @@ import py py.path.local(__file__) from pypy.jit.tl.tlc import interp, interp_nonjit, ConstantPool -from pypy.jit.metainterp.policy import JitPolicy +from pypy.jit.codewriter.policy import JitPolicy from pypy.jit.backend.hlinfo import highleveljitinfo Modified: pypy/branch/fast-forward/pypy/jit/tl/targettlr.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/tl/targettlr.py (original) +++ pypy/branch/fast-forward/pypy/jit/tl/targettlr.py Tue Jul 6 00:00:33 2010 @@ -33,7 +33,7 @@ # ____________________________________________________________ -from pypy.jit.metainterp.policy import JitPolicy +from pypy.jit.codewriter.policy import JitPolicy def jitpolicy(driver): return JitPolicy() Modified: pypy/branch/fast-forward/pypy/jit/tl/tinyframe/targettinyframe.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/tl/tinyframe/targettinyframe.py (original) +++ pypy/branch/fast-forward/pypy/jit/tl/tinyframe/targettinyframe.py Tue Jul 6 00:00:33 2010 @@ -1,6 +1,6 @@ from pypy.jit.tl.tinyframe.tinyframe import main -from pypy.jit.metainterp.policy import JitPolicy +from pypy.jit.codewriter.policy import JitPolicy def jitpolicy(driver): return JitPolicy() Modified: pypy/branch/fast-forward/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/__builtin__/functional.py (original) +++ pypy/branch/fast-forward/pypy/module/__builtin__/functional.py Tue Jul 6 00:00:33 2010 @@ -176,10 +176,75 @@ if not collections_w: msg = "map() requires at least two arguments" raise OperationError(space.w_TypeError, space.wrap(msg)) - num_collections = len(collections_w) none_func = space.is_w(w_func, space.w_None) - if none_func and num_collections == 1: - return space.call_function(space.w_list, collections_w[0]) + if len(collections_w) == 1: + w_collection = collections_w[0] + if none_func: + result_w = space.unpackiterable(w_collection) + else: + result_w = map_single_collection(space, w_func, w_collection) + else: + result_w = map_multiple_collections(space, w_func, collections_w, + none_func) + return space.newlist(result_w) +map.unwrap_spec = [ObjSpace, W_Root, "args_w"] + +def map_single_collection(space, w_func, w_collection): + """Special case for 'map(func, coll)', where 'func' is not None and there + is only one 'coll' argument.""" + w_iter = space.iter(w_collection) + # xxx special hacks for speed + from pypy.interpreter import function, pycode + if isinstance(w_func, function.Function): + # xxx compatibility issue: what if func_code is modified in the + # middle of running map()?? That's far too obscure for me to care... + code = w_func.getcode() + fast_natural_arity = code.fast_natural_arity + if fast_natural_arity == (1|pycode.PyCode.FLATPYCALL): + assert isinstance(code, pycode.PyCode) + return map_single_user_function(code, w_func, w_iter) + # /xxx end of special hacks + return map_single_other_callable(space, w_func, w_iter) + +def map_single_other_callable(space, w_func, w_iter): + result_w = [] + while True: + try: + w_item = space.next(w_iter) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break + result_w.append(space.call_function(w_func, w_item)) + return result_w +map_single_other_callable._dont_inline_ = True + +from pypy.rlib.jit import JitDriver +mapjitdriver = JitDriver(greens = ['code'], + reds = ['w_func', 'w_iter', 'result_w'], + can_inline = lambda *args: False) +def map_single_user_function(code, w_func, w_iter): + result_w = [] + while True: + mapjitdriver.can_enter_jit(code=code, w_func=w_func, + w_iter=w_iter, result_w=result_w) + mapjitdriver.jit_merge_point(code=code, w_func=w_func, + w_iter=w_iter, result_w=result_w) + space = w_func.space + try: + w_item = space.next(w_iter) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break + new_frame = space.createframe(code, w_func.w_func_globals, + w_func.closure) + new_frame.fastlocals_w[0] = w_item + w_res = new_frame.run() + result_w.append(w_res) + return result_w + +def map_multiple_collections(space, w_func, collections_w, none_func): result_w = [] iterators_w = [space.iter(w_seq) for w_seq in collections_w] num_iterators = len(iterators_w) @@ -196,16 +261,15 @@ iterators_w[i] = None else: cont = True - if cont: - w_args = space.newtuple(args_w) - if none_func: - result_w.append(w_args) - else: - w_res = space.call(w_func, w_args) - result_w.append(w_res) + if not cont: + break + w_args = space.newtuple(args_w) + if none_func: + w_res = w_args else: - return space.newlist(result_w) -map.unwrap_spec = [ObjSpace, W_Root, "args_w"] + w_res = space.call(w_func, w_args) + result_w.append(w_res) + return result_w def sum(space, w_sequence, w_start=None): if space.is_w(w_start, space.w_None): Modified: pypy/branch/fast-forward/pypy/module/__builtin__/test/test_functional.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/__builtin__/test/test_functional.py (original) +++ pypy/branch/fast-forward/pypy/module/__builtin__/test/test_functional.py Tue Jul 6 00:00:33 2010 @@ -6,6 +6,9 @@ def test_trivial_map_one_seq(self): assert map(lambda x: x+2, [1, 2, 3, 4]) == [3, 4, 5, 6] + def test_trivial_map_one_seq_2(self): + assert map(str, [1, 2, 3, 4]) == ['1', '2', '3', '4'] + def test_trivial_map_two_seq(self): assert map(lambda x,y: x+y, [1, 2, 3, 4],[1, 2, 3, 4]) == ( Modified: pypy/branch/fast-forward/pypy/module/_codecs/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_codecs/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/_codecs/__init__.py Tue Jul 6 00:00:33 2010 @@ -3,22 +3,43 @@ from pypy.module._codecs import interp_codecs class Module(MixedModule): - appleveldefs = { - '__doc__' : 'app_codecs.__doc__', - '__name__' : 'app_codecs.__name__', - 'charmap_encode' : 'app_codecs.charmap_encode', - 'escape_decode' : 'app_codecs.escape_decode', - 'escape_encode' : 'app_codecs.escape_encode', - 'raw_unicode_escape_decode' : 'app_codecs.raw_unicode_escape_decode', - 'raw_unicode_escape_encode' : 'app_codecs.raw_unicode_escape_encode', - 'unicode_escape_decode' : 'app_codecs.unicode_escape_decode', - 'unicode_escape_encode' : 'app_codecs.unicode_escape_encode', - 'unicode_internal_decode' : 'app_codecs.unicode_internal_decode', - 'unicode_internal_encode' : 'app_codecs.unicode_internal_encode', - 'utf_7_decode' : 'app_codecs.utf_7_decode', - 'utf_7_encode' : 'app_codecs.utf_7_encode', - 'charmap_build' : 'app_codecs.charmap_build' - } + """ + _codecs -- Provides access to the codec registry and the builtin + codecs. + + This module should never be imported directly. The standard library + module "codecs" wraps this builtin module for use within Python. + + The codec registry is accessible via: + + register(search_function) -> None + + lookup(encoding) -> (encoder, decoder, stream_reader, stream_writer) + + The builtin Unicode codecs use the following interface: + + _encode(Unicode_object[,errors='strict']) -> + (string object, bytes consumed) + + _decode(char_buffer_obj[,errors='strict']) -> + (Unicode object, bytes consumed) + + _encode() interfaces also accept non-Unicode object as + input. The objects are then converted to Unicode using + PyUnicode_FromObject() prior to applying the conversion. + + These s are available: utf_8, unicode_escape, + raw_unicode_escape, unicode_internal, latin_1, ascii (7-bit), + mbcs (on win32). + + +Written by Marc-Andre Lemburg (mal at lemburg.com). + +Copyright (c) Corporation for National Research Initiatives. +""" + + appleveldefs = {} + interpleveldefs = { 'encode': 'interp_codecs.encode', 'decode': 'interp_codecs.decode', @@ -26,12 +47,15 @@ 'lookup_error': 'interp_codecs.lookup_error', 'register': 'interp_codecs.register_codec', 'register_error': 'interp_codecs.register_error', + 'charmap_build' : 'interp_codecs.charmap_build', # encoders and decoders 'ascii_decode' : 'interp_codecs.ascii_decode', 'ascii_encode' : 'interp_codecs.ascii_encode', 'latin_1_decode' : 'interp_codecs.latin_1_decode', 'latin_1_encode' : 'interp_codecs.latin_1_encode', + 'utf_7_decode' : 'interp_codecs.utf_7_decode', + 'utf_7_encode' : 'interp_codecs.utf_7_encode', 'utf_8_decode' : 'interp_codecs.utf_8_decode', 'utf_8_encode' : 'interp_codecs.utf_8_encode', 'utf_16_be_decode' : 'interp_codecs.utf_16_be_decode', @@ -44,6 +68,15 @@ 'charbuffer_encode': 'interp_codecs.buffer_encode', 'readbuffer_encode': 'interp_codecs.buffer_encode', 'charmap_decode' : 'interp_codecs.charmap_decode', + 'charmap_encode' : 'interp_codecs.charmap_encode', + 'escape_encode' : 'interp_codecs.escape_encode', + 'escape_decode' : 'interp_codecs.escape_decode', + 'unicode_escape_decode' : 'interp_codecs.unicode_escape_decode', + 'unicode_escape_encode' : 'interp_codecs.unicode_escape_encode', + 'raw_unicode_escape_decode' : 'interp_codecs.raw_unicode_escape_decode', + 'raw_unicode_escape_encode' : 'interp_codecs.raw_unicode_escape_encode', + 'unicode_internal_decode' : 'interp_codecs.unicode_internal_decode', + 'unicode_internal_encode' : 'interp_codecs.unicode_internal_encode', } def __init__(self, space, *args): Modified: pypy/branch/fast-forward/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/fast-forward/pypy/module/_codecs/interp_codecs.py Tue Jul 6 00:00:33 2010 @@ -1,5 +1,6 @@ from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import ObjSpace, NoneNotWrapped, applevel +from pypy.interpreter.gateway import ObjSpace, NoneNotWrapped, interp2app +from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.baseobjspace import W_Root from pypy.rlib.rstring import StringBuilder, UnicodeBuilder from pypy.rlib.objectmodel import we_are_translated @@ -13,6 +14,8 @@ self.decode_error_handler = self.make_errorhandler(space, True) self.encode_error_handler = self.make_errorhandler(space, False) + self.unicodedata_handler = None + def make_errorhandler(self, space, decode): def unicode_call_errorhandler(errors, encoding, reason, input, startpos, endpos): @@ -53,6 +56,21 @@ return replace, newpos return unicode_call_errorhandler + def get_unicodedata_handler(self, space): + if self.unicodedata_handler: + return self.unicodedata_handler + try: + w_builtin = space.getbuiltinmodule('__builtin__') + w_import = space.getattr(w_builtin, space.wrap("__import__")) + w_unicodedata = space.call_function(w_import, + space.wrap("unicodedata")) + w_getcode = space.getattr(w_unicodedata, space.wrap("_get_code")) + except OperationError: + return None + else: + self.unicodedata_handler = UnicodeData_Handler(space, w_getcode) + return self.unicodedata_handler + def _freeze_(self): assert not self.codec_search_path return False @@ -114,78 +132,125 @@ "unknown encoding: %s", encoding) lookup_codec.unwrap_spec = [ObjSpace, str] -app_errors = applevel(""" -def check_exception(exc): +# ____________________________________________________________ +# Register standard error handlers + +def check_exception(space, w_exc): try: - delta = exc.end - exc.start - if delta < 0 or not isinstance(exc.object, (unicode, str)): - raise TypeError("wrong exception") - except AttributeError: - raise TypeError("wrong exception") - -def strict_errors(exc): - if isinstance(exc, Exception): - raise exc - else: - raise TypeError("codec must pass exception instance") - -def ignore_errors(exc): - check_exception(exc) - if isinstance(exc, UnicodeEncodeError): - return u'', exc.end - elif isinstance(exc, (UnicodeDecodeError, UnicodeTranslateError)): - return u'', exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%exc) - -Py_UNICODE_REPLACEMENT_CHARACTER = u"\ufffd" - -def replace_errors(exc): - check_exception(exc) - if isinstance(exc, UnicodeEncodeError): - return u'?'*(exc.end-exc.start), exc.end - elif isinstance(exc, (UnicodeTranslateError, UnicodeDecodeError)): - return Py_UNICODE_REPLACEMENT_CHARACTER*(exc.end-exc.start), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%exc) - -def xmlcharrefreplace_errors(exc): - if isinstance(exc, UnicodeEncodeError): - res = [] - for ch in exc.object[exc.start:exc.end]: - res += '&#' - res += str(ord(ch)) - res += ';' - return u''.join(res), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) - -def backslashreplace_errors(exc): - if isinstance(exc, UnicodeEncodeError): - p = [] - for c in exc.object[exc.start:exc.end]: - p += '\\\\' - oc = ord(c) - if (oc >= 0x00010000): - p += 'U' - p += "%.8x" % ord(c) + w_start = space.getattr(w_exc, space.wrap('start')) + w_end = space.getattr(w_exc, space.wrap('end')) + w_obj = space.getattr(w_exc, space.wrap('object')) + except OperationError, e: + if not e.match(space, space.w_AttributeError): + raise + raise OperationError(space.w_TypeError, space.wrap( + "wrong exception")) + + delta = space.int_w(w_end) - space.int_w(w_start) + if delta < 0 or not (space.isinstance_w(w_obj, space.w_str) or + space.isinstance_w(w_obj, space.w_unicode)): + raise OperationError(space.w_TypeError, space.wrap( + "wrong exception")) + +def strict_errors(space, w_exc): + check_exception(space, w_exc) + if space.isinstance_w(w_exc, space.w_BaseException): + raise OperationError(space.type(w_exc), w_exc) + else: + raise OperationError(space.w_TypeError, space.wrap( + "codec must pass exception instance")) + +def ignore_errors(space, w_exc): + check_exception(space, w_exc) + w_end = space.getattr(w_exc, space.wrap('end')) + if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): + return space.newtuple([space.wrap(''), w_end]) + elif (space.isinstance_w(w_exc, space.w_UnicodeDecodeError) or + space.isinstance_w(w_exc, space.w_UnicodeTranslateError)): + return space.newtuple([space.wrap(u''), w_end]) + else: + typename = space.type(w_exc).getname(space, '?') + raise operationerrfmt(space.w_TypeError, + "don't know how to handle %s in error callback", typename) + +def replace_errors(space, w_exc): + check_exception(space, w_exc) + w_start = space.getattr(w_exc, space.wrap('start')) + w_end = space.getattr(w_exc, space.wrap('end')) + size = space.int_w(w_end) - space.int_w(w_start) + if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): + text = '?' * size + return space.newtuple([space.wrap(text), w_end]) + elif (space.isinstance_w(w_exc, space.w_UnicodeDecodeError) or + space.isinstance_w(w_exc, space.w_UnicodeTranslateError)): + text = u'\ufffd' * size + return space.newtuple([space.wrap(text), w_end]) + else: + typename = space.type(w_exc).getname(space, '?') + raise operationerrfmt(space.w_TypeError, + "don't know how to handle %s in error callback", typename) + +def xmlcharrefreplace_errors(space, w_exc): + check_exception(space, w_exc) + if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): + obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object'))) + start = space.int_w(space.getattr(w_exc, space.wrap('start'))) + w_end = space.getattr(w_exc, space.wrap('end')) + end = space.int_w(w_end) + builder = UnicodeBuilder() + pos = start + while pos < end: + ch = obj[pos] + builder.append(u"&#") + builder.append(unicode(str(ord(ch)))) + builder.append(u";") + pos += 1 + return space.newtuple([space.wrap(builder.build()), w_end]) + else: + typename = space.type(w_exc).getname(space, '?') + raise operationerrfmt(space.w_TypeError, + "don't know how to handle %s in error callback", typename) + +def backslashreplace_errors(space, w_exc): + check_exception(space, w_exc) + if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): + obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object'))) + start = space.int_w(space.getattr(w_exc, space.wrap('start'))) + w_end = space.getattr(w_exc, space.wrap('end')) + end = space.int_w(w_end) + builder = UnicodeBuilder() + pos = start + while pos < end: + oc = ord(obj[pos]) + num = hex(oc) + if (oc >= 0x10000): + builder.append(u"\\U") + zeros = 8 elif (oc >= 0x100): - p += 'u' - p += "%.4x" % ord(c) + builder.append(u"\\u") + zeros = 4 else: - p += 'x' - p += "%.2x" % ord(c) - return u''.join(p), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) -""") + builder.append(u"\\x") + zeros = 2 + lnum = len(num) + nb = zeros + 2 - lnum # num starts with '0x' + if nb > 0: + builder.append_multiple_char(u'0', nb) + builder.append_slice(unicode(num), 2, lnum) + pos += 1 + return space.newtuple([space.wrap(builder.build()), w_end]) + else: + typename = space.type(w_exc).getname(space, '?') + raise operationerrfmt(space.w_TypeError, + "don't know how to handle %s in error callback", typename) def register_builtin_error_handlers(space): + "NOT_RPYTHON" state = space.fromcache(CodecState) for error in ("strict", "ignore", "replace", "xmlcharrefreplace", "backslashreplace"): name = error + "_errors" - state.codec_error_registry[error] = app_errors.wget(space, name) + state.codec_error_registry[error] = space.wrap(interp2app(globals()[name])) def lookup_error(space, errors): @@ -279,6 +344,38 @@ from pypy.rlib import runicode +def make_raw_encoder(name): + rname = "unicode_encode_%s" % (name.replace("_encode", ""), ) + assert hasattr(runicode, rname) + def raw_encoder(space, uni): + state = space.fromcache(CodecState) + func = getattr(runicode, rname) + errors = "strict" + return func(uni, len(uni), errors, state.encode_error_handler) + raw_encoder.func_name = rname + return raw_encoder + +def make_raw_decoder(name): + rname = "str_decode_%s" % (name.replace("_decode", ""), ) + assert hasattr(runicode, rname) + def raw_decoder(space, string): + final = True + errors = "strict" + state = space.fromcache(CodecState) + func = getattr(runicode, rname) + kwargs = {} + if name == 'unicode_escape': + unicodedata_handler = state.get_unicodedata_handler(space) + result, consumed = func(string, len(string), errors, + final, state.decode_error_handler, + unicodedata_handler=unicodedata_handler) + else: + result, consumed = func(string, len(string), errors, + final, state.decode_error_handler) + return result + raw_decoder.func_name = rname + return raw_decoder + def make_encoder_wrapper(name): rname = "unicode_encode_%s" % (name.replace("_encode", ""), ) assert hasattr(runicode, rname) @@ -308,20 +405,26 @@ for encoders in [ "ascii_encode", "latin_1_encode", + "utf_7_encode", "utf_8_encode", "utf_16_encode", "utf_16_be_encode", "utf_16_le_encode", + "unicode_escape_encode", + "raw_unicode_escape_encode", + "unicode_internal_encode", ]: make_encoder_wrapper(encoders) for decoders in [ "ascii_decode", "latin_1_decode", + "utf_7_decode", "utf_8_decode", "utf_16_decode", "utf_16_be_decode", "utf_16_le_decode", + "raw_unicode_escape_decode", ]: make_decoder_wrapper(decoders) @@ -330,8 +433,6 @@ make_decoder_wrapper('mbcs_decode') def utf_16_ex_decode(space, data, errors='strict', byteorder=0, w_final=False): - """None - """ final = space.is_true(w_final) state = space.fromcache(CodecState) if byteorder == 0: @@ -349,77 +450,213 @@ space.wrap(byteorder)]) utf_16_ex_decode.unwrap_spec = [ObjSpace, str, str, int, W_Root] -def _extract_from_mapping(space, mapping_w, w_mapping, ch): - if mapping_w is not None: +# ____________________________________________________________ +# Charmap + +class Charmap_Decode: + def __init__(self, space, w_mapping): + self.space = space + self.w_mapping = w_mapping + + # fast path for all the stuff in the encodings module + if space.is_true(space.isinstance(w_mapping, space.w_tuple)): + self.mapping_w = space.fixedview(w_mapping) + else: + self.mapping_w = None + + def get(self, ch, errorchar): + space = self.space + + # get the character from the mapping + if self.mapping_w is not None: + w_ch = self.mapping_w[ord(ch)] + else: + try: + w_ch = space.getitem(self.w_mapping, space.newint(ord(ch))) + except OperationError, e: + if not e.match(space, space.w_LookupError): + raise + return errorchar + + # Charmap may return a unicode string try: - return mapping_w[ord(ch)] - except IndexError: - pass - else: + x = space.unicode_w(w_ch) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + else: + return x + + # Charmap may return a number + try: + x = space.int_w(w_ch) + except OperationError: + if not e.match(space, space.w_TypeError): + raise + else: + if 0 <= x < 65536: # Even on wide unicode builds... + return unichr(x) + else: + raise OperationError(space.w_TypeError, space.wrap( + "character mapping must be in range(65536)")) + + # Charmap may return None + if space.is_w(w_ch, space.w_None): + return errorchar + + raise OperationError(space.w_TypeError, space.wrap("invalid mapping")) + +class Charmap_Encode: + def __init__(self, space, w_mapping): + self.space = space + self.w_mapping = w_mapping + + def get(self, ch, errorchar): + space = self.space + + # get the character from the mapping try: - return space.getitem(w_mapping, space.newint(ord(ch))) + w_ch = space.getitem(self.w_mapping, space.newint(ord(ch))) except OperationError, e: - if (not e.match(space, space.w_KeyError) and - not e.match(space, space.w_IndexError)): + if not e.match(space, space.w_LookupError): raise - pass + return errorchar -def _append_unicode(space, builder, w_x): - try: - x = space.unicode_w(w_x) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - else: - if x != u"\ufffe": - builder.append(x) - return True - return False - try: - x = space.int_w(w_x) - except OperationError: - if not e.match(space, space.w_TypeError): - raise - else: - if x < 65536: - builder.append(unichr(x)) + # Charmap may return a string + try: + x = space.realstr_w(w_ch) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise else: - raise OperationError(space.w_TypeError, space.wrap("character mapping must be in range(65536)")) - return True - if not space.is_true(w_x): - return False + return x + + # Charmap may return a number + try: + x = space.int_w(w_ch) + except OperationError: + if not e.match(space, space.w_TypeError): + raise + else: + if 0 <= x < 256: + return chr(x) + else: + raise OperationError(space.w_TypeError, space.wrap( + "character mapping must be in range(256)")) + + # Charmap may return None + if space.is_w(w_ch, space.w_None): + return errorchar + + raise OperationError(space.w_TypeError, space.wrap("invalid mapping")) + + + at unwrap_spec(ObjSpace, str, str, W_Root) +def charmap_decode(space, string, errors="strict", w_mapping=None): + if len(string) == 0: + return space.newtuple([space.wrap(u''), space.wrap(0)]) + + if space.is_w(w_mapping, space.w_None): + mapping = None else: - raise OperationError(space.w_TypeError, space.w_None) + mapping = Charmap_Decode(space, w_mapping) + + final = True + state = space.fromcache(CodecState) + result, consumed = runicode.str_decode_charmap( + string, len(string), errors, + final, state.decode_error_handler, mapping) + return space.newtuple([space.wrap(result), space.wrap(consumed)]) + + at unwrap_spec(ObjSpace, unicode, str, W_Root) +def charmap_encode(space, uni, errors="strict", w_mapping=None): + if space.is_w(w_mapping, space.w_None): + mapping = None + else: + mapping = Charmap_Encode(space, w_mapping) + + state = space.fromcache(CodecState) + result = runicode.unicode_encode_charmap( + uni, len(uni), errors, + state.encode_error_handler, mapping) + return space.newtuple([space.wrap(result), space.wrap(len(uni))]) -def charmap_decode(space, s, errors="strict", w_mapping=None): - size = len(s) - # Default to Latin-1 - if space.is_true(space.is_(w_mapping, space.w_None)): - return latin_1_decode(space, s, errors, space.w_False) + at unwrap_spec(ObjSpace, unicode) +def charmap_build(space, chars): + # XXX CPython sometimes uses a three-level trie + w_charmap = space.newdict() + for num in range(len(chars)): + elem = chars[num] + space.setitem(w_charmap, space.newint(ord(elem)), space.newint(num)) + return w_charmap - if (size == 0): +# ____________________________________________________________ +# Unicode escape + +class UnicodeData_Handler: + def __init__(self, space, w_getcode): + self.space = space + self.w_getcode = w_getcode + + def call(self, name): + space = self.space + try: + w_code = space.call_function(self.w_getcode, space.wrap(name)) + except OperationError, e: + if not e.match(space, space.w_KeyError): + raise + return -1 + return space.int_w(w_code) + + at unwrap_spec(ObjSpace, 'bufferstr', str, W_Root) +def unicode_escape_decode(space, string, errors="strict", w_final=False): + final = space.is_true(w_final) + state = space.fromcache(CodecState) + errorhandler=state.decode_error_handler + + unicode_name_handler = state.get_unicodedata_handler(space) + + result, consumed = runicode.str_decode_unicode_escape( + string, len(string), errors, + final, state.decode_error_handler, + unicode_name_handler) + + return space.newtuple([space.wrap(result), space.wrap(consumed)]) + +# ____________________________________________________________ +# Unicode-internal + + at unwrap_spec(ObjSpace, W_Root, str) +def unicode_internal_decode(space, w_string, errors="strict"): + # special case for this codec: unicodes are returned as is + if space.isinstance_w(w_string, space.w_unicode): + return space.newtuple([w_string, space.len(w_string)]) + + string = space.str_w(w_string) + + if len(string) == 0: return space.newtuple([space.wrap(u''), space.wrap(0)]) - - # fast path for all the stuff in the encodings module - if space.is_true(space.isinstance(w_mapping, space.w_tuple)): - mapping_w = space.fixedview(w_mapping) - else: - mapping_w = None - - builder = UnicodeBuilder(size) - inpos = 0 - while (inpos < len(s)): - #/* Get mapping_w (char ordinal -> integer, Unicode char or None) */ - ch = s[inpos] - w_x = _extract_from_mapping(space, mapping_w, w_mapping, ch) - if w_x is not None and _append_unicode(space, builder, w_x): - inpos += 1 - continue - state = space.fromcache(CodecState) - next, inpos = state.decode_error_handler(errors, "charmap", - "character maps to ", s, inpos, inpos+1) - builder.append(next) - res = builder.build() - return space.newtuple([space.wrap(res), space.wrap(size)]) -charmap_decode.unwrap_spec = [ObjSpace, str, str, W_Root] + + final = True + state = space.fromcache(CodecState) + result, consumed = runicode.str_decode_unicode_internal( + string, len(string), errors, + final, state.decode_error_handler) + return space.newtuple([space.wrap(result), space.wrap(consumed)]) + +# ____________________________________________________________ +# support for the "string escape" codec +# This is a bytes-to bytes transformation + + at unwrap_spec(ObjSpace, W_Root, str) +def escape_encode(space, w_string, errors='strict'): + w_repr = space.repr(w_string) + w_result = space.getslice(w_repr, space.wrap(1), space.wrap(-1)) + return space.newtuple([w_result, space.len(w_string)]) + + at unwrap_spec(ObjSpace, str, str) +def escape_decode(space, data, errors='strict'): + from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape + result = PyString_DecodeEscape(space, data, None) + return space.newtuple([space.wrap(result), space.wrap(len(data))]) Modified: pypy/branch/fast-forward/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/fast-forward/pypy/module/_codecs/test/test_codecs.py Tue Jul 6 00:00:33 2010 @@ -1,7 +1,5 @@ import autopath from pypy.conftest import gettestobjspace -from pypy.module._codecs.app_codecs import unicode_escape_encode,\ - charmap_encode, unicode_escape_decode class AppTestCodecs: @@ -14,26 +12,16 @@ raises(TypeError, _codecs.register, 1) def test_bigU_codecs(self): - import sys - oldmaxunicode = sys.maxunicode - if sys.maxunicode <= 0xffff: - return # this test cannot run on UCS2 builds u = u'\U00010001\U00020002\U00030003\U00040004\U00050005' for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', 'raw_unicode_escape', 'unicode_escape', 'unicode_internal'): assert unicode(u.encode(encoding),encoding) == u - sys.maxunicode = oldmaxunicode def test_ucs4(self): - import sys - oldmaxunicode = sys.maxunicode - if sys.maxunicode <= 0xffff: - sys.maxunicode = 0xffffffff x = u'\U00100000' y = x.encode("raw-unicode-escape").decode("raw-unicode-escape") assert x == y - sys.maxunicode = oldmaxunicode def test_named_unicode(self): assert unicode('\\N{SPACE}','unicode-escape') == u" " @@ -118,12 +106,20 @@ def test_charmap_decode(self): from _codecs import charmap_decode + import sys assert charmap_decode('', 'strict', 'blablabla') == ('', 0) assert charmap_decode('xxx') == ('xxx', 3) assert charmap_decode('xxx', 'strict', {ord('x'): u'XX'}) == ('XXXXXX', 3) map = tuple([unichr(i) for i in range(256)]) assert charmap_decode('xxx\xff', 'strict', map) == (u'xxx\xff', 4) + raises(TypeError, charmap_decode, '\xff', "replace", {0xff: 0x10001}) + + def test_unicode_escape(self): + from _codecs import unicode_escape_encode, unicode_escape_decode + assert unicode_escape_encode(u'abc') == (u'abc'.encode('unicode_escape'), 3) + assert unicode_escape_decode('abc') == (u'abc'.decode('unicode_escape'), 3) + assert unicode_escape_decode('\\x61\\x62\\x63') == (u'abc', 12) class AppTestPartialEvaluation: @@ -275,7 +271,6 @@ assert u"\u0663".encode("raw-unicode-escape") == "\u0663" def test_escape_decode(self): - test = 'a\n\\b\x00c\td\u2045'.encode('string_escape') assert test.decode('string_escape') =='a\n\\b\x00c\td\u2045' assert '\\077'.decode('string_escape') == '?' @@ -283,6 +278,14 @@ assert '\\253'.decode('string_escape') == chr(0253) assert '\\312'.decode('string_escape') == chr(0312) + def test_escape_decode_wrap_around(self): + assert '\\400'.decode('string_escape') == chr(0) + + def test_escape_decode_ignore_invalid(self): + assert '\\9'.decode('string_escape') == '\\9' + assert '\\01'.decode('string_escape') == chr(01) + assert '\\0f'.decode('string_escape') == chr(0) + 'f' + assert '\\08'.decode('string_escape') == chr(0) + '8' def test_decode_utf8_different_case(self): constant = u"a" @@ -377,6 +380,9 @@ def test_charmap_decode_1(self): import codecs + assert codecs.charmap_encode(u'xxx') == ('xxx', 3) + assert codecs.charmap_encode(u'xxx', 'strict', {ord('x'): 'XX'}) == ('XXXXXX', 3) + res = codecs.charmap_decode("\x00\x01\x02", "replace", u"ab") assert res == (u"ab\ufffd", 3) res = codecs.charmap_decode("\x00\x01\x02", "replace", u"ab\ufffe") @@ -464,6 +470,9 @@ assert '\xff'.decode('utf-7', 'ignore') == '' assert '\x00'.decode('unicode-internal', 'ignore') == '' + def test_backslahreplace(self): + assert u'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == 'a\\xac\u1234\u20ac\u8000' + def test_badhandler(self): import codecs results = ( 42, u"foo", (1,2,3), (u"foo", 1, 3), (u"foo", None), (u"foo",), ("foo", 1, 3), ("foo", None), ("foo",) ) @@ -527,9 +536,26 @@ def test_charmap_encode(self): assert 'xxx'.encode('charmap') == 'xxx' + import codecs + raises(TypeError, codecs.charmap_encode, u'\xff', "replace", {0xff: 300}) + raises(UnicodeError, codecs.charmap_encode, u"\xff", "replace", {0xff: None}) + + def test_charmap_encode_replace(self): + charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"]) + charmap[ord("?")] = "XYZ" + import codecs + sin = u"abcDEF" + sout = codecs.charmap_encode(sin, "replace", charmap)[0] + assert sout == "AABBCCXYZXYZXYZ" + def test_charmap_decode_2(self): assert 'foo'.decode('charmap') == 'foo' + def test_charmap_build(self): + import codecs + assert codecs.charmap_build(u'123456') == {49: 0, 50: 1, 51: 2, + 52: 3, 53: 4, 54: 5} + def test_utf7_start_end_in_exception(self): try: '+IC'.decode('utf-7') @@ -537,6 +563,9 @@ assert exc.start == 0 assert exc.end == 3 + def test_utf7_surrogate(self): + raises(UnicodeDecodeError, '+3ADYAA-'.decode, 'utf-7') + def test_utf_16_encode_decode(self): import codecs x = u'123abc' @@ -546,6 +575,8 @@ def test_unicode_escape(self): assert u'\\'.encode('unicode-escape') == '\\\\' assert '\\\\'.decode('unicode-escape') == u'\\' + assert u'\ud801'.encode('unicode-escape') == '\\ud801' + assert u'\u0013'.encode('unicode-escape') == '\\x13' def test_mbcs(self): import sys @@ -555,14 +586,3 @@ assert u'caf\xe9'.encode('mbcs') == 'caf\xe9' assert u'\u040a'.encode('mbcs') == '?' # some cyrillic letter assert 'cafx\e9'.decode('mbcs') == u'cafx\e9' - - -class TestDirect: - def test_charmap_encode(self): - assert charmap_encode(u'xxx') == ('xxx', 3) - assert charmap_encode(u'xxx', 'strict', {ord('x'): 'XX'}) == ('XXXXXX', 6) - - def test_unicode_escape(self): - assert unicode_escape_encode(u'abc') == (u'abc'.encode('unicode_escape'), 3) - assert unicode_escape_decode('abc') == (u'abc'.decode('unicode_escape'), 3) - assert unicode_escape_decode('\\x61\\x62\\x63') == (u'abc', 12) Modified: pypy/branch/fast-forward/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/fast-forward/pypy/module/_stackless/interp_coroutine.py Tue Jul 6 00:00:33 2010 @@ -24,7 +24,9 @@ from pypy.interpreter.function import StaticMethod from pypy.module._stackless.stackless_flags import StacklessFlags -from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState, AbstractThunk +from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState, AbstractThunk, CoroutineExit + +from pypy.module.exceptions.interp_exceptions import W_SystemExit, _new_exception from pypy.rlib import rstack # for resume points from pypy.tool import stdlib_opcode as pythonopcode @@ -48,6 +50,13 @@ rstack.resume_point("appthunk", costate, returns=w_result) costate.w_tempval = w_result +W_CoroutineExit = _new_exception('CoroutineExit', W_SystemExit, + """Coroutine killed manually.""") + +# Should be moved to interp_stackless.py if it's ever implemented... Currently +# used by pypy/lib/stackless.py. +W_TaskletExit = _new_exception('TaskletExit', W_SystemExit, + """Tasklet killed manually.""") class AppCoroutine(Coroutine): # XXX, StacklessFlags): @@ -92,6 +101,13 @@ w_ret, state.w_tempval = state.w_tempval, space.w_None return w_ret + def switch(self): + space = self.space + try: + Coroutine.switch(self) + except CoroutineExit: + raise OperationError(self.costate.w_CoroutineExit, space.w_None) + def w_finished(self, w_excinfo): pass @@ -102,6 +118,9 @@ w_excvalue = operror.get_w_value(space) w_exctraceback = operror.application_traceback w_excinfo = space.newtuple([w_exctype, w_excvalue, w_exctraceback]) + + if w_exctype is self.costate.w_CoroutineExit: + self.coroutine_exit = True else: w_N = space.w_None w_excinfo = space.newtuple([w_N, w_N, w_N]) @@ -118,6 +137,23 @@ def w_kill(self): self.kill() + + def w_throw(self, w_type, w_value=None, w_traceback=None): + space = self.space + + operror = OperationError(w_type, w_value) + operror.normalize_exception(space) + + if not space.is_w(w_traceback, space.w_None): + from pypy.interpreter import pytraceback + tb = space.interpclass_w(w_traceback) + if tb is None or not space.is_true(space.isinstance(tb, + space.gettypeobject(pytraceback.PyTraceback.typedef))): + raise OperationError(space.w_TypeError, + space.wrap("throw: arg 3 must be a traceback or None")) + operror.application_traceback = tb + + self._kill(operror) def _userdel(self): if self.get_is_zombie(): @@ -309,6 +345,7 @@ unwrap_spec=['self', W_Root, Arguments]), switch = interp2app(AppCoroutine.w_switch), kill = interp2app(AppCoroutine.w_kill), + throw = interp2app(AppCoroutine.w_throw), finished = interp2app(AppCoroutine.w_finished), is_alive = GetSetProperty(AppCoroutine.w_get_is_alive), is_zombie = GetSetProperty(AppCoroutine.w_get_is_zombie, @@ -330,6 +367,26 @@ BaseCoState.__init__(self) self.w_tempval = space.w_None self.space = space + + # Exporting new exception to space + self.w_CoroutineExit = space.gettypefor(W_CoroutineExit) + space.setitem( + space.exceptions_module.w_dict, + space.new_interned_str('CoroutineExit'), + self.w_CoroutineExit) + space.setitem(space.builtin.w_dict, + space.new_interned_str('CoroutineExit'), + self.w_CoroutineExit) + + # Should be moved to interp_stackless.py if it's ever implemented... + self.w_TaskletExit = space.gettypefor(W_TaskletExit) + space.setitem( + space.exceptions_module.w_dict, + space.new_interned_str('TaskletExit'), + self.w_TaskletExit) + space.setitem(space.builtin.w_dict, + space.new_interned_str('TaskletExit'), + self.w_TaskletExit) def post_install(self): self.current = self.main = AppCoroutine(self.space, state=self) Modified: pypy/branch/fast-forward/pypy/module/_stackless/rcoroutine.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_stackless/rcoroutine.py (original) +++ pypy/branch/fast-forward/pypy/module/_stackless/rcoroutine.py Tue Jul 6 00:00:33 2010 @@ -7,3 +7,4 @@ BaseCoState = d['BaseCoState'] AbstractThunk = d['AbstractThunk'] syncstate = d['syncstate'] +CoroutineExit = d['CoroutineExit'] Modified: pypy/branch/fast-forward/pypy/module/_stackless/test/test_coroutine.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_stackless/test/test_coroutine.py (original) +++ pypy/branch/fast-forward/pypy/module/_stackless/test/test_coroutine.py Tue Jul 6 00:00:33 2010 @@ -84,8 +84,7 @@ assert not co.is_alive def test_kill_running(self): - skip("kill is not really working (there is only CoroutineExit, " - "which is not an app-level exception)") + coroutineexit = [] import _stackless as stackless main = stackless.coroutine.getcurrent() result = [] @@ -96,6 +95,9 @@ result.append(1) main.switch() x = 3 + except CoroutineExit: + coroutineexit.append(True) + raise finally: result.append(x) result.append(4) @@ -107,6 +109,7 @@ co.kill() assert not co.is_alive assert result == [1, 2] + assert coroutineexit == [True] def test_bogus_bind(self): import _stackless as stackless Modified: pypy/branch/fast-forward/pypy/module/conftest.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/conftest.py (original) +++ pypy/branch/fast-forward/pypy/module/conftest.py Tue Jul 6 00:00:33 2010 @@ -2,16 +2,17 @@ from pypy.tool.lib_pypy import LIB_PYPY class MultipleDirCollector(py.test.collect.Collector): - def __init__(self, name, fspaths, parent=None, config=None): + def __init__(self, name, mainfspath, fspaths, parent=None, config=None): super(MultipleDirCollector, self).__init__(name, parent, config) + self.main_collector = py.test.collect.Directory(mainfspath, self) self.collectors = [py.test.collect.Directory(fspath, self) for fspath in fspaths] def collect(self): - return self.collectors + return self.main_collector.collect() + self.collectors def pytest_collect_directory(path, parent): if path.basename == 'test_lib_pypy': # collect all the test in BOTH test_lib_pypy and ../../lib_pypy - return MultipleDirCollector(path.basename, [path, LIB_PYPY], parent) + return MultipleDirCollector(path.basename, path, [LIB_PYPY], parent) Modified: pypy/branch/fast-forward/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/api.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/api.py Tue Jul 6 00:00:33 2010 @@ -45,11 +45,9 @@ pypydir = py.path.local(autopath.pypydir) include_dir = pypydir / 'module' / 'cpyext' / 'include' source_dir = pypydir / 'module' / 'cpyext' / 'src' -interfaces_dir = pypydir / "_interfaces" include_dirs = [ include_dir, udir, - interfaces_dir, ] class CConfig: @@ -100,9 +98,20 @@ udir.join('pypy_macros.h').write("/* Will be filled later */") globals().update(rffi_platform.configure(CConfig_constants)) -def copy_header_files(): +def copy_header_files(dstdir): + assert dstdir.check(dir=True) + headers = include_dir.listdir('*.h') + include_dir.listdir('*.inl') for name in ("pypy_decl.h", "pypy_macros.h"): - udir.join(name).copy(interfaces_dir / name) + headers.append(udir.join(name)) + for header in headers: + target = dstdir.join(header.basename) + try: + header.copy(dstdir) + except py.error.EACCES: + target.remove() # maybe it was a read-only file + header.copy(dstdir) + target.chmod(0444) # make the file read-only, to make sure that nobody + # edits it by mistake _NOT_SPECIFIED = object() CANNOT_FAIL = object() @@ -881,7 +890,8 @@ deco(func.get_wrapper(space)) setup_init_functions(eci) - copy_header_files() + trunk_include = pypydir.dirpath() / 'include' + copy_header_files(trunk_include) initfunctype = lltype.Ptr(lltype.FuncType([], lltype.Void)) @unwrap_spec(ObjSpace, str, str) Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_api.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_api.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_api.py Tue Jul 6 00:00:33 2010 @@ -1,3 +1,4 @@ +import py from pypy.conftest import gettestobjspace from pypy.rpython.lltypesystem import rffi, lltype from pypy.interpreter.baseobjspace import W_Root @@ -68,3 +69,13 @@ api.PyPy_GetWrapped(space.w_None) api.PyPy_GetReference(space.w_None) + +def test_copy_header_files(tmpdir): + api.copy_header_files(tmpdir) + def check(name): + f = tmpdir.join(name) + assert f.check(file=True) + py.test.raises(py.error.EACCES, "f.open('w')") # check that it's not writable + check('Python.h') + check('modsupport.inl') + check('pypy_decl.h') Modified: pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py (original) +++ pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py Tue Jul 6 00:00:33 2010 @@ -359,8 +359,8 @@ return self.reader.read(n) def get1(self): - # convince typer to use a char - return chr(ord(self.get(1))) + # the [0] is used to convince the annotator to return a char + return self.get(1)[0] def atom_str(self, typecode): self.start(typecode) Modified: pypy/branch/fast-forward/pypy/module/operator/test/test_operator.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/operator/test/test_operator.py (original) +++ pypy/branch/fast-forward/pypy/module/operator/test/test_operator.py Tue Jul 6 00:00:33 2010 @@ -164,7 +164,6 @@ def test_irepeat(self): import operator - import py class X(object): def __index__(self): Modified: pypy/branch/fast-forward/pypy/module/sys/state.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/sys/state.py (original) +++ pypy/branch/fast-forward/pypy/module/sys/state.py Tue Jul 6 00:00:33 2010 @@ -32,30 +32,19 @@ if not stat.S_ISDIR(st[0]): raise OSError(errno.ENOTDIR, path) -def getinitialpath(prefix): - from pypy.module.sys.version import PYPY_VERSION - libdir = os.path.join(prefix, 'lib') - pypyxy_dir = os.path.join(libdir, 'pypy%d.%d' % PYPY_VERSION[:2]) - # search for the stdlib both in $PREFIX/lib/pypy1.2 and $PREFIX - for lib_prefix in [pypyxy_dir, prefix]: - try: - return get_importlist(lib_prefix) - except OSError: - pass - raise OSError # stdlib not foud -def get_importlist(lib_prefix): +def getinitialpath(prefix): from pypy.module.sys.version import CPYTHON_VERSION dirname = '%d.%d.%d' % (CPYTHON_VERSION[0], CPYTHON_VERSION[1], CPYTHON_VERSION[2]) - lib_python = os.path.join(lib_prefix, 'lib-python') + lib_python = os.path.join(prefix, 'lib-python') python_std_lib = os.path.join(lib_python, dirname) checkdir(python_std_lib) python_std_lib_modified = os.path.join(lib_python, 'modified-' + dirname) checkdir(python_std_lib_modified) - lib_pypy = os.path.join(lib_prefix, 'lib_pypy') + lib_pypy = os.path.join(prefix, 'lib_pypy') checkdir(lib_pypy) importlist = [] Modified: pypy/branch/fast-forward/pypy/module/sys/test/test_initialpath.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/sys/test/test_initialpath.py (original) +++ pypy/branch/fast-forward/pypy/module/sys/test/test_initialpath.py Tue Jul 6 00:00:33 2010 @@ -12,21 +12,7 @@ return a, b, c -def test_stdlib_in_pypyxy(tmpdir): - pypyxy = tmpdir.join('lib', 'pypy%d.%d' % PYPY_VERSION[:2]) - dirs = build_hierarchy(pypyxy) - path = getinitialpath(str(tmpdir)) - assert path == map(str, dirs) - - def test_stdlib_in_prefix(tmpdir): dirs = build_hierarchy(tmpdir) path = getinitialpath(str(tmpdir)) assert path == map(str, dirs) - -def test_stdlib_precedence(tmpdir): - pypyxy = tmpdir.join('lib', 'pypy%d.%d' % PYPY_VERSION[:2]) - dirs1 = build_hierarchy(tmpdir) - dirs2 = build_hierarchy(pypyxy) - path = getinitialpath(str(tmpdir)) - assert path == map(str, dirs2) Modified: pypy/branch/fast-forward/pypy/module/sys/version.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/sys/version.py (original) +++ pypy/branch/fast-forward/pypy/module/sys/version.py Tue Jul 6 00:00:33 2010 @@ -7,8 +7,7 @@ CPYTHON_VERSION = (2, 7, 0, "final", 42) CPYTHON_API_VERSION = 1012 -# when changing the pypy version, remind to also change the name of trunk/lib/pypyX.Y -PYPY_VERSION = (1, 2, 0, "beta", '?') +PYPY_VERSION = (1, 3, 0, "beta", '?') # the last item is replaced by the svn revision ^^^ TRIM_URL_UP_TO = 'svn/pypy/' Modified: pypy/branch/fast-forward/pypy/module/termios/interp_termios.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/termios/interp_termios.py (original) +++ pypy/branch/fast-forward/pypy/module/termios/interp_termios.py Tue Jul 6 00:00:33 2010 @@ -60,8 +60,8 @@ # last one need to be chosen carefully cc_w = [space.wrap(i) for i in cc] if lflag & termios.ICANON: - cc_w[termios.VMIN] = space.wrap(ord(cc[termios.VMIN])) - cc_w[termios.VTIME] = space.wrap(ord(cc[termios.VTIME])) + cc_w[termios.VMIN] = space.wrap(ord(cc[termios.VMIN][0])) + cc_w[termios.VTIME] = space.wrap(ord(cc[termios.VTIME][0])) w_cc = space.newlist(cc_w) l_w.append(w_cc) return space.newlist(l_w) Modified: pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py Tue Jul 6 00:00:33 2010 @@ -447,11 +447,11 @@ register(TYPE_CODE, unmarshal_pycode) def marshal_w__Unicode(space, w_unicode, m): - s = space.str_w(unicodehelper.PyUnicode_EncodeUTF8(space, w_unicode)) + s = unicodehelper.PyUnicode_EncodeUTF8(space, space.unicode_w(w_unicode)) m.atom_str(TYPE_UNICODE, s) def unmarshal_Unicode(space, u, tc): - return unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(u.get_str())) + return space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, u.get_str())) register(TYPE_UNICODE, unmarshal_Unicode) app = gateway.applevel(r''' Modified: pypy/branch/fast-forward/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/stringobject.py Tue Jul 6 00:00:33 2010 @@ -863,7 +863,7 @@ space.w_TypeError, "ord() expected a character, but string " "of length %d found", len(u_str)) - return space.wrap(ord(u_str)) + return space.wrap(ord(u_str[0])) def getnewargs__String(space, w_str): return space.newtuple([wrapstr(space, w_str._value)]) Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_shadowtracking.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_shadowtracking.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_shadowtracking.py Tue Jul 6 00:00:33 2010 @@ -209,6 +209,7 @@ a = A() names = [name for name in A.__dict__.keys() if not name.startswith('_')] + names.sort() names_repeated = names * 10 result = [] __pypy__.reset_method_cache_counter() Modified: pypy/branch/fast-forward/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/unicodeobject.py Tue Jul 6 00:00:33 2010 @@ -12,6 +12,7 @@ from pypy.rlib.rarithmetic import intmask, ovfcheck from pypy.rlib.objectmodel import compute_hash from pypy.rlib.rstring import string_repeat +from pypy.rlib.runicode import unicode_encode_unicode_escape from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb from pypy.tool.sourcetools import func_with_new_name @@ -892,101 +893,11 @@ space.wrap("character mapping must return integer, None or unicode")) return W_UnicodeObject(u''.join(result)) -# Move this into the _codecs module as 'unicodeescape_string (Remember to cater for quotes)' def repr__Unicode(space, w_unicode): - hexdigits = "0123456789abcdef" chars = w_unicode._value size = len(chars) - - singlequote = doublequote = False - for c in chars: - if c == u'\'': - singlequote = True - elif c == u'"': - doublequote = True - if singlequote and not doublequote: - quote = '"' - else: - quote = '\'' - result = ['u', quote] - j = 0 - while j= 0x10000: - # Resize if needed - result.extend(['\\', "U", - hexdigits[(code >> 28) & 0xf], - hexdigits[(code >> 24) & 0xf], - hexdigits[(code >> 20) & 0xf], - hexdigits[(code >> 16) & 0xf], - hexdigits[(code >> 12) & 0xf], - hexdigits[(code >> 8) & 0xf], - hexdigits[(code >> 4) & 0xf], - hexdigits[(code >> 0) & 0xf], - ]) - j += 1 - continue - if code >= 0xD800 and code < 0xDC00: - if j < size - 1: - ch2 = chars[j+1] - code2 = ord(ch2) - if code2 >= 0xDC00 and code2 <= 0xDFFF: - code = (((code & 0x03FF) << 10) | (code2 & 0x03FF)) + 0x00010000 - result.extend(['\\', "U", - hexdigits[(code >> 28) & 0xf], - hexdigits[(code >> 24) & 0xf], - hexdigits[(code >> 20) & 0xf], - hexdigits[(code >> 16) & 0xf], - hexdigits[(code >> 12) & 0xf], - hexdigits[(code >> 8) & 0xf], - hexdigits[(code >> 4) & 0xf], - hexdigits[(code >> 0) & 0xf], - ]) - j += 2 - continue - - if code >= 0x100: - result.extend(['\\', "u", - hexdigits[(code >> 12) & 0xf], - hexdigits[(code >> 8) & 0xf], - hexdigits[(code >> 4) & 0xf], - hexdigits[(code >> 0) & 0xf], - ]) - j += 1 - continue - if code == ord('\\') or code == ord(quote): - result.append('\\') - result.append(chr(code)) - j += 1 - continue - if code == ord('\t'): - result.append('\\') - result.append('t') - j += 1 - continue - if code == ord('\r'): - result.append('\\') - result.append('r') - j += 1 - continue - if code == ord('\n'): - result.append('\\') - result.append('n') - j += 1 - continue - if code < ord(' ') or code >= 0x7f: - result.extend(['\\', "x", - hexdigits[(code >> 4) & 0xf], - hexdigits[(code >> 0) & 0xf], - ]) - j += 1 - continue - result.append(chr(code)) - j += 1 - result.append(quote) - return space.wrap(''.join(result)) - + s = unicode_encode_unicode_escape(chars, size, "strict", quotes=True) + return space.wrap(s) def mod__Unicode_ANY(space, w_format, w_values): return mod_format(space, w_format, w_values, do_unicode=True) Modified: pypy/branch/fast-forward/pypy/rlib/debug.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/debug.py (original) +++ pypy/branch/fast-forward/pypy/rlib/debug.py Tue Jul 6 00:00:33 2010 @@ -250,3 +250,16 @@ def specialize_call(self, hop): hop.exception_cannot_occur() return hop.inputarg(hop.args_r[0], arg=0) + + +class IntegerCanBeNegative(Exception): + pass + +def _check_nonneg(ann, bk): + from pypy.annotation.model import SomeInteger + s_nonneg = SomeInteger(nonneg=True) + if not s_nonneg.contains(ann): + raise IntegerCanBeNegative + +def check_nonneg(x): + check_annotation(x, _check_nonneg) Modified: pypy/branch/fast-forward/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/libffi.py (original) +++ pypy/branch/fast-forward/pypy/rlib/libffi.py Tue Jul 6 00:00:33 2010 @@ -402,7 +402,7 @@ restype = ffi_type_sint32 elif restype.c_size <= 8: restype = ffi_type_sint64 - + res = c_ffi_prep_cif(self.ll_cif, cc, rffi.cast(rffi.UINT, argnum), restype, self.ll_argtypes) Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Tue Jul 6 00:00:33 2010 @@ -57,13 +57,10 @@ from math import isinf, isnan, copysign except ImportError: def isinf(x): - return x != 0.0 and x / 2 == x + return x == INFINITY or x == -INFINITY - # To get isnan, working x-platform and both on 2.3 and 2.4, is a - # horror. I think this works (for reasons I don't really want to talk - # about), and probably when implemented on top of pypy, too. def isnan(v): - return v != v*1.0 or (v == 1.0 and v == 2.0) + return v != v def copysign(x, y): """Return x with the sign of y""" Modified: pypy/branch/fast-forward/pypy/rlib/rcoroutine.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rcoroutine.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rcoroutine.py Tue Jul 6 00:00:33 2010 @@ -32,6 +32,8 @@ from pypy.rlib.rstack import yield_current_frame_to_caller, resume_point from pypy.rlib.objectmodel import we_are_translated +from pypy.interpreter.error import OperationError + try: from greenlet import greenlet main_greenlet = greenlet.getcurrent() @@ -158,6 +160,7 @@ self.costate = state self.parent = None self.thunk = None + self.coroutine_exit = False def __repr__(self): 'NOT_RPYTHON' @@ -236,11 +239,12 @@ self = state.current self.finish(exc) except CoroutineExit: - # ignore a shutdown exception pass except Exception, e: - # redirect all unhandled exceptions to the parent - syncstate.push_exception(e) + if self.coroutine_exit is False: + # redirect all unhandled exceptions to the parent + syncstate.push_exception(e) + while self.parent is not None and self.parent.frame is None: # greenlet behavior is fine self.parent = self.parent.parent @@ -257,10 +261,13 @@ syncstate.switched(incoming_frame) def kill(self): + self._kill(CoroutineExit()) + + def _kill(self, exc): if self.frame is None: return state = self.costate - syncstate.push_exception(CoroutineExit()) + syncstate.push_exception(exc) # careful here - if setting self.parent to state.current would # create a loop, break it. The assumption is that 'self' # will die, so that state.current's chain of parents can be Modified: pypy/branch/fast-forward/pypy/rlib/rsre/test/targetrsre.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rsre/test/targetrsre.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rsre/test/targetrsre.py Tue Jul 6 00:00:33 2010 @@ -1,3 +1,4 @@ +#!/usr/bin/env python from pypy.rlib.rarithmetic import intmask from pypy.rlib.rsre import rsre import os, time Modified: pypy/branch/fast-forward/pypy/rlib/rstring.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstring.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstring.py Tue Jul 6 00:00:33 2010 @@ -56,6 +56,7 @@ self.l.append(s) def append_slice(self, s, start, end): + assert 0 <= start <= end <= len(s) self.l.append(s[start:end]) def append_multiple_char(self, c, times): Modified: pypy/branch/fast-forward/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/runicode.py (original) +++ pypy/branch/fast-forward/pypy/rlib/runicode.py Tue Jul 6 00:00:33 2010 @@ -1,7 +1,9 @@ import sys from pypy.rlib.bitmanipulation import splitter from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize +from pypy.rlib.rstring import StringBuilder, UnicodeBuilder +from pypy.rlib.rarithmetic import r_uint if rffi.sizeof(lltype.UniChar) == 4: MAXUNICODE = 0x10ffff @@ -42,8 +44,6 @@ UNICHR = unichr ORD = ord -# XXX review the functions below and think about using stringbuilders for them - def raise_unicode_exception_decode(errors, encoding, msg, s, startingpos, endingpos): @@ -55,8 +55,8 @@ assert isinstance(u, unicode) raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg) -# ____________________________________________________________ -# unicode decoding +# ____________________________________________________________ +# utf-8 utf8_code_length = [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -81,9 +81,10 @@ errorhandler=None): if errorhandler is None: errorhandler = raise_unicode_exception_decode - if (size == 0): + if size == 0: return u'', 0 - result = [] + + result = UnicodeBuilder(size) pos = 0 while pos < size: ch = s[pos] @@ -94,14 +95,14 @@ continue n = utf8_code_length[ordch1] - if (pos + n > size): + if pos + n > size: if not final: break else: r, pos = errorhandler(errors, "utf-8", "unexpected end of data", s, pos, size) result.append(r) - if (pos + n > size): + if pos + n > size: break if n == 0: r, pos = errorhandler(errors, "utf-8", "unexpected code byte", @@ -116,7 +117,7 @@ z, two = splitter[6, 2](ordch2) y, six = splitter[5, 3](ordch1) assert six == 6 - if (two != 2): + if two != 2: r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 2) result.append(r) @@ -137,7 +138,7 @@ y, two2 = splitter[6, 2](ordch2) x, fourteen = splitter[4, 4](ordch1) assert fourteen == 14 - if (two1 != 2 or two2 != 2): + if two1 != 2 or two2 != 2: r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 3) result.append(r) @@ -166,7 +167,7 @@ x, two3 = splitter[6, 2](ordch2) w, thirty = splitter[3, 5](ordch1) assert thirty == 30 - if (two1 != 2 or two2 != 2 or two3 != 2): + if two1 != 2 or two2 != 2 or two3 != 2: r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 4) result.append(r) @@ -174,7 +175,7 @@ c = (w << 18) + (x << 12) + (y << 6) + z # minimum value allowed for 4 byte encoding # maximum value allowed for UTF-16 - if ((c < 0x10000) or (c > 0x10ffff)): + if c < 0x10000 or c > 0x10ffff: r, pos = errorhandler(errors, "utf-8", "illegal encoding", s, pos, pos + 4) result.append(r) @@ -197,8 +198,53 @@ s, pos, pos + n) result.append(r) - return u"".join(result), pos + return result.build(), pos +def _encodeUCS4(result, ch): + # Encode UCS4 Unicode ordinals + result.append((chr((0xf0 | (ch >> 18))))) + result.append((chr((0x80 | ((ch >> 12) & 0x3f))))) + result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) + result.append((chr((0x80 | (ch & 0x3f))))) + +def unicode_encode_utf_8(s, size, errors, errorhandler=None): + assert(size >= 0) + result = StringBuilder(size) + i = 0 + while i < size: + ch = ord(s[i]) + i += 1 + if ch < 0x80: + # Encode ASCII + result.append(chr(ch)) + elif ch < 0x0800: + # Encode Latin-1 + result.append(chr((0xc0 | (ch >> 6)))) + result.append(chr((0x80 | (ch & 0x3f)))) + else: + # Encode UCS2 Unicode ordinals + if ch < 0x10000: + # Special case: check for high surrogate + if 0xD800 <= ch <= 0xDBFF and i != size: + ch2 = ord(s[i]) + # Check for low surrogate and combine the two to + # form a UCS4 value + if 0xDC00 <= ch2 <= 0xDFFF: + ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000 + i += 1 + _encodeUCS4(result, ch3) + continue + # Fall through: handles isolated high surrogates + result.append((chr((0xe0 | (ch >> 12))))) + result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) + result.append((chr((0x80 | (ch & 0x3f))))) + continue + else: + _encodeUCS4(result, ch) + return result.build() + +# ____________________________________________________________ +# utf-16 def str_decode_utf_16(s, size, errors, final=True, errorhandler=None): @@ -238,12 +284,11 @@ # mark is skipped, in all other modes, it is copied to the output # stream as-is (giving a ZWNBSP character). pos = 0 - result = [] if byteorder == 'native': - if (size >= 2): + if size >= 2: bom = (ord(s[ihi]) << 8) | ord(s[ilo]) if BYTEORDER == 'little': - if (bom == 0xFEFF): + if bom == 0xFEFF: pos += 2 bo = -1 elif bom == 0xFFFE: @@ -260,20 +305,22 @@ bo = -1 else: bo = 1 - if (size == 0): + if size == 0: return u'', 0, bo - if (bo == -1): + if bo == -1: # force little endian ihi = 1 ilo = 0 - elif (bo == 1): + elif bo == 1: # force big endian ihi = 0 ilo = 1 + result = UnicodeBuilder(size // 2) + #XXX I think the errors are not correctly handled here - while (pos < len(s)): + while pos < size: # remaining bytes at the end? (size should be even) if len(s) - pos < 2: if not final: @@ -285,7 +332,7 @@ break ch = (ord(s[pos + ihi]) << 8) | ord(s[pos + ilo]) pos += 2 - if (ch < 0xD800 or ch > 0xDFFF): + if ch < 0xD800 or ch > 0xDFFF: result.append(unichr(ch)) continue # UTF-16 code pair: @@ -297,10 +344,10 @@ result.append(r) if len(s) - pos < 2: break - elif (0xD800 <= ch and ch <= 0xDBFF): + elif 0xD800 <= ch <= 0xDBFF: ch2 = (ord(s[pos+ihi]) << 8) | ord(s[pos+ilo]) pos += 2 - if (0xDC00 <= ch2 and ch2 <= 0xDFFF): + if 0xDC00 <= ch2 <= 0xDFFF: if MAXUNICODE < 65536: result.append(unichr(ch)) result.append(unichr(ch2)) @@ -318,17 +365,305 @@ "illegal encoding", s, pos - 2, pos) result.append(r) - return u"".join(result), pos, bo + return result.build(), pos, bo + +def _STORECHAR(result, CH, byteorder): + hi = chr(((CH) >> 8) & 0xff) + lo = chr((CH) & 0xff) + if byteorder == 'little': + result.append(lo) + result.append(hi) + else: + result.append(hi) + result.append(lo) + +def unicode_encode_utf_16_helper(s, size, errors, + errorhandler=None, + byteorder='little'): + if size == 0: + return "" + + result = StringBuilder(size * 2 + 2) + if byteorder == 'native': + _STORECHAR(result, 0xFEFF, BYTEORDER) + byteorder = BYTEORDER + + i = 0 + while i < size: + ch = ord(s[i]) + i += 1 + ch2 = 0 + if ch >= 0x10000: + ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) + ch = 0xD800 | ((ch-0x10000) >> 10) + + _STORECHAR(result, ch, byteorder) + if ch2: + _STORECHAR(result, ch2, byteorder) + + return result.build() + +def unicode_encode_utf_16(s, size, errors, + errorhandler=None): + return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "native") + + +def unicode_encode_utf_16_be(s, size, errors, + errorhandler=None): + return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "big") + + +def unicode_encode_utf_16_le(s, size, errors, + errorhandler=None): + return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "little") + + +# ____________________________________________________________ +# utf-7 + +## indicate whether a UTF-7 character is special i.e. cannot be directly +## encoded: +## 0 - not special +## 1 - special +## 2 - whitespace (optional) +## 3 - RFC2152 Set O (optional) + +_utf7_special = [ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, + 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3, + 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1, +] + +def _utf7_SPECIAL(oc, encodeO=False, encodeWS=False): + return (oc > 127 or _utf7_special[oc] == 1 or + (encodeWS and _utf7_special[oc] == 2) or + (encodeO and _utf7_special[oc] == 3)) + +def _utf7_B64CHAR(oc): + if oc > 127: + return False + c = chr(oc) + return c.isalnum() or c == '+' or c == '/' +def _utf7_TO_BASE64(n): + "Returns the base-64 character of the bottom 6 bits of n" + return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[n & 0x3f] +def _utf7_FROM_BASE64(c): + "Retuns the base-64 value of a base-64 character" + if c == '+': + return 62 + elif c == '/': + return 63 + elif c >= 'a': + return ord(c) - 71 + elif c >= 'A': + return ord(c) - 65 + else: + return ord(c) + 4 + +def _utf7_ENCODE(result, ch, bits): + while bits >= 6: + result.append(_utf7_TO_BASE64(ch >> (bits - 6))) + bits -= 6 + return bits + +def _utf7_DECODE(s, result, errorhandler, errors, + pos, charsleft, bitsleft, surrogate): + while bitsleft >= 16: + outCh = (charsleft >> (bitsleft-16)) & 0xffff + bitsleft -= 16 + + if surrogate: + ## We have already generated an error for the high + ## surrogate so let's not bother seeing if the low + ## surrogate is correct or not + surrogate = False + elif 0xDC00 <= outCh <= 0xDFFF: + ## This is a surrogate pair. Unfortunately we can't + ## represent it in a 16-bit character + surrogate = True + msg = "code pairs are not supported" + res, pos = errorhandler(errors, 'utf-7', + msg, s, pos-1, pos) + result.append(res) + bitsleft = 0 + break + else: + result.append(unichr(outCh)) + return pos, charsleft, bitsleft, surrogate + + +def str_decode_utf_7(s, size, errors, final=False, + errorhandler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode + if size == 0: + return u'', 0 + + inShift = False + bitsleft = 0 + startinpos = 0 + charsleft = 0 + surrogate = False + + result = UnicodeBuilder(size) + pos = 0 + while pos < size: + ch = s[pos] + oc = ord(ch) + + if inShift: + if ch == '-' or not _utf7_B64CHAR(oc): + inShift = 0 + pos += 1 + + pos, charsleft, bitsleft, surrogate = _utf7_DECODE( + s, result, errorhandler, errors, + pos, charsleft, bitsleft, surrogate) + if bitsleft >= 6: + ## The shift sequence has a partial character in it. If + ## bitsleft < 6 then we could just classify it as padding + ## but that is not the case here + msg = "partial character in shift sequence" + res, pos = errorhandler(errors, 'utf-7', + msg, s, pos-1, pos) + result.append(res) + ## According to RFC2152 the remaining bits should be + ## zero. We choose to signal an error/insert a replacement + ## character here so indicate the potential of a + ## misencoded character. + if ch == '-': + if pos < size and s[pos] == '-': + result.append(u'-') + inShift = True + + elif _utf7_SPECIAL(oc): + msg = "unexpected special character" + res, pos = errorhandler(errors, 'utf-7', + msg, s, pos-1, pos) + result.append(res) + else: + result.append(unichr(ord(ch))) + else: + charsleft = (charsleft << 6) | _utf7_FROM_BASE64(ch) + bitsleft += 6 + pos += 1 + + pos, charsleft, bitsleft, surrogate = _utf7_DECODE( + s, result, errorhandler, errors, + pos, charsleft, bitsleft, surrogate) + elif ch == '+': + startinpos = pos + pos += 1 + if pos < size and s[pos] == '-': + pos += 1 + result.append(u'+') + else: + inShift = 1 + bitsleft = 0 + + elif _utf7_SPECIAL(oc): + pos += 1 + msg = "unexpected special character" + res, pos = errorhandler(errors, 'utf-7', msg, s, pos-1, pos) + result.append(res) + else: + result.append(unichr(oc)) + pos += 1 + + if inShift: + endinpos = size + msg = "unterminated shift sequence" + res, pos = errorhandler(errors, 'utf-7', msg, s, startinpos, pos) + result.append(res) + + return result.build(), pos + +def unicode_encode_utf_7(s, size, errors, errorhandler=None): + if size == 0: + return '' + result = StringBuilder(size) + + encodeSetO = encodeWhiteSpace = False + + inShift = False + bitsleft = 0 + charsleft = 0 + + pos = 0 + while pos < size: + ch = s[pos] + oc = ord(ch) + if not inShift: + if ch == u'+': + result.append('+-') + elif _utf7_SPECIAL(oc, encodeSetO, encodeWhiteSpace): + charsleft = oc + bitsleft = 16 + result.append('+') + bitsleft = _utf7_ENCODE(result, charsleft, bitsleft) + inShift = bitsleft > 0 + else: + result.append(chr(oc)) + else: + if not _utf7_SPECIAL(oc, encodeSetO, encodeWhiteSpace): + result.append(_utf7_TO_BASE64(charsleft << (6-bitsleft))) + charsleft = 0 + bitsleft = 0 + ## Characters not in the BASE64 set implicitly unshift the + ## sequence so no '-' is required, except if the character is + ## itself a '-' + if _utf7_B64CHAR(oc) or ch == u'-': + result.append('-') + inShift = False + result.append(chr(oc)) + else: + bitsleft += 16 + charsleft = (charsleft << 16) | oc + bitsleft = _utf7_ENCODE(result, charsleft, bitsleft) + ## If the next character is special then we dont' need to + ## terminate the shift sequence. If the next character is not + ## a BASE64 character or '-' then the shift sequence will be + ## terminated implicitly and we don't have to insert a '-'. + if bitsleft == 0: + if pos + 1 < size: + ch2 = s[pos + 1] + oc2 = ord(ch2) + + if _utf7_SPECIAL(oc2, encodeSetO, encodeWhiteSpace): + pass + elif _utf7_B64CHAR(oc2) or ch2 == u'-': + result.append('-') + inShift = False + else: + inShift = False + else: + result.append('-') + inShift = False + pos += 1 + + if bitsleft: + result.append(_utf7_TO_BASE64(charsleft << (6 - bitsleft))) + result.append('-') + + return result.build() + +# ____________________________________________________________ +# ascii and latin-1 def str_decode_latin_1(s, size, errors, final=False, errorhandler=None): # latin1 is equivalent to the first 256 ordinals in Unicode. pos = 0 - result = [] - while (pos < size): + result = UnicodeBuilder(size) + while pos < size: result.append(unichr(ord(s[pos]))) pos += 1 - return u"".join(result), pos + return result.build(), pos def str_decode_ascii(s, size, errors, final=False, @@ -336,9 +671,9 @@ if errorhandler is None: errorhandler = raise_unicode_exception_decode # ASCII is equivalent to the first 128 ordinals in Unicode. - result = [] + result = UnicodeBuilder(size) pos = 0 - while pos < len(s): + while pos < size: c = s[pos] if ord(c) < 128: result.append(unichr(ord(c))) @@ -347,55 +682,7 @@ r, pos = errorhandler(errors, "ascii", "ordinal not in range(128)", s, pos, pos + 1) result.append(r) - return u"".join(result), pos - - -# ____________________________________________________________ -# unicode encoding - - -def unicode_encode_utf_8(s, size, errors, errorhandler=None): - assert(size >= 0) - result = [] - i = 0 - while i < size: - ch = ord(s[i]) - i += 1 - if (ch < 0x80): - # Encode ASCII - result.append(chr(ch)) - elif (ch < 0x0800) : - # Encode Latin-1 - result.append(chr((0xc0 | (ch >> 6)))) - result.append(chr((0x80 | (ch & 0x3f)))) - else: - # Encode UCS2 Unicode ordinals - if (ch < 0x10000): - # Special case: check for high surrogate - if (0xD800 <= ch and ch <= 0xDBFF and i != size) : - ch2 = ord(s[i]) - # Check for low surrogate and combine the two to - # form a UCS4 value - if (0xDC00 <= ch2 and ch2 <= 0xDFFF) : - ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000 - i += 1 - _encodeUCS4(result, ch3) - continue - # Fall through: handles isolated high surrogates - result.append((chr((0xe0 | (ch >> 12))))) - result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) - result.append((chr((0x80 | (ch & 0x3f))))) - continue - else: - _encodeUCS4(result, ch) - return "".join(result) - -def _encodeUCS4(result, ch): - # Encode UCS4 Unicode ordinals - result.append((chr((0xf0 | (ch >> 18))))) - result.append((chr((0x80 | ((ch >> 12) & 0x3f))))) - result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) - result.append((chr((0x80 | (ch & 0x3f))))) + return result.build(), pos def unicode_encode_ucs1_helper(p, size, errors, @@ -408,12 +695,12 @@ else: reason = "ordinal not in range(128)" encoding = "ascii" - - if (size == 0): + + if size == 0: return '' - result = [] + result = StringBuilder(size) pos = 0 - while pos < len(p): + while pos < size: ch = p[pos] if ord(ch) < limit: @@ -427,9 +714,9 @@ collend += 1 r, pos = errorhandler(errors, encoding, reason, p, collstart, collend) - result += r # extend 'result' as a list of characters + result.append(r) - return "".join(result) + return result.build() def unicode_encode_latin_1(p, size, errors, errorhandler=None): res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 256) @@ -439,57 +726,479 @@ res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 128) return res +# ____________________________________________________________ +# Charmap -def _STORECHAR(result, CH, byteorder): - hi = chr(((CH) >> 8) & 0xff) - lo = chr((CH) & 0xff) - if byteorder == 'little': - result.append(lo) - result.append(hi) +ERROR_CHAR = u'\ufffe' + + at specialize.argtype(5) +def str_decode_charmap(s, size, errors, final=False, + errorhandler=None, mapping=None): + "mapping can be a rpython dictionary, or a dict-like object." + + # Default to Latin-1 + if mapping is None: + return str_decode_latin_1(s, size, errors, final=final, + errorhandler=errorhandler) + if errorhandler is None: + errorhandler = raise_unicode_exception_decode + if size == 0: + return u'', 0 + + pos = 0 + result = UnicodeBuilder(size) + while pos < size: + ch = s[pos] + + c = mapping.get(ch, ERROR_CHAR) + if c == ERROR_CHAR: + r, pos = errorhandler(errors, "charmap", + "character maps to ", + s, pos, pos + 1) + result.append(r) + continue + result.append(c) + pos += 1 + return result.build(), pos + +def unicode_encode_charmap(s, size, errors, errorhandler=None, + mapping=None): + if mapping is None: + return unicode_encode_latin_1(s, size, errors, + errorhandler=errorhandler) + + if errorhandler is None: + errorhandler = raise_unicode_exception_encode + + if size == 0: + return '' + result = StringBuilder(size) + pos = 0 + while pos < size: + ch = s[pos] + + c = mapping.get(ch, '') + if len(c) == 0: + res, pos = errorhandler(errors, "charmap", + "character maps to ", + s, pos, pos + 1) + for ch2 in res: + c2 = mapping.get(unichr(ord(ch2)), '') + if len(c2) == 0: + errorhandler( + "strict", "charmap", + "character maps to ", + s, pos, pos + 1) + result.append(c2) + continue + result.append(c) + pos += 1 + return result.build() + +# ____________________________________________________________ +# Unicode escape + +hexdigits = "0123456789ABCDEFabcdef" + +def hexescape(builder, s, pos, digits, + encoding, errorhandler, message, errors): + import sys + chr = 0 + if pos + digits > len(s): + message = "end of string in escape sequence" + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-2, len(s)) + builder.append(res) else: - result.append(hi) - result.append(lo) + try: + chr = r_uint(int(s[pos:pos+digits], 16)) + except ValueError: + endinpos = pos + while s[endinpos] in hexdigits: + endinpos += 1 + res, pos = errorhandler(errors, encoding, + message, s, pos-2, endinpos+1) + builder.append(res) + else: + # when we get here, chr is a 32-bit unicode character + if chr <= MAXUNICODE: + builder.append(UNICHR(chr)) + pos += digits + + elif chr <= 0x10ffff: + chr -= 0x10000L + builder.append(unichr(0xD800 + (chr >> 10))) + builder.append(unichr(0xDC00 + (chr & 0x03FF))) + pos += digits + else: + message = "illegal Unicode character" + res, pos = errorhandler(errors, encoding, + message, s, pos-2, pos+digits) + builder.append(res) + return pos + +def str_decode_unicode_escape(s, size, errors, final=False, + errorhandler=False, + unicodedata_handler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode -def unicode_encode_utf_16_helper(s, size, errors, - errorhandler=None, - byteorder='little'): - result = [] - if (byteorder == 'native'): - _STORECHAR(result, 0xFEFF, BYTEORDER) - byteorder = BYTEORDER - if size == 0: - return "" + return u'', 0 - i = 0 - while i < size: - ch = ord(s[i]) - i += 1 - ch2 = 0 - if (ch >= 0x10000) : - ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) - ch = 0xD800 | ((ch-0x10000) >> 10) + builder = UnicodeBuilder(size) + pos = 0 + while pos < size: + ch = s[pos] - _STORECHAR(result, ch, byteorder) - if ch2: - _STORECHAR(result, ch2, byteorder) + # Non-escape characters are interpreted as Unicode ordinals + if ch != '\\': + builder.append(unichr(ord(ch))) + pos += 1 + continue - return "".join(result) + # - Escapes + pos += 1 + if pos >= size: + message = "\\ at end of string" + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, size) + builder.append(res) + continue -def unicode_encode_utf_16(s, size, errors, - errorhandler=None): - return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "native") + ch = s[pos] + pos += 1 + # \x escapes + if ch == '\n': pass + elif ch == '\\': builder.append(u'\\') + elif ch == '\'': builder.append(u'\'') + elif ch == '\"': builder.append(u'\"') + elif ch == 'b' : builder.append(u'\b') + elif ch == 'f' : builder.append(u'\f') + elif ch == 't' : builder.append(u'\t') + elif ch == 'n' : builder.append(u'\n') + elif ch == 'r' : builder.append(u'\r') + elif ch == 'v' : builder.append(u'\v') + elif ch == 'a' : builder.append(u'\a') + elif '0' <= ch <= '7': + x = ord(ch) - ord('0') + if pos < size: + ch = s[pos] + if '0' <= ch <= '7': + pos += 1 + x = (x<<3) + ord(ch) - ord('0') + if pos < size: + ch = s[pos] + if '0' <= ch <= '7': + pos += 1 + x = (x<<3) + ord(ch) - ord('0') + builder.append(unichr(x)) + # hex escapes + # \xXX + elif ch == 'x': + digits = 2 + message = "truncated \\xXX escape" + pos = hexescape(builder, s, pos, digits, + "unicodeescape", errorhandler, message, errors) + + # \uXXXX + elif ch == 'u': + digits = 4 + message = "truncated \\uXXXX escape" + pos = hexescape(builder, s, pos, digits, + "unicodeescape", errorhandler, message, errors) + + # \UXXXXXXXX + elif ch == 'U': + digits = 8 + message = "truncated \\UXXXXXXXX escape" + pos = hexescape(builder, s, pos, digits, + "unicodeescape", errorhandler, message, errors) + + # \N{name} + elif ch == 'N': + message = "malformed \\N character escape" + look = pos + if unicodedata_handler is None: + message = ("\\N escapes not supported " + "(can't load unicodedata module)") + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, size) + builder.append(res) + continue + + if look < size and s[look] == '{': + # look for the closing brace + while look < size and s[look] != '}': + look += 1 + if look < size and s[look] == '}': + # found a name. look it up in the unicode database + message = "unknown Unicode character name" + name = s[pos+1:look] + code = unicodedata_handler.call(name) + if code < 0: + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, look+1) + builder.append(res) + continue + pos = look + 1 + if code <= MAXUNICODE: + builder.append(UNICHR(code)) + else: + code -= 0x10000L + builder.append(unichr(0xD800 + (code >> 10))) + builder.append(unichr(0xDC00 + (code & 0x03FF))) + else: + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, look+1) + builder.append(res) + else: + res, pos = errorhandler(errors, "unicodeescape", + message, s, pos-1, look+1) + builder.append(res) + else: + builder.append(u'\\') + builder.append(unichr(ord(ch))) + return builder.build(), pos -def unicode_encode_utf_16_be(s, size, errors, - errorhandler=None): - return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "big") +def unicode_encode_unicode_escape(s, size, errors, errorhandler=None, quotes=False): + # errorhandler is not used: this function cannot cause Unicode errors + result = StringBuilder(size) + + if quotes: + if s.find(u'\'') != -1 and s.find(u'\"') == -1: + quote = ord('\"') + result.append('u"') + else: + quote = ord('\'') + result.append('u\'') + else: + quote = 0 + if size == 0: + return '' -def unicode_encode_utf_16_le(s, size, errors, - errorhandler=None): - return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "little") + pos = 0 + while pos < size: + ch = s[pos] + oc = ord(ch) + + # Escape quotes + if quotes and (oc == quote or ch == '\\'): + result.append('\\') + result.append(chr(oc)) + pos += 1 + continue + + if 0xD800 <= oc < 0xDC00 and pos + 1 < size: + # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes + pos += 1 + oc2 = ord(s[pos]) + + if 0xDC00 <= oc2 <= 0xDFFF: + ucs = (((oc & 0x03FF) << 10) | (oc2 & 0x03FF)) + 0x00010000 + raw_unicode_escape_helper(result, ucs) + pos += 1 + continue + # Fall through: isolated surrogates are copied as-is + pos -= 1 + + # Map special whitespace to '\t', \n', '\r' + if ch == '\t': + result.append('\\t') + elif ch == '\n': + result.append('\\n') + elif ch == '\r': + result.append('\\r') + elif ch == '\\': + result.append('\\\\') + + # Map non-printable or non-ascii to '\xhh' or '\uhhhh' + elif oc < 32 or oc >= 0x7F: + raw_unicode_escape_helper(result, oc) + + # Copy everything else as-is + else: + result.append(chr(oc)) + pos += 1 + + if quotes: + result.append(chr(quote)) + return result.build() + +# ____________________________________________________________ +# Raw unicode escape + +def str_decode_raw_unicode_escape(s, size, errors, final=False, + errorhandler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode + if size == 0: + return u'', 0 + + result = UnicodeBuilder(size) + pos = 0 + while pos < size: + ch = s[pos] + + # Non-escape characters are interpreted as Unicode ordinals + if ch != '\\': + result.append(unichr(ord(ch))) + pos += 1 + continue + + startinpos = pos + # \u-escapes are only interpreted iff the number of leading + # backslashes is odd + bs = pos + while pos < size: + pos += 1 + if pos == size or s[pos] != '\\': + break + result.append(u'\\') + + # we have a backslash at the end of the string, stop here + if pos >= size: + result.append(u'\\') + break + + if ((pos - bs) & 1 == 0 or + pos >= size or + (s[pos] != 'u' and s[pos] != 'U')): + result.append(u'\\') + result.append(unichr(ord(s[pos]))) + pos += 1 + continue + + if s[pos] == 'u': + digits = 4 + message = "truncated \\uXXXX escape" + else: + digits = 8 + message = "truncated \\UXXXXXXXX escape" + pos += 1 + pos = hexescape(result, s, pos, digits, + "rawunicodeescape", errorhandler, message, errors) + + return result.build(), pos + +def raw_unicode_escape_helper(result, char): + num = hex(char) + if char >= 0x10000: + result.append("\\U") + zeros = 8 + elif char >= 0x100: + result.append("\\u") + zeros = 4 + else: + result.append("\\x") + zeros = 2 + lnum = len(num) + nb = zeros + 2 - lnum # num starts with '0x' + if nb > 0: + result.append_multiple_char('0', nb) + result.append_slice(num, 2, lnum) + +def unicode_encode_raw_unicode_escape(s, size, errors, errorhandler=None): + # errorhandler is not used: this function cannot cause Unicode errors + if size == 0: + return '' + result = StringBuilder(size) + pos = 0 + while pos < size: + oc = ord(s[pos]) + if oc < 0x100: + result.append(chr(oc)) + else: + raw_unicode_escape_helper(result, oc) + pos += 1 + + return result.build() + +# ____________________________________________________________ +# unicode-internal + +def str_decode_unicode_internal(s, size, errors, final=False, + errorhandler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode + if size == 0: + return u'', 0 + + if MAXUNICODE < 65536: + unicode_bytes = 2 + else: + unicode_bytes = 4 + if BYTEORDER == "little": + start = 0 + stop = unicode_bytes + step = 1 + else: + start = unicode_bytes - 1 + stop = -1 + step = -1 + + result = UnicodeBuilder(size // unicode_bytes) + pos = 0 + while pos < size: + if pos > size - unicode_bytes: + res, pos = errorhandler(errors, "unicode_internal", + "truncated input", + s, pos, size) + result.append(res) + if pos > size - unicode_bytes: + break + continue + t = r_uint(0) + h = 0 + for j in range(start, stop, step): + t += r_uint(ord(s[pos + j])) << (h*8) + h += 1 + if t > MAXUNICODE: + res, pos = errorhandler(errors, "unicode_internal", + "unichr(%d) not in range" % (t,), + s, pos, pos + unicode_bytes) + result.append(res) + continue + result.append(unichr(t)) + pos += unicode_bytes + return result.build(), pos + +def unicode_encode_unicode_internal(s, size, errors, errorhandler=None): + if size == 0: + return '' + + if MAXUNICODE < 65536: + unicode_bytes = 2 + else: + unicode_bytes = 4 + + result = StringBuilder(size * unicode_bytes) + pos = 0 + while pos < size: + oc = ord(s[pos]) + if MAXUNICODE < 65536: + if BYTEORDER == "little": + result.append(chr(oc & 0xFF)) + result.append(chr(oc >> 8 & 0xFF)) + else: + result.append(chr(oc >> 8 & 0xFF)) + result.append(chr(oc & 0xFF)) + else: + if BYTEORDER == "little": + result.append(chr(oc & 0xFF)) + result.append(chr(oc >> 8 & 0xFF)) + result.append(chr(oc >> 16 & 0xFF)) + result.append(chr(oc >> 24 & 0xFF)) + else: + result.append(chr(oc >> 24 & 0xFF)) + result.append(chr(oc >> 16 & 0xFF)) + result.append(chr(oc >> 8 & 0xFF)) + result.append(chr(oc & 0xFF)) + pos += 1 + return result.build() # ____________________________________________________________ # MBCS codecs for Windows Modified: pypy/branch/fast-forward/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/test/test_debug.py (original) +++ pypy/branch/fast-forward/pypy/rlib/test/test_debug.py Tue Jul 6 00:00:33 2010 @@ -3,6 +3,7 @@ from pypy.rlib.debug import check_annotation, make_sure_not_resized from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rlib.debug import have_debug_prints +from pypy.rlib.debug import check_nonneg, IntegerCanBeNegative from pypy.rlib import debug from pypy.rpython.test.test_llinterp import interpret @@ -30,6 +31,16 @@ py.test.raises(Error, "interpret(g, [3])") +def test_check_nonneg(): + def f(x): + assert x >= 5 + check_nonneg(x) + interpret(f, [9]) + + def g(x): + check_nonneg(x-1) + py.test.raises(IntegerCanBeNegative, interpret, g, [9]) + def test_make_sure_not_resized(): from pypy.annotation.listdef import TooLateForChange def f(): Modified: pypy/branch/fast-forward/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/fast-forward/pypy/rlib/test/test_runicode.py Tue Jul 6 00:00:33 2010 @@ -5,11 +5,14 @@ def test_unichr(): a = runicode.UNICHR(0xffff) assert a == u'\uffff' - a = runicode.UNICHR(0x10000) - if sys.maxunicode < 0x10000: - assert len(a) == 2 # surrogates + if runicode.MAXUNICODE > 0xffff: + a = runicode.UNICHR(0x10000) + if sys.maxunicode < 0x10000: + assert len(a) == 2 # surrogates + else: + assert len(a) == 1 else: - assert len(a) == 1 + py.test.raises(ValueError, runicode.UNICHR, 0x10000) class UnicodeTests(object): @@ -228,7 +231,7 @@ class TestTranslation(object): def setup_class(cls): - if len(runicode.UNICHR(0x10000)) == 2: + if runicode.MAXUNICODE != sys.maxunicode: py.test.skip("these tests cannot run on the llinterp") def test_utf8(self): Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/rffi.py Tue Jul 6 00:00:33 2010 @@ -642,16 +642,8 @@ allows for the process to be performed without an extra copy. Make sure to call keep_buffer_alive_until_here on the returned values. """ - str_chars_offset = (offsetof(STRTYPE, 'chars') + \ - itemoffsetof(STRTYPE.chars, 0)) - gc_buf = rgc.malloc_nonmovable(STRTYPE, count) - if gc_buf: - realbuf = cast_ptr_to_adr(gc_buf) + str_chars_offset - raw_buf = cast(TYPEP, realbuf) - return raw_buf, gc_buf - else: - raw_buf = lltype.malloc(TYPEP.TO, count, flavor='raw') - return raw_buf, lltype.nullptr(STRTYPE) + raw_buf = lltype.malloc(TYPEP.TO, count, flavor='raw') + return raw_buf, lltype.nullptr(STRTYPE) alloc_buffer._always_inline_ = True # to get rid of the returned tuple # (char*, str, int, int) -> None @@ -794,6 +786,8 @@ return r_wchar_t.BITS/8 if tp is lltype.Float: return 8 + if tp is lltype.SingleFloat: + return 4 assert isinstance(tp, lltype.Number) if tp is lltype.Signed: return ULONG._type.BITS/8 Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/test/test_rffi.py Tue Jul 6 00:00:33 2010 @@ -707,6 +707,7 @@ lltype.UniChar: ctypes.c_wchar, lltype.Char: ctypes.c_ubyte, DOUBLE: ctypes.c_double, + FLOAT: ctypes.c_float, SIGNEDCHAR: ctypes.c_byte, UCHAR: ctypes.c_ubyte, SHORT: ctypes.c_short, Modified: pypy/branch/fast-forward/pypy/rpython/microbench/microbench.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/microbench/microbench.py (original) +++ pypy/branch/fast-forward/pypy/rpython/microbench/microbench.py Tue Jul 6 00:00:33 2010 @@ -3,7 +3,7 @@ import sys import autopath from time import clock -from py.compat import subprocess +import subprocess from pypy.translator.interactive import Translation LOOPS = 10000000 Modified: pypy/branch/fast-forward/pypy/rpython/module/ll_termios.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/module/ll_termios.py (original) +++ pypy/branch/fast-forward/pypy/rpython/module/ll_termios.py Tue Jul 6 00:00:33 2010 @@ -85,7 +85,7 @@ c_struct.c_c_lflag, ispeed, ospeed, cc = attributes try: for i in range(NCCS): - c_struct.c_c_cc[i] = rffi.r_uchar(ord(cc[i])) + c_struct.c_c_cc[i] = rffi.r_uchar(ord(cc[i][0])) error = c_cfsetispeed(c_struct, ispeed) if error == -1: raise termios.error(error, 'tcsetattr failed') Modified: pypy/branch/fast-forward/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/rstr.py (original) +++ pypy/branch/fast-forward/pypy/rpython/rstr.py Tue Jul 6 00:00:33 2010 @@ -80,17 +80,6 @@ # defaults to checking the length return super(AbstractStringRepr, self).rtype_is_true(hop) - def rtype_ord(self, hop): - string_repr = hop.args_r[0].repr - v_str, = hop.inputargs(string_repr) - c_zero = inputconst(Signed, 0) - v_chr = hop.gendirectcall(self.ll.ll_stritem_nonneg, v_str, c_zero) - if string_repr is hop.rtyper.type_system.rstr.string_repr: - return hop.genop('cast_char_to_int', [v_chr], resulttype=Signed) - else: - assert string_repr is hop.rtyper.type_system.rstr.unicode_repr - return hop.genop('cast_unichar_to_int', [v_chr], resulttype=Signed) - def rtype_method_startswith(self, hop): str1_repr, str2_repr = self._str_reprs(hop) v_str, v_value = hop.inputargs(str1_repr, str2_repr) Modified: pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py (original) +++ pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py Tue Jul 6 00:00:33 2010 @@ -131,7 +131,7 @@ s = c * mul res = 0 for i in range(len(s)): - res = res*10 + ord(const(s[i])) - ord(const('0')) + res = res*10 + ord(const(s[i])[0]) - ord(const('0')[0]) c2 = c c2 *= mul res = 10 * res + (c2 == s) @@ -577,7 +577,7 @@ sum = 0 for num in l: if len(num): - sum += ord(num) - ord(const('0')) + sum += ord(num[0]) - ord(const('0')[0]) return sum + len(l) * 100 for i in range(5): res = self.interpret(fn, [i]) Modified: pypy/branch/fast-forward/pypy/tool/release/force-builds.py ============================================================================== --- pypy/branch/fast-forward/pypy/tool/release/force-builds.py (original) +++ pypy/branch/fast-forward/pypy/tool/release/force-builds.py Tue Jul 6 00:00:33 2010 @@ -25,7 +25,7 @@ 'pypy-c-stackless-app-level-linux-x86-32', 'pypy-c-app-level-win-32', 'pypy-c-jit-linux-x86-32', - 'pypy-c-jit-macosx-x86-32', +# 'pypy-c-jit-macosx-x86-32', 'pypy-c-jit-win-x86-32', ] Modified: pypy/branch/fast-forward/pypy/tool/release/make_release.py ============================================================================== --- pypy/branch/fast-forward/pypy/tool/release/make_release.py (original) +++ pypy/branch/fast-forward/pypy/tool/release/make_release.py Tue Jul 6 00:00:33 2010 @@ -16,6 +16,8 @@ from pypy.tool.udir import udir from pypy.tool.release.package import package import tarfile +import os +import shutil BASEURL = 'http://buildbot.pypy.org/nightly/' @@ -47,28 +49,39 @@ to_download = browse_nightly(branch) tmpdir = udir.join('download') tmpdir.ensure(dir=True) - for (kind, platform), (rev, name) in to_download.iteritems(): - if platform == 'win32': - print 'Ignoring %s, windows unsupported' % name - else: - print "Downloading %s at rev %d" % (name, rev) - url = BASEURL + branch + "/" + name - data = urllib2.urlopen(url).read() - tmpdir.join(name).write(data, mode="wb") - t = tarfile.open(str(tmpdir.join(name))) - data = t.extractfile('pypy-c').read() - pypy_c = tmpdir.join('pypy-c') - pypy_c.write(data, mode="wb") - if kind == 'jit': - kind = '' + alltars = [] + try: + os.chdir(str(tmpdir)) + for (kind, platform), (rev, name) in to_download.iteritems(): + if platform == 'win32': + print 'Ignoring %s, windows unsupported' % name else: - kind = '-' + kind - name = 'pypy-%s%s-%s' % (release, kind, platform) - builddir = package(py.path.local(autopath.pypydir).join('..'), - name=name, - override_pypy_c=pypy_c) - print "Build %s/%s.tar.bz2" % (builddir, name) - print "\nLook into %s for packages" % builddir + print "Downloading %s at rev %d" % (name, rev) + url = BASEURL + branch + "/" + name + data = urllib2.urlopen(url).read() + tmpdir.join(name).write(data, mode="wb") + t = tarfile.open(str(tmpdir.join(name))) + dirname = t.getmembers()[0].name + t.extractall(path=str(tmpdir)) + os.system('mv %s %s' % (str(tmpdir.join(dirname)), + str(tmpdir.join('pypy-%s' % release)))) + if kind == 'jit': + kind = '' + else: + kind = '-' + kind + olddir = os.getcwd() + name = 'pypy-%s-%s%s.tar.bz2' % (release, platform, kind) + print "Building %s" % name + t = tarfile.open(name, 'w:bz2') + t.add('pypy-%s' % release) + alltars.append(name) + t.close() + shutil.rmtree(str(tmpdir.join('pypy-1.3'))) + for name in alltars: + print "Uploading %s" % name + os.system('scp %s codespeak.net:/www/pypy.org/htdocs/download' % name) + finally: + os.chdir(olddir) if __name__ == '__main__': if len(sys.argv) != 2: Modified: pypy/branch/fast-forward/pypy/tool/release/package.py ============================================================================== --- pypy/branch/fast-forward/pypy/tool/release/package.py (original) +++ pypy/branch/fast-forward/pypy/tool/release/package.py Tue Jul 6 00:00:33 2010 @@ -57,11 +57,11 @@ for file in ['LICENSE', 'README']: shutil.copy(str(basedir.join(file)), str(pypydir)) pypydir.ensure('include', dir=True) - # we want to put there all *.h from module/cpyext/include + # we want to put there all *.h and *.inl from trunk/include # and from pypy/_interfaces - for n in basedir.join('pypy', 'module', 'cpyext', 'include').listdir('*.h'): - shutil.copy(str(n), str(pypydir.join('include'))) - for n in basedir.join('pypy', '_interfaces').listdir('*.h'): + includedir = basedir.join('include') + headers = includedir.listdir('*.h') + includedir.listdir('*.inl') + for n in headers: shutil.copy(str(n), str(pypydir.join('include'))) pypydir.ensure('bin', dir=True) archive_pypy_c = pypydir.join('bin', rename_pypy_c) Modified: pypy/branch/fast-forward/pypy/tool/release/test/test_package.py ============================================================================== --- pypy/branch/fast-forward/pypy/tool/release/test/test_package.py (original) +++ pypy/branch/fast-forward/pypy/tool/release/test/test_package.py Tue Jul 6 00:00:33 2010 @@ -26,6 +26,16 @@ assert prefix.join('README').check() th = tarfile.open(str(builddir.join('test.tar.bz2'))) assert th.getmember('test/lib_pypy/syslog.py') + + # the headers file could be not there, because they are copied into + # trunk/include only during translation + includedir = py.path.local(pypydir).dirpath().join('include') + def check_include(name): + if includedir.join(name).check(file=True): + assert th.getmember('test/include/%s' % name) + check_include('Python.h') + check_include('modsupport.inl') + check_include('pypy_decl.h') finally: if fake_pypy_c: pypy_c.remove() Modified: pypy/branch/fast-forward/pypy/tool/test/test_killsubprocess.py ============================================================================== --- pypy/branch/fast-forward/pypy/tool/test/test_killsubprocess.py (original) +++ pypy/branch/fast-forward/pypy/tool/test/test_killsubprocess.py Tue Jul 6 00:00:33 2010 @@ -1,5 +1,5 @@ import sys, time -from py.compat import subprocess +import subprocess from pypy.tool.killsubprocess import killsubprocess def waitdead(process): Modified: pypy/branch/fast-forward/pypy/translator/backendopt/test/test_mallocprediction.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/backendopt/test/test_mallocprediction.py (original) +++ pypy/branch/fast-forward/pypy/translator/backendopt/test/test_mallocprediction.py Tue Jul 6 00:00:33 2010 @@ -169,7 +169,7 @@ t, graph = rtype(entrypoint, [int]) total0 = preparation(t, t.graphs, heuristic=heuristic) total = clever_inlining_and_malloc_removal(t) - assert total0 + total == 16 + assert total == 5 # XXX total0 appears to vary def test_richards(): from pypy.translator.goal.richards import entry_point Modified: pypy/branch/fast-forward/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/fast-forward/pypy/translator/c/gcc/trackgcroot.py Tue Jul 6 00:00:33 2010 @@ -17,6 +17,7 @@ class FunctionGcRootTracker(object): skip = 0 + COMMENT = "([#;].*)?" @classmethod def init_regexp(cls): @@ -25,10 +26,10 @@ cls.r_globllabel = re.compile(cls.LABEL+r"=[.][+]%d\s*$"%cls.OFFSET_LABELS) cls.r_insn = re.compile(r"\t([a-z]\w*)\s") - cls.r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+cls.OPERAND+")\s*$") + cls.r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+cls.OPERAND+")\s*" + cls.COMMENT + "$") cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+"),\s*(?P"+cls.OPERAND+")\s*$") - cls.r_jump = re.compile(r"\tj\w+\s+"+cls.LABEL+"\s*$") + cls.r_jump = re.compile(r"\tj\w+\s+"+cls.LABEL+"\s*" + cls.COMMENT + "$") cls.r_jmp_switch = re.compile(r"\tjmp\t[*]"+cls.LABEL+"[(]") cls.r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") @@ -616,7 +617,7 @@ # tail-calls are equivalent to RET for us return InsnRet(self.CALLEE_SAVE_REGISTERS) return InsnStop() - + def register_jump_to(self, label): if not isinstance(self.insns[-1], InsnStop): self.labels[label].previous_insns.append(self.insns[-1]) @@ -641,6 +642,7 @@ self.register_jump_to(label) return [] + visit_jmpl = visit_jmp visit_je = conditional_jump visit_jne = conditional_jump visit_jg = conditional_jump Modified: pypy/branch/fast-forward/pypy/translator/cli/gencli.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/cli/gencli.py (original) +++ pypy/branch/fast-forward/pypy/translator/cli/gencli.py Tue Jul 6 00:00:33 2010 @@ -2,7 +2,7 @@ import shutil import py -from py.compat import subprocess +import subprocess from pypy.config.config import Config from pypy.translator.oosupport.genoo import GenOO from pypy.translator.cli import conftest Modified: pypy/branch/fast-forward/pypy/translator/cli/query.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/cli/query.py (original) +++ pypy/branch/fast-forward/pypy/translator/cli/query.py Tue Jul 6 00:00:33 2010 @@ -2,7 +2,7 @@ import cPickle as pickle import os.path import py -from py.compat import subprocess +import subprocess from pypy.tool.udir import udir from pypy.rpython.ootypesystem import ootype from pypy.translator.cli.rte import Query Modified: pypy/branch/fast-forward/pypy/translator/cli/rte.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/cli/rte.py (original) +++ pypy/branch/fast-forward/pypy/translator/cli/rte.py Tue Jul 6 00:00:33 2010 @@ -7,7 +7,7 @@ import shutil import py -from py.compat import subprocess +import subprocess from pypy.translator.cli.sdk import SDK from pypy.tool.ansi_print import ansi_log log = py.log.Producer("cli") Modified: pypy/branch/fast-forward/pypy/translator/cli/support.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/cli/support.py (original) +++ pypy/branch/fast-forward/pypy/translator/cli/support.py Tue Jul 6 00:00:33 2010 @@ -1,3 +1,4 @@ +import sys import py from pypy.rpython.ootypesystem import ootype from pypy.translator.cli.rte import Support Modified: pypy/branch/fast-forward/pypy/translator/cli/test/runtest.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/cli/test/runtest.py (original) +++ pypy/branch/fast-forward/pypy/translator/cli/test/runtest.py Tue Jul 6 00:00:33 2010 @@ -2,7 +2,7 @@ import platform import py -from py.compat import subprocess +import subprocess from pypy.tool.udir import udir from pypy.translator.translator import TranslationContext from pypy.rpython.test.tool import BaseRtypingTest, OORtypeMixin Modified: pypy/branch/fast-forward/pypy/translator/driver.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/driver.py (original) +++ pypy/branch/fast-forward/pypy/translator/driver.py Tue Jul 6 00:00:33 2010 @@ -729,7 +729,7 @@ 'Compiling JVM source') def copy_jvm_jar(self): - from py.compat import subprocess + import subprocess basename = self.exe_name % self.get_info() root = udir.join('pypy') manifest = self.create_manifest(root) @@ -764,7 +764,7 @@ return filename def create_classlist(self, root, additional_jars=[]): - from py.compat import subprocess + import subprocess # first, uncompress additional jars for jarfile in additional_jars: oldpwd = root.chdir() Modified: pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py Tue Jul 6 00:00:33 2010 @@ -8,19 +8,6 @@ banner = sys.version.splitlines()[0] -def relpath(path): - # force 'path' to be a relative path, for testing purposes - curdir = py.path.local() - p = py.path.local(path) - result = [] - while not p.relto(curdir): - result.append(os.pardir) - if curdir == curdir.dirpath(): - return str(path) # no relative path found, give up - curdir = curdir.dirpath() - result.append(p.relto(curdir)) - return os.path.join(*result) - app_main = os.path.join(autopath.this_dir, os.pardir, 'app_main.py') app_main = os.path.abspath(app_main) @@ -30,7 +17,8 @@ p = udir.join('demo_test_app_main_%d.py' % (_counter,)) _counter += 1 p.write(str(py.code.Source(source))) - return relpath(p) + # return relative path for testing purposes + return py.path.local().bestrelpath(p) demo_script = getscript(""" @@ -495,16 +483,14 @@ # setup code for test_get_library_path # ------------------------------------ from pypy.module.sys.version import CPYTHON_VERSION, PYPY_VERSION - libroot = 'lib/pypy%d.%d' % PYPY_VERSION[:2] cpy_ver = '%d.%d.%d' % CPYTHON_VERSION[:3] goal_dir = os.path.dirname(app_main) # build a directory hierarchy like which contains both bin/pypy-c and # lib/pypy1.2/* - prefix = udir.join('pathtest') + prefix = udir.join('pathtest').ensure(dir=1) fake_exe = prefix.join('bin/pypy-c').ensure(file=1) - pypyxy = prefix.join(libroot).ensure(dir=1) - expected_path = [str(pypyxy.join(subdir).ensure(dir=1)) + expected_path = [str(prefix.join(subdir).ensure(dir=1)) for subdir in ('lib_pypy', 'lib-python/modified-%s' % cpy_ver, 'lib-python/%s' % cpy_ver)] Modified: pypy/branch/fast-forward/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/jvm/genjvm.py (original) +++ pypy/branch/fast-forward/pypy/translator/jvm/genjvm.py Tue Jul 6 00:00:33 2010 @@ -6,7 +6,7 @@ import os import py -from py.compat import subprocess +import subprocess from pypy.tool.udir import udir from pypy.translator.translator import TranslationContext from pypy.translator.oosupport.genoo import GenOO Modified: pypy/branch/fast-forward/pypy/translator/jvm/test/runtest.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/jvm/test/runtest.py (original) +++ pypy/branch/fast-forward/pypy/translator/jvm/test/runtest.py Tue Jul 6 00:00:33 2010 @@ -2,7 +2,7 @@ import platform import py -from py.compat import subprocess +import subprocess from pypy.tool.udir import udir from pypy.rpython.test.tool import BaseRtypingTest, OORtypeMixin from pypy.rpython.lltypesystem.lltype import typeOf Modified: pypy/branch/fast-forward/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/platform/windows.py (original) +++ pypy/branch/fast-forward/pypy/translator/platform/windows.py Tue Jul 6 00:00:33 2010 @@ -168,7 +168,7 @@ # Tell the linker to generate a manifest file temp_manifest = ofile.dirpath().join( ofile.purebasename + '.manifest') - args += ["/MANIFESTFILE:%s" % (temp_manifest,)] + args += ["/MANIFEST", "/MANIFESTFILE:%s" % (temp_manifest,)] self._execute_c_compiler(self.link, args, exe_name) @@ -277,7 +277,7 @@ '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS)') else: m.rule('$(TARGET)', '$(OBJECTS)', - ['$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFESTFILE:$*.manifest', + ['$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST /MANIFESTFILE:$*.manifest', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) @@ -290,7 +290,7 @@ 'int main(int argc, char* argv[]) ' '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); } > $@') m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.obj'], - ['$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFESTFILE:$*.manifest', + ['$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFEST /MANIFESTFILE:$*.manifest', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) From benjamin at codespeak.net Tue Jul 6 00:08:03 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 00:08:03 +0200 (CEST) Subject: [pypy-svn] r75864 - pypy/branch/fast-forward/lib_pypy Message-ID: <20100705220803.9FF9F282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 00:08:02 2010 New Revision: 75864 Removed: pypy/branch/fast-forward/lib_pypy/py Log: kill link From benjamin at codespeak.net Tue Jul 6 00:23:27 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 00:23:27 +0200 (CEST) Subject: [pypy-svn] r75865 - pypy/branch/fast-forward/pypy/rlib/rstruct Message-ID: <20100705222327.924A4282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 00:23:25 2010 New Revision: 75865 Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Log: death to print Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Tue Jul 6 00:23:25 2010 @@ -125,7 +125,6 @@ def pack_float8(result, x): unsigned = float_pack(x, 8) - print unsigned for i in range(8): result.append(chr((unsigned >> (i * 8)) & 0xFF)) From benjamin at codespeak.net Tue Jul 6 00:43:06 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 00:43:06 +0200 (CEST) Subject: [pypy-svn] r75866 - in pypy/branch/fast-forward: lib_pypy/app_test pypy/rlib/rstruct Message-ID: <20100705224306.815CE282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 00:43:04 2010 New Revision: 75866 Removed: pypy/branch/fast-forward/lib_pypy/app_test/ Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Log: kill app_test Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Tue Jul 6 00:43:04 2010 @@ -5,6 +5,7 @@ import math from pypy.rlib import rarithmetic +from pypy.rlib.rarithmetic import r_ulonglong def round_to_nearest(x): @@ -47,7 +48,7 @@ # extract pieces sign = Q >> BITS - 1 - exp = (Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1 + exp = int((Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1) mant = Q & ((1 << MANT_DIG - 1) - 1) if exp == MAX_EXP - MIN_EXP + 2: @@ -120,7 +121,7 @@ assert 0 <= mant < 1 << MANT_DIG - 1 assert 0 <= exp <= MAX_EXP - MIN_EXP + 2 assert 0 <= sign <= 1 - return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant + return r_ulonglong(((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant) def pack_float8(result, x): From benjamin at codespeak.net Tue Jul 6 00:50:52 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 00:50:52 +0200 (CEST) Subject: [pypy-svn] r75867 - pypy/branch/fast-forward/pypy/rlib/rstruct Message-ID: <20100705225052.A8A30282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 00:50:50 2010 New Revision: 75867 Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Log: revert unintended change Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Tue Jul 6 00:50:50 2010 @@ -5,7 +5,6 @@ import math from pypy.rlib import rarithmetic -from pypy.rlib.rarithmetic import r_ulonglong def round_to_nearest(x): @@ -48,7 +47,7 @@ # extract pieces sign = Q >> BITS - 1 - exp = int((Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1) + exp = (Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1 mant = Q & ((1 << MANT_DIG - 1) - 1) if exp == MAX_EXP - MIN_EXP + 2: @@ -121,7 +120,7 @@ assert 0 <= mant < 1 << MANT_DIG - 1 assert 0 <= exp <= MAX_EXP - MIN_EXP + 2 assert 0 <= sign <= 1 - return r_ulonglong(((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant) + return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant def pack_float8(result, x): From benjamin at codespeak.net Tue Jul 6 01:18:48 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 01:18:48 +0200 (CEST) Subject: [pypy-svn] r75868 - pypy/branch/fast-forward/lib_pypy/ctypes_config_cache Message-ID: <20100705231848.4C2AA282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 01:18:46 2010 New Revision: 75868 Modified: pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/hashlib.ctc.py pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/locale.ctc.py pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/pyexpat.ctc.py pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/rebuild.py pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/resource.ctc.py pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/syslog.ctc.py Log: kill stuff Modified: pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/hashlib.ctc.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/hashlib.ctc.py (original) +++ pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/hashlib.ctc.py Tue Jul 6 01:18:46 2010 @@ -3,7 +3,6 @@ Run this to rebuild _hashlib_cache.py. """ -import autopath from ctypes import * from ctypes_configure import configure, dumpcache Modified: pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/locale.ctc.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/locale.ctc.py (original) +++ pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/locale.ctc.py Tue Jul 6 01:18:46 2010 @@ -3,7 +3,6 @@ Run this to rebuild _locale_cache.py. """ -import autopath from ctypes_configure.configure import (configure, ExternalCompilationInfo, ConstantInteger, DefinedConstantInteger, SimpleType, check_eci) from ctypes_configure.dumpcache import dumpcache Modified: pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/pyexpat.ctc.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/pyexpat.ctc.py (original) +++ pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/pyexpat.ctc.py Tue Jul 6 01:18:46 2010 @@ -3,7 +3,6 @@ Run this to rebuild _pyexpat_cache.py. """ -import autopath import ctypes from ctypes import c_char_p, c_int, c_void_p, c_char from ctypes_configure import configure, dumpcache Modified: pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/rebuild.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/rebuild.py (original) +++ pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/rebuild.py Tue Jul 6 01:18:46 2010 @@ -1,7 +1,15 @@ #! /usr/bin/env python # Run this script to rebuild all caches from the *.ctc.py files. -import autopath +# hack: we cannot directly import autopath, as we are outside the pypy +# package. However, we pretend to be inside pypy/tool and manually run it, to +# get the correct path +import os.path +this_dir = os.path.dirname(__file__) +autopath_py = os.path.join(this_dir, '../../pypy/tool/autopath.py') +autopath_py = os.path.abspath(autopath_py) +execfile(autopath_py, dict(__name__='autopath', __file__=autopath_py)) + import os, sys import py Modified: pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/resource.ctc.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/resource.ctc.py (original) +++ pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/resource.ctc.py Tue Jul 6 01:18:46 2010 @@ -3,7 +3,6 @@ Run this to rebuild _resource_cache.py. """ -import autopath from ctypes import sizeof from ctypes_configure.dumpcache import dumpcache Modified: pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/syslog.ctc.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/syslog.ctc.py (original) +++ pypy/branch/fast-forward/lib_pypy/ctypes_config_cache/syslog.ctc.py Tue Jul 6 01:18:46 2010 @@ -3,7 +3,6 @@ Run this to rebuild _syslog_cache.py. """ -import autopath from ctypes_configure.configure import (configure, ExternalCompilationInfo, ConstantInteger, DefinedConstantInteger) from ctypes_configure.dumpcache import dumpcache From benjamin at codespeak.net Tue Jul 6 01:21:04 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 01:21:04 +0200 (CEST) Subject: [pypy-svn] r75869 - in pypy/branch/fast-forward/lib_pypy: . test2 Message-ID: <20100705232104.3A4BA282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 01:21:03 2010 New Revision: 75869 Removed: pypy/branch/fast-forward/lib_pypy/test2/ Modified: pypy/branch/fast-forward/lib_pypy/cmath.py Log: not in trunk Modified: pypy/branch/fast-forward/lib_pypy/cmath.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/cmath.py (original) +++ pypy/branch/fast-forward/lib_pypy/cmath.py Tue Jul 6 01:21:03 2010 @@ -24,6 +24,23 @@ return complex(real, imag) +def phase(x): + x = complex(x) + return math.atan2(x.imag, x.real) + + +def polar(x): + x = complex(x) + phi = math.atan2(x.imag, x.real) + r = abs(x) + return r, phi + + +def rect(r, phi): + r, phi = complex(r), complex(phi) + return complex(r * math.cos(phi), r * math.sin(phi)) + + def acos(x): """acos(x) From benjamin at codespeak.net Tue Jul 6 01:25:46 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 01:25:46 +0200 (CEST) Subject: [pypy-svn] r75870 - pypy/branch/fast-forward/lib_pypy Message-ID: <20100705232546.25945282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 01:25:44 2010 New Revision: 75870 Modified: pypy/branch/fast-forward/lib_pypy/cmath.py Log: revert in-progress change Modified: pypy/branch/fast-forward/lib_pypy/cmath.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/cmath.py (original) +++ pypy/branch/fast-forward/lib_pypy/cmath.py Tue Jul 6 01:25:44 2010 @@ -24,23 +24,6 @@ return complex(real, imag) -def phase(x): - x = complex(x) - return math.atan2(x.imag, x.real) - - -def polar(x): - x = complex(x) - phi = math.atan2(x.imag, x.real) - r = abs(x) - return r, phi - - -def rect(r, phi): - r, phi = complex(r), complex(phi) - return complex(r * math.cos(phi), r * math.sin(phi)) - - def acos(x): """acos(x) From benjamin at codespeak.net Tue Jul 6 01:36:46 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 01:36:46 +0200 (CEST) Subject: [pypy-svn] r75871 - pypy/trunk/lib_pypy Message-ID: <20100705233646.219D8282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 01:36:44 2010 New Revision: 75871 Modified: pypy/trunk/lib_pypy/__init__.py Log: port from trunk Modified: pypy/trunk/lib_pypy/__init__.py ============================================================================== --- pypy/trunk/lib_pypy/__init__.py (original) +++ pypy/trunk/lib_pypy/__init__.py Tue Jul 6 01:36:44 2010 @@ -3,5 +3,6 @@ # Without this check, you would be able to do 'import __init__' from a pypy # prompt +import sys; print sys.path if __name__ != 'lib_pypy': raise ImportError, '__init__' From benjamin at codespeak.net Tue Jul 6 01:39:04 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 01:39:04 +0200 (CEST) Subject: [pypy-svn] r75872 - pypy/branch/fast-forward/lib_pypy Message-ID: <20100705233904.B7178282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 01:39:03 2010 New Revision: 75872 Modified: pypy/branch/fast-forward/lib_pypy/__init__.py Log: port from trunk Modified: pypy/branch/fast-forward/lib_pypy/__init__.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/__init__.py (original) +++ pypy/branch/fast-forward/lib_pypy/__init__.py Tue Jul 6 01:39:03 2010 @@ -1,4 +1,4 @@ # This __init__.py shows up in PyPy's app-level standard library. # Let's try to prevent that confusion... -if __name__ != 'pypy.lib': +if __name__ != 'lib_pypy': raise ImportError, '__init__' From benjamin at codespeak.net Tue Jul 6 01:39:35 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 01:39:35 +0200 (CEST) Subject: [pypy-svn] r75873 - pypy/branch/fast-forward/lib_pypy Message-ID: <20100705233935.12857282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 01:39:33 2010 New Revision: 75873 Modified: pypy/branch/fast-forward/lib_pypy/cmath.py Log: simple new cmath functions Modified: pypy/branch/fast-forward/lib_pypy/cmath.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/cmath.py (original) +++ pypy/branch/fast-forward/lib_pypy/cmath.py Tue Jul 6 01:39:33 2010 @@ -24,6 +24,23 @@ return complex(real, imag) +def phase(x): + x = complex(x) + return math.atan2(x.imag, x.real) + + +def polar(x): + x = complex(x) + phi = math.atan2(x.imag, x.real) + r = abs(x) + return r, phi + + +def rect(r, phi): + r, phi = complex(r), complex(phi) + return complex(r * math.cos(phi), r * math.sin(phi)) + + def acos(x): """acos(x) From benjamin at codespeak.net Tue Jul 6 03:07:01 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 03:07:01 +0200 (CEST) Subject: [pypy-svn] r75874 - in pypy/branch/fast-forward/pypy/rpython: . lltypesystem/module Message-ID: <20100706010701.55A4B282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 03:06:59 2010 New Revision: 75874 Modified: pypy/branch/fast-forward/pypy/rpython/extfuncregistry.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Log: add ll implementations of isinf and isnan Modified: pypy/branch/fast-forward/pypy/rpython/extfuncregistry.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/extfuncregistry.py (original) +++ pypy/branch/fast-forward/pypy/rpython/extfuncregistry.py Tue Jul 6 03:06:59 2010 @@ -10,6 +10,7 @@ from pypy.rpython.ootypesystem.module import ll_math as oo_math from pypy.rpython.module import ll_os from pypy.rpython.module import ll_time +from pypy.rlib import rarithmetic try: import termios except ImportError: @@ -21,10 +22,18 @@ # and are part of math.h for name in ll_math.unary_math_functions: llimpl = getattr(ll_math, 'll_math_%s' % name, None) - register_external(getattr(math, name), [float], float, + f = getattr(math, name) + register_external(f, [float], float, export_name="ll_math.ll_math_%s" % name, sandboxsafe=True, llimpl=llimpl) +register_external(rarithmetic.isinf, [float], bool, + export_name="ll_math.ll_math_isinf", sandboxsafe=True, + llimpl=ll_math.ll_math_isinf) +register_external(rarithmetic.isnan, [float], bool, + export_name="ll_math.ll_math_isnan", sandboxsafe=True, + llimpl=ll_math.ll_math_isnan) + complex_math_functions = [ ('frexp', [float], (float, int)), ('ldexp', [float, int], float), Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Tue Jul 6 03:06:59 2010 @@ -36,6 +36,8 @@ math_fmod = llexternal('fmod', [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE) math_hypot = llexternal(underscore + 'hypot', [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE) +math_isinf = llexternal('isinf', [rffi.DOUBLE], rffi.INT) +math_isnan = llexternal('isnan', [rffi.DOUBLE], rffi.INT) # ____________________________________________________________ # @@ -69,6 +71,14 @@ # Custom implementations +def ll_math_isnan(y): + return bool(math_isnan(y)) + + +def ll_math_isinf(y): + return bool(math_isinf(y)) + + def ll_math_atan2(y, x): """wrapper for atan2 that deals directly with special cases before delegating to the platform libm for the remaining cases. This From benjamin at codespeak.net Tue Jul 6 03:15:30 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 03:15:30 +0200 (CEST) Subject: [pypy-svn] r75875 - pypy/branch/fast-forward/pypy/module/math Message-ID: <20100706011530.4E838282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 03:15:28 2010 New Revision: 75875 Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py Log: placate annotator Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Tue Jul 6 03:15:28 2010 @@ -361,6 +361,7 @@ if partials: hi = partials[-1] j = 0 + lo = 0 for j in range(len(partials) - 2, -1, -1): v = hi y = partials[j] From benjamin at codespeak.net Tue Jul 6 03:25:38 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 03:25:38 +0200 (CEST) Subject: [pypy-svn] r75876 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100706012538.6081F282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 03:25:36 2010 New Revision: 75876 Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py Log: help annotator Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floattype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floattype.py Tue Jul 6 03:25:36 2010 @@ -80,6 +80,7 @@ def descr_fromhex(space, w_cls, s): length = len(s) i = 0 + value = 0.0 while i < length and s[i].isspace(): i += 1 if i == length: From benjamin at codespeak.net Tue Jul 6 04:27:15 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 04:27:15 +0200 (CEST) Subject: [pypy-svn] r75877 - in pypy/branch/fast-forward/pypy: module/exceptions rpython rpython/lltypesystem/module Message-ID: <20100706022715.1B05C282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 04:27:13 2010 New Revision: 75877 Modified: pypy/branch/fast-forward/pypy/module/exceptions/interp_exceptions.py pypy/branch/fast-forward/pypy/rpython/extfuncregistry.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Log: pass correct signature Modified: pypy/branch/fast-forward/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/fast-forward/pypy/module/exceptions/interp_exceptions.py Tue Jul 6 04:27:13 2010 @@ -196,7 +196,7 @@ try: space.delitem(w_dict, space.wrap("message")) except OperationError, e: - if not e.match(space.w_KeyError): + if not e.match(space, space.w_KeyError): raise self.w_message = None Modified: pypy/branch/fast-forward/pypy/rpython/extfuncregistry.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/extfuncregistry.py (original) +++ pypy/branch/fast-forward/pypy/rpython/extfuncregistry.py Tue Jul 6 04:27:13 2010 @@ -33,6 +33,9 @@ register_external(rarithmetic.isnan, [float], bool, export_name="ll_math.ll_math_isnan", sandboxsafe=True, llimpl=ll_math.ll_math_isnan) +register_external(rarithmetic.copysign, [float, float], float, + export_name="ll_math.ll_math_copysign", sandboxsafe=True, + llimpl=ll_math.ll_math_copysign) complex_math_functions = [ ('frexp', [float], (float, int)), Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Tue Jul 6 04:27:13 2010 @@ -79,6 +79,9 @@ return bool(math_isinf(y)) +ll_math_copysign = math_copysign + + def ll_math_atan2(y, x): """wrapper for atan2 that deals directly with special cases before delegating to the platform libm for the remaining cases. This From benjamin at codespeak.net Tue Jul 6 05:53:29 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 05:53:29 +0200 (CEST) Subject: [pypy-svn] r75878 - pypy/branch/fast-forward/pypy/module/math Message-ID: <20100706035329.C6128282BF6@codespeak.net> Author: benjamin Date: Tue Jul 6 05:53:27 2010 New Revision: 75878 Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py Log: correct namespace of isinf Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Tue Jul 6 05:53:27 2010 @@ -347,7 +347,7 @@ not rarithmetic.isnan(original)): raise OperationError(space.w_OverflowError, space.wrap("intermediate overflow")) - if isinf(original): + if rarithmetic.isinf(original): inf_sum += original special_sum += original del partials[:] From fijal at codespeak.net Tue Jul 6 08:19:41 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Jul 2010 08:19:41 +0200 (CEST) Subject: [pypy-svn] r75879 - in pypy/trunk/pypy: jit/backend/llsupport jit/backend/llsupport/test rpython/memory Message-ID: <20100706061941.4070A282B9C@codespeak.net> Author: fijal Date: Tue Jul 6 08:19:39 2010 New Revision: 75879 Modified: pypy/trunk/pypy/jit/backend/llsupport/symbolic.py pypy/trunk/pypy/jit/backend/llsupport/test/test_symbolic.py pypy/trunk/pypy/rpython/memory/lltypelayout.py Log: Support also nolength arrays Modified: pypy/trunk/pypy/jit/backend/llsupport/symbolic.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/symbolic.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/symbolic.py Tue Jul 6 08:19:39 2010 @@ -36,8 +36,11 @@ ofs_length = (llmemory.offsetof(T, T._arrayfld) + llmemory.ArrayLengthOffset(SUBARRAY)) else: + if T._hints.get('nolength', None): + ofs_length = -1 + else: + ofs_length = llmemory.ArrayLengthOffset(T) itemsize = llmemory.sizeof(T.OF) - ofs_length = llmemory.ArrayLengthOffset(T) else: if isinstance(T, lltype.Struct): assert T._arrayfld is not None, "%r is not variable-sized" % (T,) @@ -48,8 +51,11 @@ else: before_array_part = 0 carray = ll2ctypes.get_ctypes_type(T) - assert carray.length.size == WORD - ofs_length = before_array_part + carray.length.offset + if T._hints.get('nolength', None): + ofs_length = -1 + else: + assert carray.length.size == WORD + ofs_length = before_array_part + carray.length.offset basesize = before_array_part + carray.items.offset carrayitem = ll2ctypes.get_ctypes_type(T.OF) itemsize = ctypes.sizeof(carrayitem) Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_symbolic.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_symbolic.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_symbolic.py Tue Jul 6 08:19:39 2010 @@ -61,6 +61,12 @@ assert basesize >= WORD # at least the 'length', maybe some gc headers assert itemsize == WORD assert ofs_length == basesize - WORD + A = rffi.CArray(lltype.Signed) + arraytok = get_array_token(A, translate_support_code) + basesize, itemsize, ofs_length = convert(arraytok) + assert basesize == 0 + assert itemsize == WORD + assert ofs_length == -1 def test_varsized_struct_size(): S1 = lltype.GcStruct('S1', ('parent', S), Modified: pypy/trunk/pypy/rpython/memory/lltypelayout.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/lltypelayout.py (original) +++ pypy/trunk/pypy/rpython/memory/lltypelayout.py Tue Jul 6 08:19:39 2010 @@ -111,6 +111,8 @@ elif isinstance(offset, llmemory.ItemOffset): return sizeof(offset.TYPE) * offset.repeat elif isinstance(offset, llmemory.ArrayItemsOffset): + if offset.TYPE._hints.get('nolength', None): + return 0 return get_fixed_size(lltype.Signed) elif isinstance(offset, llmemory.GCHeaderOffset): return sizeof(offset.gcheaderbuilder.HDR) From fijal at codespeak.net Tue Jul 6 08:46:17 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Jul 2010 08:46:17 +0200 (CEST) Subject: [pypy-svn] r75880 - in pypy/trunk/pypy/jit/backend/llsupport: . test Message-ID: <20100706064617.D8D48282B9C@codespeak.net> Author: fijal Date: Tue Jul 6 08:46:16 2010 New Revision: 75880 Modified: pypy/trunk/pypy/jit/backend/llsupport/descr.py pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py Log: Support descrs of nolength arrays Modified: pypy/trunk/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/descr.py Tue Jul 6 08:46:16 2010 @@ -151,7 +151,6 @@ def repr_of_descr(self): return '<%s>' % self._clsname - class NonGcPtrArrayDescr(BaseArrayDescr): _clsname = 'NonGcPtrArrayDescr' def get_item_size(self, translate_support_code): @@ -161,17 +160,45 @@ _clsname = 'GcPtrArrayDescr' _is_array_of_pointers = True +_CA = rffi.CArray(lltype.Signed) + +class BaseArrayNoLengthDescr(BaseArrayDescr): + def get_base_size(self, translate_support_code): + basesize, _, _ = symbolic.get_array_token(_CA, translate_support_code) + return basesize + + def get_ofs_length(self, translate_support_code): + _, _, ofslength = symbolic.get_array_token(_CA, translate_support_code) + return ofslength + +class NonGcPtrArrayNoLengthDescr(BaseArrayNoLengthDescr): + _clsname = 'NonGcPtrArrayNoLengthDescr' + def get_item_size(self, translate_support_code): + return symbolic.get_size_of_ptr(translate_support_code) + +class GcPtrArrayNoLengthDescr(NonGcPtrArrayNoLengthDescr): + _clsname = 'GcPtrArrayNoLengthDescr' + _is_array_of_pointers = True + def getArrayDescrClass(ARRAY): return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr, NonGcPtrArrayDescr, 'Array', 'get_item_size', '_is_array_of_floats') +def getArrayNoLengthDescrClass(ARRAY): + return getDescrClass(ARRAY.OF, BaseArrayNoLengthDescr, GcPtrArrayNoLengthDescr, + NonGcPtrArrayNoLengthDescr, 'ArrayNoLength', 'get_item_size', + '_is_array_of_floats') + def get_array_descr(gccache, ARRAY): cache = gccache._cache_array try: return cache[ARRAY] except KeyError: - arraydescr = getArrayDescrClass(ARRAY)() + if ARRAY._hints.get('nolength', False): + arraydescr = getArrayNoLengthDescrClass(ARRAY)() + else: + arraydescr = getArrayDescrClass(ARRAY)() # verify basic assumption that all arrays' basesize and ofslength # are equal basesize, itemsize, ofslength = symbolic.get_array_token(ARRAY, False) Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py Tue Jul 6 08:46:16 2010 @@ -140,7 +140,25 @@ assert isinstance(descr2.get_item_size(True), Symbolic) assert isinstance(descr3.get_item_size(True), Symbolic) assert isinstance(descr4.get_item_size(True), Symbolic) - + CA = rffi.CArray(lltype.Signed) + descr = get_array_descr(c0, CA) + assert not descr.is_array_of_floats() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Ptr(lltype.GcStruct('S'))) + descr = get_array_descr(c0, CA) + assert descr.is_array_of_pointers() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Ptr(lltype.Struct('S'))) + descr = get_array_descr(c0, CA) + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Float) + descr = get_array_descr(c0, CA) + assert descr.is_array_of_floats() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 def test_get_call_descr_not_translated(): c0 = GcCache(False) From fijal at codespeak.net Tue Jul 6 08:53:53 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Jul 2010 08:53:53 +0200 (CEST) Subject: [pypy-svn] r75881 - in pypy/trunk/pypy/jit/backend: llsupport x86 Message-ID: <20100706065353.07131282B9C@codespeak.net> Author: fijal Date: Tue Jul 6 08:53:52 2010 New Revision: 75881 Modified: pypy/trunk/pypy/jit/backend/llsupport/llmodel.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py Log: support raw arrays in x86 backend Modified: pypy/trunk/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/llmodel.py Tue Jul 6 08:53:52 2010 @@ -311,6 +311,12 @@ items[itemindex] = newvalue # --- end of GC unsafe code --- + bh_setarrayitem_raw_i = bh_setarrayitem_gc_i + bh_setarrayitem_raw_f = bh_setarrayitem_gc_f + + bh_getarrayitem_raw_i = bh_getarrayitem_gc_i + bh_getarrayitem_raw_f = bh_getarrayitem_gc_f + def bh_strlen(self, string): s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) return len(s.chars) Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Tue Jul 6 08:53:52 2010 @@ -815,6 +815,7 @@ raise NotImplementedError() genop_getarrayitem_gc_pure = genop_getarrayitem_gc + genop_getarrayitem_raw = genop_getarrayitem_gc def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Tue Jul 6 08:53:52 2010 @@ -867,6 +867,7 @@ result_loc = self.force_allocate_reg(op.result) self.Perform(op, [base_loc, ofs_loc, imm(scale), imm(ofs)], result_loc) + consider_getarrayitem_raw = consider_getarrayitem_gc consider_getarrayitem_gc_pure = consider_getarrayitem_gc def consider_int_is_true(self, op, guard_op): From arigo at codespeak.net Tue Jul 6 09:12:01 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 09:12:01 +0200 (CEST) Subject: [pypy-svn] r75882 - pypy/branch/reflex-support/pypy/module/cppyy/test Message-ID: <20100706071201.4A01D282B9C@codespeak.net> Author: arigo Date: Tue Jul 6 09:11:59 2010 New Revision: 75882 Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx (props changed) Log: Kill the svn:executable property on this file. From arigo at codespeak.net Tue Jul 6 09:15:16 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 09:15:16 +0200 (CEST) Subject: [pypy-svn] r75883 - pypy/branch/rsre2/pypy/rlib/rsre/test Message-ID: <20100706071516.ADCCD282B9C@codespeak.net> Author: arigo Date: Tue Jul 6 09:15:15 2010 New Revision: 75883 Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Log: A test showing that the part between REPEAT and UNTIL must lead all possible answers, not just the first. Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Tue Jul 6 09:15:15 2010 @@ -168,3 +168,7 @@ res = rsre.match(r, "abcdedede") assert res.flatten_marks() == [0, 9, 1, 2, 7, 9, 7, 8, 8, 9] assert res.flatten_marks() == [0, 9, 1, 2, 7, 9, 7, 8, 8, 9] + + def test_bug1(self): + r = get_code(r"(?:.+)?B") + assert rsre.match(r, "AB") is not None From hakanardo at codespeak.net Tue Jul 6 09:31:03 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 6 Jul 2010 09:31:03 +0200 (CEST) Subject: [pypy-svn] r75884 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100706073103.4BEFB282B9C@codespeak.net> Author: hakanardo Date: Tue Jul 6 09:31:01 2010 New Revision: 75884 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: tostring, tolist, tofile and tounicode Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Jul 6 09:31:01 2010 @@ -254,8 +254,29 @@ self.descr_fromlist(self.space.wrap(s)) descr_fromunicode.unwrap_spec = ['self', unicode] - - + def descr_tolist(self): + return self.space.newlist([self.space.wrap(i) for i in self.buffer]) + descr_tolist.unwrap_spec = ['self'] + + def descr_tostring(self): + import struct + return self.space.wrap(''.join([struct.pack(self.typecode, i) for i in self.buffer])) + descr_tostring.unwrap_spec = ['self'] + + def descr_tofile(self, w_f): + space=self.space + w_s = space.call_function( + space.getattr(w_f, space.wrap('write')), + self.descr_tostring()) + descr_tofile.unwrap_spec = ['self', W_Root] + + def descr_tounicode(self): + if self.typecode != 'u': + msg = "tounicode() may only be called on type 'u' arrays" + raise OperationError(self.space.w_ValueError, self.space.wrap(msg)) + return self.space.wrap(u''.join([i for i in self.buffer])) + descr_tounicode.unwrap_spec = ['self'] + def descr_itemsize(space, self): return space.wrap(self.itemsize) @@ -268,10 +289,15 @@ __getitem__ = interp2app(W_Array.descr_getitem), __setitem__ = interp2app(W_Array.descr_setitem), itemsize = GetSetProperty(descr_itemsize, cls=W_Array), + fromstring = interp2app(W_Array.descr_fromstring), fromfile = interp2app(W_Array.descr_fromfile), fromlist = interp2app(W_Array.descr_fromlist), fromunicode = interp2app(W_Array.descr_fromunicode), + tostring = interp2app(W_Array.descr_tostring), + tolist = interp2app(W_Array.descr_tolist), + tofile = interp2app(W_Array.descr_tofile), + tounicode = interp2app(W_Array.descr_tounicode), ) Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Tue Jul 6 09:31:01 2010 @@ -263,3 +263,25 @@ assert False except TypeError: pass + + def test_toxxx(self): + a = self.array('i', [1,2,3]) + l = a.tolist() + assert type(l) is list and len(l)==3 + assert a[0] == 1 and a[1] == 2 and a[2] == 3 + + b = self.array('i', a.tostring()) + assert len(b) == 3 and b[0] == 1 and b[1] == 2 and b[2] == 3 + + assert self.array('c', ('h', 'i')).tostring() == 'hi' + a = self.array('i',[0,0,0]) + assert a.tostring() == '\x00'*3*a.itemsize + + from cStringIO import StringIO + f=StringIO() + self.array('c', ('h', 'i')).tofile(f) + assert f.getvalue() == 'hi' + + raises(ValueError, self.array('i').tounicode) + assert self.array('u', unicode('hello')).tounicode() == unicode('hello') + From getxsick at codespeak.net Tue Jul 6 11:10:59 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 6 Jul 2010 11:10:59 +0200 (CEST) Subject: [pypy-svn] r75885 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100706091059.DE2BB282B9C@codespeak.net> Author: getxsick Date: Tue Jul 6 11:10:58 2010 New Revision: 75885 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: free memory to avoid memory leaks (thanks amaury) Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Tue Jul 6 11:10:58 2010 @@ -21,10 +21,13 @@ class _LibHandler(object): def __init__(self, name): + name_ptr = rffi.str2charp(name) try: - self.handler = rdynload.dlopen(rffi.str2charp(name)) + self.handler = rdynload.dlopen(name_ptr) except rdynload.DLOpenError, e: raise OSError('%s: %s', name, e.msg or 'unspecified error') + finally: + rffi.free_charp(name_ptr) class _Get(object): def __init__(self, cpu, lib, func, args_type, res_type='v'): From arigo at codespeak.net Tue Jul 6 11:17:49 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 11:17:49 +0200 (CEST) Subject: [pypy-svn] r75886 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100706091749.B45A5282B9C@codespeak.net> Author: arigo Date: Tue Jul 6 11:17:48 2010 New Revision: 75886 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py Log: Fix rsre by writing it as a generator. Will need fixing to make it RPython. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Tue Jul 6 11:17:48 2010 @@ -107,73 +107,81 @@ mark = mark.prev return -1 +def any(enum): + for _ in enum: + return True + return False + def match(pattern, string, start=0, flags=0): ctx = MatchContext(pattern, string, start, flags) - if sre_match(ctx, 0, start, None): + for match_end, match_marks in sre_match(ctx, 0, start, None): + ctx.match_end = match_end + ctx.match_marks = match_marks return ctx return None def sre_match(ctx, ppos, ptr, marks): + """Enumerate answers. Usually we only need the first one, but + there is the case of REPEAT...UNTIL where we need all of them.""" while True: op = ctx.pat(ppos) ppos += 1 if op == OPCODE_FAILURE: - return False + return if (op == OPCODE_SUCCESS or op == OPCODE_MAX_UNTIL or op == OPCODE_MIN_UNTIL): - ctx.match_end = ptr - ctx.match_marks = marks - return True + yield (ptr, marks) + return elif op == OPCODE_ANY: # match anything (except a newline) # if ptr >= ctx.end or rsre_char.is_linebreak(ctx.str(ptr)): - return False + return ptr += 1 elif op == OPCODE_ANY_ALL: # match anything # if ptr >= ctx.end: - return False + return ptr += 1 elif op == OPCODE_ASSERT: # assert subpattern # <0=skip> <1=back> ptr1 = ptr - ctx.pat(ppos+1) - if ptr1 < 0 or not sre_match(ctx, ppos + 2, ptr1, marks): - return False + if ptr1 < 0 or not any(sre_match(ctx, ppos + 2, ptr1, marks)): + return ppos += ctx.pat(ppos) elif op == OPCODE_ASSERT_NOT: # assert not subpattern # <0=skip> <1=back> ptr1 = ptr - ctx.pat(ppos+1) - if ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks): - return False + if ptr1 >= 0 and any(sre_match(ctx, ppos + 2, ptr1, marks)): + return ppos += ctx.pat(ppos) elif op == OPCODE_AT: # match at given position (e.g. at beginning, at boundary, etc.) # if not sre_at(ctx, ctx.pat(ppos), ptr): - return False + return ppos += 1 elif op == OPCODE_BRANCH: # alternation # <0=skip> code ... while ctx.pat(ppos): - if sre_match(ctx, ppos + 1, ptr, marks): - return True + for answer in sre_match(ctx, ppos + 1, ptr, marks): + yield answer ppos += ctx.pat(ppos) - return False + return #elif op == OPCODE_CATEGORY: # seems to be never produced @@ -184,13 +192,13 @@ gid = ctx.pat(ppos) * 2 startptr = find_mark(marks, gid) if startptr < 0: - return False + return endptr = find_mark(marks, gid + 1) if endptr < startptr: # also includes the case "endptr == -1" - return False + return for i in range(startptr, endptr): if ptr >= ctx.end or ctx.str(ptr) != ctx.str(i): - return False + return ptr += 1 ppos += 1 @@ -200,13 +208,13 @@ gid = ctx.pat(ppos) * 2 startptr = find_mark(marks, gid) if startptr < 0: - return False + return endptr = find_mark(marks, gid + 1) if endptr < startptr: # also includes the case "endptr == -1" - return False + return for i in range(startptr, endptr): if ptr >= ctx.end or ctx.lowstr(ptr) != ctx.lowstr(i): - return False + return ptr += 1 ppos += 1 @@ -216,7 +224,7 @@ if ptr >= ctx.end or not rsre_char.check_charset(ctx.pattern, ppos+1, ctx.str(ptr)): - return False + return ppos += ctx.pat(ppos) ptr += 1 @@ -226,7 +234,7 @@ if ptr >= ctx.end or not rsre_char.check_charset(ctx.pattern, ppos+1, ctx.lowstr(ptr)): - return False + return ppos += ctx.pat(ppos) ptr += 1 @@ -234,7 +242,7 @@ # optimization info block # <0=skip> <1=flags> <2=min> ... if (ctx.end - ptr) < ctx.pat(ppos+2): - return False + return ppos += ctx.pat(ppos) elif op == OPCODE_JUMP: @@ -244,7 +252,7 @@ # match literal string # if ptr >= ctx.end or ctx.str(ptr) != ctx.pat(ppos): - return False + return ppos += 1 ptr += 1 @@ -252,7 +260,7 @@ # match literal string, ignoring case # if ptr >= ctx.end or ctx.lowstr(ptr) != ctx.pat(ppos): - return False + return ppos += 1 ptr += 1 @@ -267,7 +275,7 @@ # match if it's not a literal string # if ptr >= ctx.end or ctx.str(ptr) == ctx.pat(ppos): - return False + return ppos += 1 ptr += 1 @@ -275,7 +283,7 @@ # match if it's not a literal string, ignoring case # if ptr >= ctx.end or ctx.lowstr(ptr) == ctx.pat(ppos): - return False + return ppos += 1 ptr += 1 @@ -286,15 +294,8 @@ itemppos = ppos + 3 # FIXME: we probably need to deal with zero-width matches in here.. - # first match 'min' repetitions of 'item' + # get the minimum and maximum number of repetitions allowed min = ctx.pat(ppos+1) - for i in range(min): - if not sre_match(ctx, itemppos, ptr, marks): - return False - ptr = ctx.match_end - marks = ctx.match_marks - - # get the maximum number of repetitions allowed max = ctx.pat(ppos+2) # decode the later UNTIL operator to see if it is actually @@ -307,42 +308,64 @@ # possible, followed by the 'tail'. we do this by # remembering each state for each possible number of # 'item' matching. - pending_ptr = [] - pending_marks = [] - # match as many repetitions of 'item' as possible, - # bounded by 'max' - count = min - while max == 65535 or count < max: - if not sre_match(ctx, itemppos, ptr, marks): + pending = [] + # + while True: + if max == 65535 or len(pending) < max: + # try to match one more 'item' + enum = sre_match(ctx, itemppos, ptr, marks) + else: + enum = iter(()) # 'max' reached, no more matches + + while True: + for next in enum: + # matched one more 'item'. record it and continue + pending.append((ptr, marks, enum)) + ptr, marks = next + break + else: + # 'item' no longer matches. + if len(pending) >= min: + # try to match 'tail' if we have enough 'item' + for answer in sre_match(ctx, tailppos, + ptr, marks): + yield answer + if len(pending) == 0: + return + ptr, marks, enum = pending.pop() + continue break - pending_ptr.append(ptr) - pending_marks.append(marks) - ptr = ctx.match_end - marks = ctx.match_marks - count += 1 - # for each pending_ptr in the chain, try to match 'tail'; - # return the first match found. 'ptr' is currently set - # to where the last match of 'item' failed. - while not sre_match(ctx, tailppos, ptr, marks): - if len(pending_ptr) == 0: - return False - ptr = pending_ptr.pop() - marks = pending_marks.pop() - return True elif op == OPCODE_MIN_UNTIL: # first try to match the 'tail', and if it fails, try # to match one more 'item' and try again - count = min - while not sre_match(ctx, tailppos, ptr, marks): - if max != 65535 and count >= max: - return False - if not sre_match(ctx, itemppos, ptr, marks): - return False - ptr = ctx.match_end - marks = ctx.match_marks - count += 1 - return True + pending = [] + # + while True: + # try to match 'tail' if we have enough 'item' + if len(pending) >= min: + for answer in sre_match(ctx, tailppos, ptr, marks): + yield answer + + if max == 65535 or len(pending) < max: + # try to match one more 'item' + enum = sre_match(ctx, itemppos, ptr, marks) + else: + enum = iter(()) # 'max' reached, no more matches + + while True: + for next in enum: + # matched one more 'item'. record it and continue + pending.append((ptr, marks, enum)) + ptr, marks = next + break + else: + # 'item' no longer matches. + if len(pending) == 0: + return + ptr, marks, enum = pending.pop() + continue + break else: raise AssertionError("missing UNTIL after REPEAT") @@ -357,17 +380,17 @@ start = ptr minptr = start + ctx.pat(ppos+1) if minptr > ctx.end: - return False # cannot match + return # cannot match ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2)) # when we arrive here, ptr points to the tail of the target # string. check if the rest of the pattern matches, # and backtrack if not. nextppos = ppos + ctx.pat(ppos) while ptr >= minptr: - if sre_match(ctx, nextppos, ptr, marks): - return True + for answer in sre_match(ctx, nextppos, ptr, marks): + yield answer ptr -= 1 - return False + return elif op == OPCODE_MIN_REPEAT_ONE: # match repeated sequence (minimizing regexp). @@ -381,11 +404,11 @@ if min > 0: minptr = ptr + min if minptr > ctx.end: - return False # cannot match + return # cannot match # count using pattern min as the maximum ptr = find_repetition_end(ctx, ppos+3, ptr, min) if ptr < minptr: - return False # did not match minimum number of times + return # did not match minimum number of times maxptr = ctx.end max = ctx.pat(ppos+2) @@ -395,13 +418,13 @@ maxptr = maxptr1 nextppos = ppos + ctx.pat(ppos) while ptr <= maxptr: - if sre_match(ctx, nextppos, ptr, marks): - return True + for answer in sre_match(ctx, nextppos, ptr, marks): + yield answer ptr1 = find_repetition_end(ctx, ppos+3, ptr, 1) if ptr1 == ptr: break ptr = ptr1 - return False + return else: assert 0, "bad pattern code %d" % op Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Tue Jul 6 11:17:48 2010 @@ -170,5 +170,21 @@ assert res.flatten_marks() == [0, 9, 1, 2, 7, 9, 7, 8, 8, 9] def test_bug1(self): + # REPEAT_ONE instead REPEAT r = get_code(r"(?:.+)?B") assert rsre.match(r, "AB") is not None + r = get_code(r"(?:AA+?)+B") + assert rsre.match(r, "AAAB") is not None + r = get_code(r"(?:AA+)+?B") + assert rsre.match(r, "AAAB") is not None + r = get_code(r"(?:AA+?)+?B") + assert rsre.match(r, "AAAB") is not None + # REPEAT instead REPEAT + r = get_code(r"(?:(?:xy)+)?B") + assert rsre.match(r, "xyB") is not None + r = get_code(r"(?:xy(?:xy)+?)+B") + assert rsre.match(r, "xyxyxyB") is not None + r = get_code(r"(?:xy(?:xy)+)+?B") + assert rsre.match(r, "xyxyxyB") is not None + r = get_code(r"(?:xy(?:xy)+?)+?B") + assert rsre.match(r, "xyxyxyB") is not None Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py Tue Jul 6 11:17:48 2010 @@ -14,10 +14,10 @@ def test_max_until_0_65535(self): r_code2 = get_code(r'(?:xy)*xy') - res = rsre.match(r_code2, 'def') - assert res is None - res = rsre.match(r_code2, 'xydef') - assert res is not None + #res = rsre.match(r_code2, 'def') + #assert res is None + #res = rsre.match(r_code2, 'xydef') + #assert res is not None res = rsre.match(r_code2, 'xyxyxydef') assert res is not None res = rsre.match(r_code2, '' + 'xy'*1000 + 'def') From arigo at codespeak.net Tue Jul 6 11:19:35 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 11:19:35 +0200 (CEST) Subject: [pypy-svn] r75887 - pypy/branch/rsre2/pypy/rlib/rsre/test Message-ID: <20100706091935.AAC76282B9C@codespeak.net> Author: arigo Date: Tue Jul 6 11:19:34 2010 New Revision: 75887 Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Log: Typo. Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Tue Jul 6 11:19:34 2010 @@ -170,7 +170,7 @@ assert res.flatten_marks() == [0, 9, 1, 2, 7, 9, 7, 8, 8, 9] def test_bug1(self): - # REPEAT_ONE instead REPEAT + # REPEAT_ONE inside REPEAT r = get_code(r"(?:.+)?B") assert rsre.match(r, "AB") is not None r = get_code(r"(?:AA+?)+B") @@ -179,7 +179,7 @@ assert rsre.match(r, "AAAB") is not None r = get_code(r"(?:AA+?)+?B") assert rsre.match(r, "AAAB") is not None - # REPEAT instead REPEAT + # REPEAT inside REPEAT r = get_code(r"(?:(?:xy)+)?B") assert rsre.match(r, "xyB") is not None r = get_code(r"(?:xy(?:xy)+?)+B") From arigo at codespeak.net Tue Jul 6 11:31:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 11:31:13 +0200 (CEST) Subject: [pypy-svn] r75888 - pypy/branch/rsre2/pypy/rlib/rsre/test Message-ID: <20100706093113.A592F282B9C@codespeak.net> Author: arigo Date: Tue Jul 6 11:31:12 2010 New Revision: 75888 Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py Log: Skip these two tests. Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py Tue Jul 6 11:31:12 2010 @@ -266,8 +266,8 @@ # Test symbolic groups ('(?Paaa)a', 'aaaa', SYNTAX_ERROR), - ('(?Paaa)a', 'aaaa', SUCCEED, 'found+"-"+id', 'aaaa-aaa'), - ('(?Paa)(?P=id)', 'aaaa', SUCCEED, 'found+"-"+id', 'aaaa-aa'), + #('(?Paaa)a', 'aaaa', SUCCEED, 'found+"-"+id', 'aaaa-aaa'), + #('(?Paa)(?P=id)', 'aaaa', SUCCEED, 'found+"-"+id', 'aaaa-aa'), ('(?Paa)(?P=xd)', 'aaaa', SYNTAX_ERROR), # Test octal escapes/memory references From getxsick at codespeak.net Tue Jul 6 11:41:08 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 6 Jul 2010 11:41:08 +0200 (CEST) Subject: [pypy-svn] r75889 - pypy/trunk/pypy/module/_locale Message-ID: <20100706094108.810C3282B9C@codespeak.net> Author: getxsick Date: Tue Jul 6 11:41:07 2010 New Revision: 75889 Modified: pypy/trunk/pypy/module/_locale/interp_locale.py Log: fix memory leaks Modified: pypy/trunk/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/trunk/pypy/module/_locale/interp_locale.py (original) +++ pypy/trunk/pypy/module/_locale/interp_locale.py Tue Jul 6 11:41:07 2010 @@ -132,7 +132,13 @@ space.is_true(space.isinstance(w_s2, space.w_str)): s1, s2 = space.str_w(w_s1), space.str_w(w_s2) - return space.wrap(_strcoll(rffi.str2charp(s1), rffi.str2charp(s2))) + s1_c = rffi.str2charp(s1) + s2_c = rffi.str2charp(s2) + try: + return space.wrap(_strcoll(s1_c, s2_c)) + finally: + rffi.free_charp(s1_c) + rffi.free_charp(s2_c) #if not space.is_true(space.isinstance(w_s1, space.w_unicode)) and \ # not space.is_true(space.isinstance(w_s2, space.w_unicode)): @@ -143,7 +149,12 @@ s1_c = rffi.unicode2wcharp(s1) s2_c = rffi.unicode2wcharp(s2) - result = _wcscoll(s1_c, s2_c) + try: + result = _wcscoll(s1_c, s2_c) + finally: + rffi.free_wcharp(s1_c) + rffi.free_wcharp(s2_c) + return space.wrap(result) strcoll.unwrap_spec = [ObjSpace, W_Root, W_Root] @@ -156,13 +167,21 @@ n1 = len(s) + 1 buf = lltype.malloc(rffi.CCHARP.TO, n1, flavor="raw", zero=True) - n2 = _strxfrm(buf, rffi.str2charp(s), n1) + 1 + s_c = rffi.str2charp(s) + try: + n2 = _strxfrm(buf, s_c, n1) + 1 + finally: + rffi.free_charp(s_c) if n2 > n1: # more space needed lltype.free(buf, flavor="raw") buf = lltype.malloc(rffi.CCHARP.TO, intmask(n2), flavor="raw", zero=True) - _strxfrm(buf, rffi.str2charp(s), n2) + s_c = rffi.str2charp(s) + try: + _strxfrm(buf, s_c, n2) + finally: + rffi.free_charp(s_c) val = rffi.charp2str(buf) lltype.free(buf, flavor="raw") @@ -194,7 +213,11 @@ def gettext(space, msg): """gettext(msg) -> string Return translation of msg.""" - return space.wrap(rffi.charp2str(_gettext(rffi.str2charp(msg)))) + msg_c = rffi.str2charp(msg) + try: + return space.wrap(rffi.charp2str(_gettext(msg_c))) + finally: + rffi.free_charp(msg_c) gettext.unwrap_spec = [ObjSpace, str] @@ -205,10 +228,20 @@ Return translation of msg in domain.""" if space.is_w(w_domain, space.w_None): domain = None - result = _dgettext(domain, rffi.str2charp(msg)) + msg_c = rffi.str2charp(msg) + try: + result = _dgettext(domain, msg_c) + finally: + rffi.free_charp(msg_c) else: domain = space.str_w(w_domain) - result = _dgettext(rffi.str2charp(domain), rffi.str2charp(msg)) + domain_c = rffi.str2charp(domain) + msg_c = rffi.str2charp(msg) + try: + result = _dgettext(domain_c, msg_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(msg_c) return space.wrap(rffi.charp2str(result)) @@ -223,12 +256,21 @@ if space.is_w(w_domain, space.w_None): domain = None - result = _dcgettext(domain, rffi.str2charp(msg), - rffi.cast(rffi.INT, category)) + msg_c = rffi.str2charp(msg) + try: + result = _dcgettext(domain, msg_c, rffi.cast(rffi.INT, category)) + finally: + rffi.free_charp(msg_c) else: domain = space.str_w(w_domain) - result = _dcgettext(rffi.str2charp(domain), rffi.str2charp(msg), - rffi.cast(rffi.INT, category)) + domain_c = rffi.str2charp(domain) + msg_c = rffi.str2charp(msg) + try: + result = _dcgettext(domain_c, msg_c, + rffi.cast(rffi.INT, category)) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(msg_c) return space.wrap(rffi.charp2str(result)) @@ -246,7 +288,11 @@ result = _textdomain(domain) else: domain = space.str_w(w_domain) - result = _textdomain(rffi.str2charp(domain)) + domain_c = rffi.str2charp(domain) + try: + result = _textdomain(domain_c) + finally: + rffi.free_charp(domain_c) return space.wrap(rffi.charp2str(result)) @@ -261,11 +307,20 @@ if space.is_w(w_dir, space.w_None): dir = None - dirname = _bindtextdomain(rffi.str2charp(domain), dir) + domain_c = rffi.str2charp(domain) + try: + dirname = _bindtextdomain(domain_c, dir) + finally: + rffi.free_charp(domain_c) else: dir = space.str_w(w_dir) - dirname = _bindtextdomain(rffi.str2charp(domain), - rffi.str2charp(dir)) + domain_c = rffi.str2charp(domain) + dir_c = rffi.str2charp(dir) + try: + dirname = _bindtextdomain(domain_c, dir_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(dir_c) if not dirname: errno = rposix.get_errno() @@ -284,12 +339,20 @@ if space.is_w(w_codeset, space.w_None): codeset = None - result = _bind_textdomain_codeset( - rffi.str2charp(domain), codeset) + domain_c = rffi.str2charp(domain) + try: + result = _bind_textdomain_codeset(domain_c, codeset) + finally: + rffi.free_charp(domain_c) else: codeset = space.str_w(w_codeset) - result = _bind_textdomain_codeset(rffi.str2charp(domain), - rffi.str2charp(codeset)) + domain_c = rffi.str2charp(domain) + codeset_c = rffi.str2charp(codeset) + try: + result = _bind_textdomain_codeset(domain_c, codeset_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(codeset_c) if not result: return space.w_None From arigo at codespeak.net Tue Jul 6 11:41:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 11:41:24 +0200 (CEST) Subject: [pypy-svn] r75890 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100706094124.E582C282B9C@codespeak.net> Author: arigo Date: Tue Jul 6 11:41:23 2010 New Revision: 75890 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py Log: Fix the last XXXes. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Tue Jul 6 11:41:23 2010 @@ -91,6 +91,24 @@ return None return self.string[frm:to] + def at_boundary(self, ptr, word_checker): + if self.end == 0: + return False + prevptr = ptr - 1 + that = prevptr >= 0 and word_checker(self.str(prevptr)) + this = ptr < self.end and word_checker(self.str(ptr)) + return this != that + at_boundary._annspecialcase_ = 'specialize:arg(2)' + + def at_non_boundary(self, ptr, word_checker): + if self.end == 0: + return False + prevptr = ptr - 1 + that = prevptr >= 0 and word_checker(self.str(prevptr)) + this = ptr < self.end and word_checker(self.str(ptr)) + return this == that + at_non_boundary._annspecialcase_ = 'specialize:arg(2)' + class Mark(object): _immutable_ = True @@ -483,7 +501,7 @@ ptr += 1 else: - assert 0, "XXX %d" % op + raise NotImplementedError("rsre.find_repetition_end[%d]" % op) return ptr @@ -513,20 +531,10 @@ return prevptr < 0 or rsre_char.is_linebreak(ctx.str(prevptr)) elif atcode == AT_BOUNDARY: - if ctx.end == 0: - return False - prevptr = ptr - 1 - that = prevptr >= 0 and rsre_char.is_word(ctx.str(prevptr)) - this = ptr < ctx.end and rsre_char.is_word(ctx.str(ptr)) - return this != that + return ctx.at_boundary(ptr, rsre_char.is_word) elif atcode == AT_NON_BOUNDARY: - if ctx.end == 0: - return False - prevptr = ptr - 1 - that = prevptr >= 0 and rsre_char.is_word(ctx.str(prevptr)) - this = ptr < ctx.end and rsre_char.is_word(ctx.str(ptr)) - return this == that + return ctx.at_non_boundary(ptr, rsre_char.is_word) elif atcode == AT_END: remaining_chars = ctx.end - ptr @@ -540,15 +548,15 @@ return ptr == ctx.end elif atcode == AT_LOC_BOUNDARY: - XXX + return ctx.at_boundary(ptr, rsre_char.is_loc_word) elif atcode == AT_LOC_NON_BOUNDARY: - XXX + return ctx.at_non_boundary(ptr, rsre_char.is_loc_word) elif atcode == AT_UNI_BOUNDARY: - XXX + return ctx.at_boundary(ptr, rsre_char.is_uni_word) elif atcode == AT_UNI_NON_BOUNDARY: - XXX + return ctx.at_non_boundary(ptr, rsre_char.is_uni_word) return False Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py Tue Jul 6 11:41:23 2010 @@ -669,6 +669,6 @@ tests.extend([ # bug 410271: \b broken under locales (r'\b.\b', 'a', SUCCEED, 'found', 'a'), - (r'(?u)\b.\b', u, SUCCEED, 'found', u), - (r'(?u)\w', u, SUCCEED, 'found', u), + #(r'(?u)\b.\b', u, SUCCEED, 'found', u), + #(r'(?u)\w', u, SUCCEED, 'found', u), ]) From hakanardo at codespeak.net Tue Jul 6 11:43:33 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 6 Jul 2010 11:43:33 +0200 (CEST) Subject: [pypy-svn] r75891 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100706094333.9F62A282B9C@codespeak.net> Author: hakanardo Date: Tue Jul 6 11:43:32 2010 New Revision: 75891 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: failing to compile Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Jul 6 11:43:32 2010 @@ -6,6 +6,7 @@ from pypy.rlib.jit import dont_look_inside from pypy.rlib import rgc from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.rstruct.runpack import runpack FloatArray = lltype.GcArray(lltype.Float) @@ -54,6 +55,7 @@ self.bytes = 4 else: self.bytes = rffi.sizeof(itemtype) + self.arraytype = lltype.GcArray(itemtype) self.unwrap = unwrap self.signed = signed self.canoverflow = canoverflow @@ -84,6 +86,7 @@ if typecode not in 'cbBuhHiIlLfd': msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' raise OperationError(space.w_ValueError, space.wrap(msg)) + typecode = typecode[0] self.space = space self.typecode = typecode self.len = 0 @@ -124,7 +127,9 @@ return rffi.cast(tc.itemtype, item) def setlen(self, size): - new_buffer = lltype.malloc(lltype.GcArray(types[self.typecode].itemtype), size) + for tc in unroll_typecodes: + if self.typecode == tc: + new_buffer = lltype.malloc(types[tc].arraytype, size) for i in range(self.len): new_buffer[i] = self.buffer[i] self.buffer = new_buffer @@ -201,7 +206,6 @@ descr_len.unwrap_spec = ['self'] def descr_fromstring(self, s): - import struct if len(s)%self.itemsize !=0: msg = 'string length not a multiple of item size' raise OperationError(self.space.w_ValueError, self.space.wrap(msg)) @@ -210,8 +214,10 @@ self.setlen(oldlen + new) for i in range(new): p = i * self.itemsize - item=struct.unpack(self.typecode, s[p:p + self.itemsize])[0] - self.buffer[oldlen + i]=self.item_w(self.space.wrap(item)) + for tc in unroll_typecodes: + if self.typecode == tc: + item=runpack(tc, s[p:p + self.itemsize]) + self.buffer[oldlen + i]=self.item_w(self.space.wrap(item)) descr_fromstring.unwrap_spec = ['self', str] def descr_fromfile(self, w_f, n): @@ -259,8 +265,10 @@ descr_tolist.unwrap_spec = ['self'] def descr_tostring(self): - import struct - return self.space.wrap(''.join([struct.pack(self.typecode, i) for i in self.buffer])) + #import struct + #return self.space.wrap(''.join([struct.pack(self.typecode, i) for i in self.buffer])) + return self.space.wrap('FIXME') + descr_tostring.unwrap_spec = ['self'] def descr_tofile(self, w_f): From arigo at codespeak.net Tue Jul 6 12:22:12 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 12:22:12 +0200 (CEST) Subject: [pypy-svn] r75892 - pypy/branch/rsre2/pypy/rlib/rsre_gen.py Message-ID: <20100706102212.4A5B6282B9C@codespeak.net> Author: arigo Date: Tue Jul 6 12:22:10 2010 New Revision: 75892 Added: pypy/branch/rsre2/pypy/rlib/rsre_gen.py/ - copied from r75891, pypy/branch/rsre2/pypy/rlib/rsre/ Log: Copy the current rsre.py as rsre_gen.py, using generators, before I go ahead and write tons of cruft to make it RPython. From arigo at codespeak.net Tue Jul 6 12:24:28 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 12:24:28 +0200 (CEST) Subject: [pypy-svn] r75893 - pypy/branch/rsre2/pypy/rlib/rsre_gen.py Message-ID: <20100706102428.6129B282B9C@codespeak.net> Author: arigo Date: Tue Jul 6 12:24:26 2010 New Revision: 75893 Removed: pypy/branch/rsre2/pypy/rlib/rsre_gen.py/ Log: Oups, nonsense copying. From arigo at codespeak.net Tue Jul 6 12:25:01 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 12:25:01 +0200 (CEST) Subject: [pypy-svn] r75894 - pypy/branch/rsre2/pypy/rlib/rsre Message-ID: <20100706102501.61699282B9C@codespeak.net> Author: arigo Date: Tue Jul 6 12:24:59 2010 New Revision: 75894 Added: pypy/branch/rsre2/pypy/rlib/rsre/rsre_gen.py - copied unchanged from r75893, pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Log: Copy the current rsre.py as rsre_gen.py, using generators, before I go ahead and write tons of cruft to make it RPython. From cfbolz at codespeak.net Tue Jul 6 12:37:25 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Jul 2010 12:37:25 +0200 (CEST) Subject: [pypy-svn] r75895 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src test Message-ID: <20100706103725.9934C282B9C@codespeak.net> Author: cfbolz Date: Tue Jul 6 12:37:23 2010 New Revision: 75895 Added: pypy/branch/reflex-support/pypy/module/cppyy/converter.py (contents, props changed) Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Log: (antocuni, cfbolz, wlav, arigo around): start going in the direction of supporting overloading. support returning doubles from static methods. Added: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Tue Jul 6 12:37:23 2010 @@ -0,0 +1,27 @@ +from pypy.rpython.lltypesystem import rffi, lltype + +class TypeConverter(object): + def convert_argument(self, space, w_obj): + raise NotImplementedError("abstract base class") + + +class IntConverter(TypeConverter): + def convert_argument(self, space, w_obj): + arg = space.int_w(w_obj) + x = lltype.malloc(rffi.LONGP.TO, 1, flavor='raw') + x[0] = arg + return rffi.cast(rffi.VOIDP, x) + +class DoubleConverter(TypeConverter): + def convert_argument(self, space, w_obj): + arg = space.float_w(w_obj) + x = lltype.malloc(rffi.DOUBLEP.TO, 1, flavor='raw') + x[0] = arg + return rffi.cast(rffi.VOIDP, x) + +def get_converter(name): + if name == "int": + return IntConverter() + if name == "double": + return DoubleConverter() + raise TypeError("no clue what %s is" % name) Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Tue Jul 6 12:37:23 2010 @@ -3,10 +3,21 @@ #define CPPYY_REFLEXCWRAPPER extern "C" { - long callstatic_l(const char* class_name, const char* method_name, int numargs, void* args[]); - long callmethod_l(const char* class_name, const char* method_name, void* self, int numargs, void* args[]); + long callstatic_l(const char* class_name, int method_index, int numargs, void* args[]); + double callstatic_d(const char* class_name, int method_index, int numargs, void* args[]); + long callmethod_l(const char* class_name, int method_index, void* self, int numargs, void* args[]); void* construct(const char* class_name, int numargs, void* args[]); void destruct(const char* class_name, void* self); + + int num_methods(const char* class_name); + char* method_name(const char* class_name, int method_index); + char* result_type_method(const char* class_name, int method_index); + int num_args_method(const char* class_name, int method_index); + char* arg_type_method(const char* class_name, int method_index, int index); + int is_constructor(const char* class_name, int method_index); + int is_static(const char* class_name, int method_index); + + void myfree(void* ptr); } #endif // ifndef CPPYY_REFLEXCWRAPPER Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Tue Jul 6 12:37:23 2010 @@ -10,6 +10,8 @@ from pypy.rlib.libffi import CDLL +from pypy.module.cppyy import converter + srcpath = py.path.local(__file__).dirpath().join("src") incpath = py.path.local(__file__).dirpath().join("include") @@ -26,7 +28,11 @@ callstatic_l = rffi.llexternal( "callstatic_l", - [rffi.CCHARP, rffi.CCHARP, rffi.INT, rffi.VOIDPP], rffi.LONG, + [rffi.CCHARP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.LONG, + compilation_info=eci) +callstatic_d = rffi.llexternal( + "callstatic_d", + [rffi.CCHARP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.DOUBLE, compilation_info=eci) construct = rffi.llexternal( "construct", @@ -34,7 +40,7 @@ compilation_info=eci) callmethod_l = rffi.llexternal( "callmethod_l", - [rffi.CCHARP, rffi.CCHARP, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.LONG, + [rffi.CCHARP, rffi.INT, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) destruct = rffi.llexternal( "destruct", @@ -42,19 +48,54 @@ compilation_info=eci) +num_methods = rffi.llexternal( + "num_methods", + [rffi.CCHARP], rffi.INT, + compilation_info=eci) +method_name = rffi.llexternal( + "method_name", + [rffi.CCHARP, rffi.INT], rffi.CCHARP, + compilation_info=eci) +result_type_method = rffi.llexternal( + "result_type_method", + [rffi.CCHARP, rffi.INT], rffi.CCHARP, + compilation_info=eci) +num_args_method = rffi.llexternal( + "num_args_method", + [rffi.CCHARP, rffi.INT], rffi.INT, + compilation_info=eci) +arg_type_method = rffi.llexternal( + "arg_type_method", + [rffi.CCHARP, rffi.INT, rffi.INT], rffi.CCHARP, + compilation_info=eci) +is_constructor = rffi.llexternal( + "is_constructor", + [rffi.CCHARP, rffi.INT], rffi.INT, + compilation_info=eci) +is_static = rffi.llexternal( + "is_static", + [rffi.CCHARP, rffi.INT], rffi.INT, + compilation_info=eci) +myfree = rffi.llexternal( + "myfree", + [rffi.VOIDP], lltype.Void, + compilation_info=eci) + def load_lib(space, name): cdll = CDLL(name) return W_CPPLibrary(space, cdll) load_lib.unwrap_spec = [ObjSpace, str] -def prepare_arguments(space, args_w): +def prepare_arguments(space, args_w, argtypes): + if len(args_w) != len(argtypes): + raise OperationError(space.w_TypeError, space.wrap("wrong number of args")) args = lltype.malloc(rffi.CArray(rffi.VOIDP), len(args_w), flavor='raw') for i in range(len(args_w)): - arg = space.int_w(args_w[i]) - x = lltype.malloc(rffi.LONGP.TO, 1, flavor='raw') - x[0] = arg - args[i] = rffi.cast(rffi.VOIDP, x) + argtype = argtypes[i] + conv = converter.get_converter(argtype) + # XXX this can leak so far + args[i] = conv.convert_argument(space, args_w[i]) return args def free_arguments(args, numargs): @@ -75,24 +116,120 @@ type_byname = interp2app(W_CPPLibrary.type_byname, unwrap_spec=['self', str]), ) +class CPPMethod(object): + """ A concrete function after overloading has been resolved """ + + def __init__(self, cpptype, method_index, result_type, arg_types): + self.cpptype = cpptype + self.space = cpptype.space + self.method_index = method_index + self.result_type = result_type + self.arg_types = arg_types + + def call(self, cppthis, args_w): + args = prepare_arguments(self.space, args_w, self.arg_types) + result = callmethod_l(self.cpptype.name, self.method_index, + cppthis, len(args_w), args) + free_arguments(args, len(args_w)) + return result + + def __repr__(self): + return "CPPFunction(%s, %s, %s, %s)" % ( + self.cpptype, self.method_index, self.result_type, self.arg_types) + +class CPPFunction(CPPMethod): + def call(self, cppthis, args_w): + assert cppthis is None + args = prepare_arguments(self.space, args_w, self.arg_types) + if self.result_type == "int": + result = callstatic_l(self.cpptype.name, self.method_index, len(args_w), args) + return self.space.wrap(result) + if self.result_type == "double": + result = callstatic_d(self.cpptype.name, self.method_index, len(args_w), args) + return self.space.wrap(result) + else: + raise NotImplementedError + free_arguments(args, len(args_w)) + + +class CPPConstructor(CPPFunction): + def call(self, cppthis, args_w): + assert cppthis is None + args = prepare_arguments(self.space, args_w, self.arg_types) + result = construct(self.cpptype.name, len(args_w), args) + free_arguments(args, len(args_w)) + return result + + +class CPPOverload(object): + def __init__(self, space, func_name): + self.space = space + self.func_name = func_name + self.functions = [] + + def add_function(self, cppfunc): + self.functions.append(cppfunc) + + def call(self, cppthis, args_w): + space = self.space + for cppyyfunc in self.functions: + try: + return cppyyfunc.call(cppthis, args_w) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + # XXX better error reporting + raise OperationError(space.w_TypeError, space.wrap("none of the overloads matched")) + + def __repr__(self): + return "CPPOverload(%s, %s)" % (self.func_name, self.functions) + +def charp2str(charp): + string = rffi.charp2str(charp) + myfree(charp) + return string class W_CPPType(Wrappable): def __init__(self, cpplib, name): self.space = cpplib.space self.cpplib = cpplib self.name = name + self.function_members = {} + self._find_func_members() + + def _find_func_members(self): + num_func_members = num_methods(self.name) + for i in range(num_func_members): + func_member_name = charp2str(method_name(self.name, i)) + cppfunction = self._make_cppfunction(i) + overload = self.function_members.get(func_member_name, None) + if overload is None: + overload = CPPOverload(self.space, func_member_name) + self.function_members[func_member_name] = overload + overload.add_function(cppfunction) + + def _make_cppfunction(self, method_index): + result_type = charp2str(result_type_method(self.name, method_index)) + num_args = num_args_method(self.name, method_index) + argtypes = [] + for i in range(num_args): + argtype = charp2str(arg_type_method(self.name, method_index, i)) + argtypes.append(argtype) + if is_constructor(self.name, method_index): + cls = CPPConstructor + elif is_static(self.name, method_index): + cls = CPPFunction + else: + cls = CPPMethod + return cls(self, method_index, result_type, argtypes) def invoke(self, name, args_w): - args = prepare_arguments(self.space, args_w) - result = callstatic_l(self.name, name, len(args_w), args) - free_arguments(args, len(args_w)) - return self.space.wrap(result) + overload = self.function_members[name] + return overload.call(None, args_w) def construct(self, args_w): - args = prepare_arguments(self.space, args_w) - result = construct(self.name, len(args_w), args) - free_arguments(args, len(args_w)) - return W_CPPObject(self, result) + overload = self.function_members[self.name] + return W_CPPObject(self, overload.call(None, args_w)) W_CPPType.typedef = TypeDef( 'CPPType', @@ -109,10 +246,8 @@ self.rawobject = rawobject def invoke(self, method_name, args_w): - args = prepare_arguments(self.space, args_w) - result = callmethod_l(self.cppclass.name, method_name, - self.rawobject, len(args_w), args) - free_arguments(args, len(args_w)) + overload = self.cppclass.function_members[method_name] + result = overload.call(self.rawobject, args_w) return self.space.wrap(result) def destruct(self): Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Tue Jul 6 12:37:23 2010 @@ -3,22 +3,31 @@ #include #include -long callstatic_l(const char* class_name, const char* method_name, int numargs, void* args[]) { +long callstatic_l(const char* class_name, int method_index, int numargs, void* args[]) { long result; std::vector arguments(args, args+numargs); Reflex::Type t = Reflex::Type::ByName(class_name); - Reflex::Member m = t.FunctionMemberByName(method_name); + Reflex::Member m = t.FunctionMemberAt(method_index); + m.Invoke(result, arguments); + return result; +} +double callstatic_d(const char* class_name, int method_index, int numargs, void* args[]) { + double result; + std::vector arguments(args, args+numargs); + Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Member m = t.FunctionMemberAt(method_index); m.Invoke(result, arguments); return result; } -long callmethod_l(const char* class_name, const char* method_name, +long callmethod_l(const char* class_name, int method_index, void* self, int numargs, void* args[]) { long result; std::vector arguments(args, args+numargs); Reflex::Type t = Reflex::Type::ByName(class_name); Reflex::Object o(t, self); - o.Invoke(method_name, result, arguments); + Reflex::Member m = t.FunctionMemberAt(method_index); + m.Invoke(o, result, arguments); return result; } @@ -39,3 +48,69 @@ Reflex::Type t = Reflex::Type::ByName(class_name); t.Destruct(self, true); } + + +int num_methods(const char* class_name) { + Reflex::Type t = Reflex::Type::ByName(class_name); + for (int i = 0; i < t.FunctionMemberSize(); i++) { + Reflex::Member m = t.FunctionMemberAt(i); + std::cout << i << " " << m.Name() << std::endl; + for (int j = 0; j < m.FunctionParameterSize(); j++) { + Reflex::Type at = m.TypeOf().FunctionParameterAt(j); + std::cout << " " << j << " " << at.Name() << std::endl; + } + } + return t.FunctionMemberSize(); +} + +char* method_name(const char* class_name, int method_index) { + Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Member m = t.FunctionMemberAt(method_index); + std::string name = m.Name(); + char* name_char = (char*)malloc(name.size() + 1); + strcpy(name_char, name.c_str()); + return name_char; +} + +char* result_type_method(const char* class_name, int method_index) { + Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Member m = t.FunctionMemberAt(method_index); + Reflex::Type rt = m.TypeOf().ReturnType(); + std::string name = rt.Name(Reflex::FINAL); + char* name_char = (char*)malloc(name.size() + 1); + strcpy(name_char, name.c_str()); + return name_char; +} + +int num_args_method(const char* class_name, int method_index) { + Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Member m = t.FunctionMemberAt(method_index); + return m.FunctionParameterSize(); +} + +char* arg_type_method(const char* class_name, int method_index, int arg_index) { + + Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Member m = t.FunctionMemberAt(method_index); + Reflex::Type at = m.TypeOf().FunctionParameterAt(arg_index); + std::string name = at.Name(Reflex::FINAL); + char* name_char = (char*)malloc(name.size() + 1); + strcpy(name_char, name.c_str()); + return name_char; +} + +int is_constructor(const char* class_name, int method_index) { + Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Member m = t.FunctionMemberAt(method_index); + return m.IsConstructor(); +} + +int is_static(const char* class_name, int method_index) { + Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Member m = t.FunctionMemberAt(method_index); + return m.IsStatic(); +} + +void myfree(void* ptr) { + free(ptr); +} Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx Tue Jul 6 12:37:23 2010 @@ -29,6 +29,9 @@ static int add1(int a) { return a + 1; } + static double adddouble(double a) { + return a + 0.01; + } static int getcount() { std::cout << "getcount called" << std::endl; return count; Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Tue Jul 6 12:37:23 2010 @@ -1,22 +1,41 @@ import py import os from pypy.conftest import gettestobjspace +from pypy.module.cppyy import interp_cppyy currpath = py.path.local(__file__).dirpath() +shared_lib = str(currpath.join("example01Dict.so")) + +space = gettestobjspace(usemodules=['cppyy']) + +class TestCPPYYImplementation: + def test_class_query(self): + lib = interp_cppyy.load_lib(space, shared_lib) + w_cppyyclass = lib.type_byname("example01") + adddouble = w_cppyyclass.function_members["adddouble"] + func, = adddouble.functions + assert func.result_type == "double" + assert func.arg_types == ["double"] + class AppTestCPPYY: def setup_class(cls): - cls.space = gettestobjspace(usemodules=['cppyy']) + cls.space = space env = os.environ cls.w_example01 = cls.space.appexec([], """(): import cppyy - return cppyy.load_lib(%r)""" % (str(currpath.join("example01Dict.so")), )) + return cppyy.load_lib(%r)""" % (shared_lib, )) def test_example01static(self): t = self.example01.type_byname("example01") res = t.invoke("add1", 1) assert res == 2 + def test_example01static_double(self): + t = self.example01.type_byname("example01") + res = t.invoke("adddouble", 0.09) + assert res == 0.09 + 0.01 + def test_example01method(self): t = self.example01.type_byname("example01") count = t.invoke("getcount") From cfbolz at codespeak.net Tue Jul 6 12:50:13 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Jul 2010 12:50:13 +0200 (CEST) Subject: [pypy-svn] r75896 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100706105013.50DD5282B9C@codespeak.net> Author: cfbolz Date: Tue Jul 6 12:50:11 2010 New Revision: 75896 Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Log: (antocuni, wlav, cfbolz, arigo around): fix memory leak. test overloading by number of arguments. Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Tue Jul 6 12:50:11 2010 @@ -91,11 +91,18 @@ if len(args_w) != len(argtypes): raise OperationError(space.w_TypeError, space.wrap("wrong number of args")) args = lltype.malloc(rffi.CArray(rffi.VOIDP), len(args_w), flavor='raw') - for i in range(len(args_w)): - argtype = argtypes[i] - conv = converter.get_converter(argtype) - # XXX this can leak so far - args[i] = conv.convert_argument(space, args_w[i]) + try: + i = 0 # appease RPython: i is used below + for i in range(len(args_w)): + argtype = argtypes[i] + conv = converter.get_converter(argtype) + args[i] = conv.convert_argument(space, args_w[i]) + except: + # fun :-( + for j in range(i): + lltype.free(args[j]) + lltype.free(args, flavor='raw') + raise return args def free_arguments(args, numargs): @@ -178,6 +185,8 @@ except OperationError, e: if not e.match(space, space.w_TypeError): raise + except KeyError: + pass # XXX better error reporting raise OperationError(space.w_TypeError, space.wrap("none of the overloads matched")) Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx Tue Jul 6 12:50:11 2010 @@ -29,6 +29,9 @@ static int add1(int a) { return a + 1; } + static int add1(int a, int b) { + return a + b + 1; + } static double adddouble(double a) { return a + 0.01; } Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Tue Jul 6 12:50:11 2010 @@ -8,6 +8,9 @@ space = gettestobjspace(usemodules=['cppyy']) +def setup_module(mod): + os.system("make") + class TestCPPYYImplementation: def test_class_query(self): lib = interp_cppyy.load_lib(space, shared_lib) @@ -28,8 +31,11 @@ def test_example01static(self): t = self.example01.type_byname("example01") + # also tests overloading by number of args res = t.invoke("add1", 1) assert res == 2 + res = t.invoke("add1", 1, 2) + assert res == 4 def test_example01static_double(self): t = self.example01.type_byname("example01") From arigo at codespeak.net Tue Jul 6 13:03:58 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 13:03:58 +0200 (CEST) Subject: [pypy-svn] r75897 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100706110358.5C88E282B9C@codespeak.net> Author: arigo Date: Tue Jul 6 13:03:56 2010 New Revision: 75897 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Log: Start obfuscating the code :-( Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Tue Jul 6 13:03:56 2010 @@ -125,23 +125,82 @@ mark = mark.prev return -1 -def any(enum): - for _ in enum: - return True - return False + +class MatchResult(object): + + def __init__(self, end, marks): + self.end = end + self.marks = marks + + def move_to_next_result(self, ctx): + return False + +class BranchMatchResult(MatchResult): + + def __init__(self, ppos, ptr, marks): + self.ppos = ppos + self.start_ptr = ptr + self.start_marks = marks + + def find_first_result(self, ctx): + ppos = self.ppos + while ctx.pat(ppos): + result = sre_match(ctx, ppos + 1, self.start_ptr, self.start_marks) + ppos += ctx.pat(ppos) + if result is not None: + self.subresult = result + self.end = result.end + self.marks = result.marks + self.ppos = ppos + return True + return False + + def move_to_next_result(self, ctx): + if self.subresult.move_to_next_result(ctx): + return True + return self.find_first_result(ctx) + +class RepeatOneMatchResult(MatchResult): + + def __init__(self, nextppos, minptr, ptr, marks): + self.nextppos = nextppos + self.minptr = minptr + self.start_ptr = ptr + self.start_marks = marks + + def find_first_result(self, ctx): + ptr = self.start_ptr + while ptr >= self.minptr: + result = sre_match(ctx, self.nextppos, ptr, self.start_marks) + ptr -= 1 + if result is not None: + self.subresult = result + self.end = result.end + self.marks = result.marks + self.start_ptr = ptr + return True + return False + + def move_to_next_result(self, ctx): + if self.subresult.move_to_next_result(ctx): + return True + return self.find_first_result(ctx) def match(pattern, string, start=0, flags=0): ctx = MatchContext(pattern, string, start, flags) - for match_end, match_marks in sre_match(ctx, 0, start, None): - ctx.match_end = match_end - ctx.match_marks = match_marks + result = sre_match(ctx, 0, start, None) + if result is not None: + ctx.match_end = result.end + ctx.match_marks = result.marks return ctx return None def sre_match(ctx, ppos, ptr, marks): - """Enumerate answers. Usually we only need the first one, but - there is the case of REPEAT...UNTIL where we need all of them.""" + """Returns either None or a MatchResult object. Usually we only need + the first result, but there is the case of REPEAT...UNTIL where we + need all results; in that case we use the method move_to_next_result() + of the MatchResult.""" while True: op = ctx.pat(ppos) ppos += 1 @@ -152,8 +211,7 @@ if (op == OPCODE_SUCCESS or op == OPCODE_MAX_UNTIL or op == OPCODE_MIN_UNTIL): - yield (ptr, marks) - return + return MatchResult(ptr, marks) elif op == OPCODE_ANY: # match anything (except a newline) @@ -173,7 +231,7 @@ # assert subpattern # <0=skip> <1=back> ptr1 = ptr - ctx.pat(ppos+1) - if ptr1 < 0 or not any(sre_match(ctx, ppos + 2, ptr1, marks)): + if ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None: return ppos += ctx.pat(ppos) @@ -181,7 +239,7 @@ # assert not subpattern # <0=skip> <1=back> ptr1 = ptr - ctx.pat(ppos+1) - if ptr1 >= 0 and any(sre_match(ctx, ppos + 2, ptr1, marks)): + if ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks) is not None: return ppos += ctx.pat(ppos) @@ -195,11 +253,10 @@ elif op == OPCODE_BRANCH: # alternation # <0=skip> code ... - while ctx.pat(ppos): - for answer in sre_match(ctx, ppos + 1, ptr, marks): - yield answer - ppos += ctx.pat(ppos) - return + result = BranchMatchResult(ppos, ptr, marks) + if not result.find_first_result(ctx): + result = None + return result #elif op == OPCODE_CATEGORY: # seems to be never produced @@ -347,7 +404,7 @@ # try to match 'tail' if we have enough 'item' for answer in sre_match(ctx, tailppos, ptr, marks): - yield answer + yield_answer if len(pending) == 0: return ptr, marks, enum = pending.pop() @@ -363,7 +420,7 @@ # try to match 'tail' if we have enough 'item' if len(pending) >= min: for answer in sre_match(ctx, tailppos, ptr, marks): - yield answer + yield_answer if max == 65535 or len(pending) < max: # try to match one more 'item' @@ -404,11 +461,10 @@ # string. check if the rest of the pattern matches, # and backtrack if not. nextppos = ppos + ctx.pat(ppos) - while ptr >= minptr: - for answer in sre_match(ctx, nextppos, ptr, marks): - yield answer - ptr -= 1 - return + result = RepeatOneMatchResult(nextppos, minptr, ptr, marks) + if not result.find_first_result(ctx): + result = None + return result elif op == OPCODE_MIN_REPEAT_ONE: # match repeated sequence (minimizing regexp). @@ -437,7 +493,7 @@ nextppos = ppos + ctx.pat(ppos) while ptr <= maxptr: for answer in sre_match(ctx, nextppos, ptr, marks): - yield answer + yield_answer ptr1 = find_repetition_end(ctx, ppos+3, ptr, 1) if ptr1 == ptr: break Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Tue Jul 6 13:03:56 2010 @@ -32,6 +32,13 @@ class TestMatch: + def test_or(self): + r = get_code(r"a|bc|def") + assert rsre.match(r, "a") + assert rsre.match(r, "bc") + assert rsre.match(r, "def") + assert not rsre.match(r, "ghij") + def test_any(self): r = get_code(r"ab.cd") assert rsre.match(r, "abXcdef") From fijal at codespeak.net Tue Jul 6 13:09:33 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Jul 2010 13:09:33 +0200 (CEST) Subject: [pypy-svn] r75898 - pypy/trunk/pypy/jit/backend/llsupport Message-ID: <20100706110933.693A3282B9C@codespeak.net> Author: fijal Date: Tue Jul 6 13:09:32 2010 New Revision: 75898 Modified: pypy/trunk/pypy/jit/backend/llsupport/llmodel.py Log: Fix translation Modified: pypy/trunk/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/llmodel.py Tue Jul 6 13:09:32 2010 @@ -250,6 +250,7 @@ ofs = arraydescr.get_ofs_length(self.translate_support_code) return rffi.cast(rffi.CArrayPtr(lltype.Signed), array)[ofs/WORD] + @specialize.argtype(2) def bh_getarrayitem_gc_i(self, arraydescr, gcref, itemindex): ofs, size = self.unpack_arraydescr_size(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -272,6 +273,7 @@ # --- end of GC unsafe code --- return pval + @specialize.argtype(2) def bh_getarrayitem_gc_f(self, arraydescr, gcref, itemindex): ofs = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -281,6 +283,7 @@ # --- end of GC unsafe code --- return fval + @specialize.argtype(2) def bh_setarrayitem_gc_i(self, arraydescr, gcref, itemindex, newvalue): ofs, size = self.unpack_arraydescr_size(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -303,6 +306,7 @@ items[itemindex] = self.cast_gcref_to_int(newvalue) # --- end of GC unsafe code --- + @specialize.argtype(2) def bh_setarrayitem_gc_f(self, arraydescr, gcref, itemindex, newvalue): ofs = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- From cfbolz at codespeak.net Tue Jul 6 13:13:15 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Jul 2010 13:13:15 +0200 (CEST) Subject: [pypy-svn] r75899 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100706111315.B29BF282B9C@codespeak.net> Author: cfbolz Date: Tue Jul 6 13:13:14 2010 New Revision: 75899 Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Log: (wlav, cfbolz, antocuni, arigo around): test and fix Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Tue Jul 6 13:13:14 2010 @@ -82,6 +82,8 @@ compilation_info=eci) +NULL_VOIDP = lltype.nullptr(rffi.VOIDP.TO) + def load_lib(space, name): cdll = CDLL(name) return W_CPPLibrary(space, cdll) @@ -100,7 +102,7 @@ except: # fun :-( for j in range(i): - lltype.free(args[j]) + lltype.free(args[j], flavor='raw') lltype.free(args, flavor='raw') raise return args @@ -146,7 +148,7 @@ class CPPFunction(CPPMethod): def call(self, cppthis, args_w): - assert cppthis is None + assert not cppthis args = prepare_arguments(self.space, args_w, self.arg_types) if self.result_type == "int": result = callstatic_l(self.cpptype.name, self.method_index, len(args_w), args) @@ -161,7 +163,7 @@ class CPPConstructor(CPPFunction): def call(self, cppthis, args_w): - assert cppthis is None + assert not cppthis args = prepare_arguments(self.space, args_w, self.arg_types) result = construct(self.cpptype.name, len(args_w), args) free_arguments(args, len(args_w)) @@ -234,11 +236,11 @@ def invoke(self, name, args_w): overload = self.function_members[name] - return overload.call(None, args_w) + return overload.call(NULL_VOIDP, args_w) def construct(self, args_w): overload = self.function_members[self.name] - return W_CPPObject(self, overload.call(None, args_w)) + return W_CPPObject(self, overload.call(NULL_VOIDP, args_w)) W_CPPType.typedef = TypeDef( 'CPPType', Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Tue Jul 6 13:13:14 2010 @@ -36,6 +36,8 @@ assert res == 2 res = t.invoke("add1", 1, 2) assert res == 4 + raises(TypeError, 't.invoke("add1", 1, [])') + def test_example01static_double(self): t = self.example01.type_byname("example01") @@ -54,3 +56,4 @@ instance.destruct() count = t.invoke("getcount") assert count == 0 + From hakanardo at codespeak.net Tue Jul 6 13:17:00 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 6 Jul 2010 13:17:00 +0200 (CEST) Subject: [pypy-svn] r75900 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100706111700.76FC1282B9C@codespeak.net> Author: hakanardo Date: Tue Jul 6 13:16:59 2010 New Revision: 75900 Added: pypy/branch/interplevel-array/pypy/module/array/interp_array_try1.py (contents, props changed) - copied, changed from r75891, pypy/branch/interplevel-array/pypy/module/array/interp_array.py Removed: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: multiclass version Copied: pypy/branch/interplevel-array/pypy/module/array/interp_array_try1.py (from r75891, pypy/branch/interplevel-array/pypy/module/array/interp_array.py) ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array_try1.py Tue Jul 6 13:16:59 2010 @@ -83,10 +83,10 @@ if len(typecode) != 1: msg = 'array() argument 1 must be char, not str' raise OperationError(space.w_TypeError, space.wrap(msg)) + typecode = typecode[0] if typecode not in 'cbBuhHiIlLfd': msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' raise OperationError(space.w_ValueError, space.wrap(msg)) - typecode = typecode[0] self.space = space self.typecode = typecode self.len = 0 From hakanardo at codespeak.net Tue Jul 6 13:17:14 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 6 Jul 2010 13:17:14 +0200 (CEST) Subject: [pypy-svn] r75901 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100706111714.9F55A282B9C@codespeak.net> Author: hakanardo Date: Tue Jul 6 13:17:13 2010 New Revision: 75901 Added: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: multiclass version Added: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Jul 6 13:17:13 2010 @@ -0,0 +1,175 @@ +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.error import OperationError +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root +from pypy.rlib.jit import dont_look_inside +from pypy.rlib import rgc +from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.rstruct.runpack import runpack + +class W_ArrayBase(Wrappable): + def descr_append(self, w_x): + x = self.item_w(w_x) + self.setlen(self.len + 1) + self.buffer[self.len - 1] = x + descr_append.unwrap_spec = ['self', W_Root] + + def descr_extend(self, w_initializer): + space = self.space + w_iterator = space.iter(w_initializer) + while True: + try: + w_item = space.next(w_iterator) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break + self.descr_append(w_item) + descr_extend.unwrap_spec = ['self', W_Root] + + +class TypeCode(object): + def __init__(self, itemtype, unwrap, canoverflow=False, signed=False): + self.itemtype = itemtype + if itemtype is lltype.SingleFloat: + self.bytes = 4 + else: + self.bytes = rffi.sizeof(itemtype) + self.arraytype = lltype.GcArray(itemtype) + self.unwrap = unwrap + self.signed = signed + self.canoverflow = canoverflow + self.w_class = None + + +types = { + 'c': TypeCode(lltype.Char, 'str_w'), + 'u': TypeCode(lltype.UniChar, 'unicode_w'), + 'b': TypeCode(rffi.SIGNEDCHAR, 'int_w', True, True), + 'B': TypeCode(rffi.UCHAR, 'int_w', True), + 'h': TypeCode(rffi.SHORT, 'int_w', True, True), + 'H': TypeCode(rffi.USHORT, 'int_w', True), + 'i': TypeCode(rffi.INT, 'int_w', True, True), + 'I': TypeCode(rffi.UINT, 'int_w', True), + 'l': TypeCode(rffi.LONG, 'int_w', True, True), + 'L': TypeCode(rffi.ULONG, 'bigint_w', True), + 'f': TypeCode(lltype.SingleFloat, 'float_w'), + 'd': TypeCode(lltype.Float, 'float_w'), + } +for k, v in types.items(): v.typecode=k +unroll_typecodes = unrolling_iterable(types.keys()) + +for thetypecode, thetype in types.items(): + class W_Array(W_ArrayBase): + mytype = thetype + def __init__(self, space): + self.space = space + self.len = 0 + self.buffer = None + + def item_w(self, w_item): + space = self.space + unwrap = getattr(space, self.mytype.unwrap) + item = unwrap(w_item) + if self.mytype.unwrap == 'bigint_w': + try: + if self.mytype.signed: + item = item.tolonglong() + else: + item = item.toulonglong() + except (ValueError, OverflowError): + msg = 'unsigned %d-byte integer out of range' % self.mytype.bytes + raise OperationError(space.w_OverflowError, space.wrap(msg)) + + if self.mytype.canoverflow: + msg = None + if self.mytype.signed: + if item < -2 ** (self.mytype.bytes * 8) / 2: + msg = 'signed %d-byte integer is less than minimum' % self.mytype.bytes + elif item > 2 ** (self.mytype.bytes * 8) / 2 - 1: + msg = 'signed %d-byte integer is greater than maximum' % self.mytype.bytes + else: + if item < 0: + msg = 'unsigned %d-byte integer is less than minimum' % self.mytype.bytes + elif item > 2 ** (self.mytype.bytes * 8) - 1: + msg = 'unsigned %d-byte integer is greater than maximum' % self.mytype.bytes + if msg is not None: + raise OperationError(space.w_OverflowError, space.wrap(msg)) + return rffi.cast(self.mytype.itemtype, item) + + + def setlen(self, size): + new_buffer = lltype.malloc(self.mytype.arraytype, size) + for i in range(self.len): + new_buffer[i] = self.buffer[i] + self.buffer = new_buffer + self.len = size + + + def descr_len(self): + return self.space.wrap(self.len) + descr_len.unwrap_spec = ['self'] + + + def descr_getitem(self, w_idx): + space=self.space + start, stop, step = space.decode_index(w_idx, self.len) + if step==0: + item = self.buffer[start] + if self.mytype.typecode in ('b', 'B', 'h', 'H', 'i', 'l'): + item = rffi.cast(lltype.Signed, item) + elif self.mytype.typecode == 'f': + item = float(item) + return self.space.wrap(item) + else: + size = (stop - start) / step + if (stop - start) % step > 0: size += 1 + w_a=W_Array(self.space, self.mytype.typecode) + w_a.setlen(size) + j=0 + for i in range(start, stop, step): + w_a.buffer[j]=self.buffer[i] + j+=1 + return w_a + descr_getitem.unwrap_spec = ['self', W_Root] + + W_Array.typedef = TypeDef( + 'ArrayType_'+thetypecode, + append = interp2app(W_Array.descr_append), + extend = interp2app(W_Array.descr_extend), + __len__ = interp2app(W_Array.descr_len), + __getitem__ = interp2app(W_Array.descr_getitem), + ) + + thetype.w_class = W_Array + + +def array(space, typecode, w_initializer=None): + if len(typecode) != 1: + msg = 'array() argument 1 must be char, not str' + raise OperationError(space.w_TypeError, space.wrap(msg)) + typecode=typecode[0] + + for tc in unroll_typecodes: + if typecode == tc: + a = types[tc].w_class(space) + break + else: + msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' + raise OperationError(space.w_ValueError, space.wrap(msg)) + + if w_initializer is not None: + if not space.is_w(w_initializer, space.w_None): + a.descr_extend(w_initializer) + ## if space.is_w(space.type(w_initializer), space.w_str): + ## a.descr_fromstring(space.str_w(w_initializer)) + ## elif space.is_w(space.type(w_initializer), space.w_unicode): + ## a.descr_fromunicode(space.unicode_w(w_initializer)) + ## elif space.is_w(space.type(w_initializer), space.w_list): + ## a.descr_fromlist(w_initializer) + ## elif not space.is_w(w_initializer, space.w_None): + ## a.descr_extend(w_initializer) + + return a +array.unwrap_spec = (ObjSpace, str, W_Root) From hakanardo at codespeak.net Tue Jul 6 13:45:50 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 6 Jul 2010 13:45:50 +0200 (CEST) Subject: [pypy-svn] r75902 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100706114550.E530B282B9C@codespeak.net> Author: hakanardo Date: Tue Jul 6 13:45:47 2010 New Revision: 75902 Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: failing to compile Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/__init__.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/__init__.py Tue Jul 6 13:45:47 2010 @@ -9,5 +9,5 @@ interpleveldefs = { 'array': 'interp_array.array', - 'sized_array': 'interp_array.sized_array', + #'sized_array': 'interp_array.sized_array', } Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Jul 6 13:45:47 2010 @@ -134,6 +134,7 @@ return w_a descr_getitem.unwrap_spec = ['self', W_Root] + W_Array.__name__ = 'W_ArrayType_'+thetypecode W_Array.typedef = TypeDef( 'ArrayType_'+thetypecode, append = interp2app(W_Array.descr_append), From jcreigh at codespeak.net Tue Jul 6 14:30:00 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 6 Jul 2010 14:30:00 +0200 (CEST) Subject: [pypy-svn] r75903 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100706123000.9AAD4282B9C@codespeak.net> Author: jcreigh Date: Tue Jul 6 14:29:59 2010 New Revision: 75903 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/arch.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Log: minor comment cleanups Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/arch.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/arch.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/arch.py Tue Jul 6 14:29:59 2010 @@ -3,13 +3,15 @@ import sys if sys.maxint == (2**31 - 1): WORD = 4 - FRAME_FIXED_SIZE = 5 # ebp + ebx + esi + edi + force_index = 5 words - FORCE_INDEX_OFS = -4*WORD + # ebp + ebx + esi + edi + force_index = 5 words + FRAME_FIXED_SIZE = 5 IS_X86_32 = True IS_X86_64 = False else: WORD = 8 + # rbp + rbx + r12 + r13 + r14 + r15 + force_index = 7 words FRAME_FIXED_SIZE = 7 - FORCE_INDEX_OFS = -6*WORD IS_X86_32 = False IS_X86_64 = True + +FORCE_INDEX_OFS = -(FRAME_FIXED_SIZE-1)*WORD Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Tue Jul 6 14:29:59 2010 @@ -31,9 +31,6 @@ from pypy.rlib import rgc from pypy.jit.backend.x86.jump import remap_frame_layout -# our calling convention - we pass first 6 args in registers -# and the rest stays on the stack - # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry CALL_ALIGN = 16 // WORD @@ -499,6 +496,8 @@ self.mc.MOVSD(loc, X86_64_XMM_SCRATCH_REG) else: self.mc.MOV_rb(X86_64_SCRATCH_REG.value, (2 + i) * WORD) + # XXX: We're assuming that "loc" won't require regloc to + # clobber the scratch register self.mc.MOV(loc, X86_64_SCRATCH_REG) self.mc.JMP(imm(jmpadr)) From cfbolz at codespeak.net Tue Jul 6 15:09:57 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Jul 2010 15:09:57 +0200 (CEST) Subject: [pypy-svn] r75904 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100706130957.24D42282B9C@codespeak.net> Author: cfbolz Date: Tue Jul 6 15:09:56 2010 New Revision: 75904 Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Log: (antocuni, cfbolz, wlav, arigo): fix one annotation problem, not sure it's the only one. Prefix all C-functions with "c_". Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Tue Jul 6 15:09:56 2010 @@ -26,57 +26,57 @@ use_cpp_linker=True, ) -callstatic_l = rffi.llexternal( +c_callstatic_l = rffi.llexternal( "callstatic_l", [rffi.CCHARP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) -callstatic_d = rffi.llexternal( +c_callstatic_d = rffi.llexternal( "callstatic_d", [rffi.CCHARP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.DOUBLE, compilation_info=eci) -construct = rffi.llexternal( +c_construct = rffi.llexternal( "construct", [rffi.CCHARP, rffi.INT, rffi.VOIDPP], rffi.VOIDP, compilation_info=eci) -callmethod_l = rffi.llexternal( +c_callmethod_l = rffi.llexternal( "callmethod_l", [rffi.CCHARP, rffi.INT, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) -destruct = rffi.llexternal( +c_destruct = rffi.llexternal( "destruct", [rffi.CCHARP, rffi.VOIDP], lltype.Void, compilation_info=eci) -num_methods = rffi.llexternal( +c_num_methods = rffi.llexternal( "num_methods", [rffi.CCHARP], rffi.INT, compilation_info=eci) -method_name = rffi.llexternal( +c_method_name = rffi.llexternal( "method_name", [rffi.CCHARP, rffi.INT], rffi.CCHARP, compilation_info=eci) -result_type_method = rffi.llexternal( +c_result_type_method = rffi.llexternal( "result_type_method", [rffi.CCHARP, rffi.INT], rffi.CCHARP, compilation_info=eci) -num_args_method = rffi.llexternal( +c_num_args_method = rffi.llexternal( "num_args_method", [rffi.CCHARP, rffi.INT], rffi.INT, compilation_info=eci) -arg_type_method = rffi.llexternal( +c_arg_type_method = rffi.llexternal( "arg_type_method", [rffi.CCHARP, rffi.INT, rffi.INT], rffi.CCHARP, compilation_info=eci) -is_constructor = rffi.llexternal( +c_is_constructor = rffi.llexternal( "is_constructor", [rffi.CCHARP, rffi.INT], rffi.INT, compilation_info=eci) -is_static = rffi.llexternal( +c_is_static = rffi.llexternal( "is_static", [rffi.CCHARP, rffi.INT], rffi.INT, compilation_info=eci) -myfree = rffi.llexternal( +c_myfree = rffi.llexternal( "myfree", [rffi.VOIDP], lltype.Void, compilation_info=eci) @@ -137,10 +137,10 @@ def call(self, cppthis, args_w): args = prepare_arguments(self.space, args_w, self.arg_types) - result = callmethod_l(self.cpptype.name, self.method_index, + result = c_callmethod_l(self.cpptype.name, self.method_index, cppthis, len(args_w), args) free_arguments(args, len(args_w)) - return result + return self.space.wrap(result) def __repr__(self): return "CPPFunction(%s, %s, %s, %s)" % ( @@ -150,24 +150,26 @@ def call(self, cppthis, args_w): assert not cppthis args = prepare_arguments(self.space, args_w, self.arg_types) - if self.result_type == "int": - result = callstatic_l(self.cpptype.name, self.method_index, len(args_w), args) - return self.space.wrap(result) - if self.result_type == "double": - result = callstatic_d(self.cpptype.name, self.method_index, len(args_w), args) - return self.space.wrap(result) - else: - raise NotImplementedError - free_arguments(args, len(args_w)) + try: + if self.result_type == "int": + result = c_callstatic_l(self.cpptype.name, self.method_index, len(args_w), args) + return self.space.wrap(result) + if self.result_type == "double": + result = c_callstatic_d(self.cpptype.name, self.method_index, len(args_w), args) + return self.space.wrap(result) + else: + raise NotImplementedError + finally: + free_arguments(args, len(args_w)) class CPPConstructor(CPPFunction): def call(self, cppthis, args_w): assert not cppthis args = prepare_arguments(self.space, args_w, self.arg_types) - result = construct(self.cpptype.name, len(args_w), args) + result = c_construct(self.cpptype.name, len(args_w), args) free_arguments(args, len(args_w)) - return result + return W_CPPObject(self.cpptype, result) class CPPOverload(object): @@ -197,7 +199,7 @@ def charp2str(charp): string = rffi.charp2str(charp) - myfree(charp) + c_myfree(charp) return string class W_CPPType(Wrappable): @@ -209,9 +211,9 @@ self._find_func_members() def _find_func_members(self): - num_func_members = num_methods(self.name) + num_func_members = c_num_methods(self.name) for i in range(num_func_members): - func_member_name = charp2str(method_name(self.name, i)) + func_member_name = charp2str(c_method_name(self.name, i)) cppfunction = self._make_cppfunction(i) overload = self.function_members.get(func_member_name, None) if overload is None: @@ -220,15 +222,15 @@ overload.add_function(cppfunction) def _make_cppfunction(self, method_index): - result_type = charp2str(result_type_method(self.name, method_index)) - num_args = num_args_method(self.name, method_index) + result_type = charp2str(c_result_type_method(self.name, method_index)) + num_args = c_num_args_method(self.name, method_index) argtypes = [] for i in range(num_args): - argtype = charp2str(arg_type_method(self.name, method_index, i)) + argtype = charp2str(c_arg_type_method(self.name, method_index, i)) argtypes.append(argtype) - if is_constructor(self.name, method_index): + if c_is_constructor(self.name, method_index): cls = CPPConstructor - elif is_static(self.name, method_index): + elif c_is_static(self.name, method_index): cls = CPPFunction else: cls = CPPMethod @@ -240,7 +242,7 @@ def construct(self, args_w): overload = self.function_members[self.name] - return W_CPPObject(self, overload.call(NULL_VOIDP, args_w)) + return overload.call(NULL_VOIDP, args_w) W_CPPType.typedef = TypeDef( 'CPPType', @@ -248,8 +250,6 @@ construct = interp2app(W_CPPType.construct, unwrap_spec=['self', 'args_w']), ) - - class W_CPPObject(Wrappable): def __init__(self, cppclass, rawobject): self.space = cppclass.space @@ -258,11 +258,10 @@ def invoke(self, method_name, args_w): overload = self.cppclass.function_members[method_name] - result = overload.call(self.rawobject, args_w) - return self.space.wrap(result) + return overload.call(self.rawobject, args_w) def destruct(self): - destruct(self.cppclass.name, self.rawobject) + c_destruct(self.cppclass.name, self.rawobject) W_CPPObject.typedef = TypeDef( 'CPPObject', From arigo at codespeak.net Tue Jul 6 15:24:16 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 15:24:16 +0200 (CEST) Subject: [pypy-svn] r75906 - in pypy/branch: asmgcc-64 asmgcc-64-old Message-ID: <20100706132416.CD9ED282BF7@codespeak.net> Author: arigo Date: Tue Jul 6 15:24:15 2010 New Revision: 75906 Added: pypy/branch/asmgcc-64-old/ - copied from r75905, pypy/branch/asmgcc-64/ Removed: pypy/branch/asmgcc-64/ Log: Rename that branch. From jcreigh at codespeak.net Tue Jul 6 15:27:02 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 6 Jul 2010 15:27:02 +0200 (CEST) Subject: [pypy-svn] r75907 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/test Message-ID: <20100706132702.CB3D5282BF7@codespeak.net> Author: jcreigh Date: Tue Jul 6 15:27:01 2010 New Revision: 75907 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py Log: fix some tests by ensuring that there are no duplicates in "inputargs" to compile_loop Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py Tue Jul 6 15:27:01 2010 @@ -997,6 +997,8 @@ else: assert 0 operations.append(ResOperation(opnum, boxargs, boxres)) + # Unique-ify inputargs + inputargs = list(set(inputargs)) faildescr = BasicFailDescr(1) operations.append(ResOperation(rop.FINISH, [], None, descr=faildescr)) @@ -1069,7 +1071,8 @@ descr=BasicFailDescr(5))] operations[1].fail_args = [] looptoken = LoopToken() - self.cpu.compile_loop(list(testcase), operations, + # Use "set" to unique-ify inputargs + self.cpu.compile_loop(list(set(testcase)), operations, looptoken) for i, box in enumerate(testcase): self.cpu.set_future_value_float(i, box.value) From jcreigh at codespeak.net Tue Jul 6 15:30:29 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 6 Jul 2010 15:30:29 +0200 (CEST) Subject: [pypy-svn] r75908 - pypy/branch/asmgcc-64 Message-ID: <20100706133029.A5D9B282BF7@codespeak.net> Author: jcreigh Date: Tue Jul 6 15:30:28 2010 New Revision: 75908 Added: pypy/branch/asmgcc-64/ (props changed) - copied from r75907, pypy/trunk/ Log: make new branch From arigo at codespeak.net Tue Jul 6 15:47:01 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 15:47:01 +0200 (CEST) Subject: [pypy-svn] r75909 - pypy/branch/rsre2/pypy/rlib/rsre Message-ID: <20100706134701.B0C24282B9C@codespeak.net> Author: arigo Date: Tue Jul 6 15:47:00 2010 New Revision: 75909 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Log: Finish replacing this generator code with obscure non-generator code. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Tue Jul 6 15:47:00 2010 @@ -135,7 +135,17 @@ def move_to_next_result(self, ctx): return False -class BranchMatchResult(MatchResult): +class AbstractMultipleMatchResult(MatchResult): + + def move_to_next_result(self, ctx): + result = self.subresult + if result.move_to_next_result(ctx): + self.end = result.end + self.marks = result.marks + return True + return self.find_next_result(ctx) + +class BranchMatchResult(AbstractMultipleMatchResult): def __init__(self, ppos, ptr, marks): self.ppos = ppos @@ -154,13 +164,9 @@ self.ppos = ppos return True return False + find_next_result = find_first_result - def move_to_next_result(self, ctx): - if self.subresult.move_to_next_result(ctx): - return True - return self.find_first_result(ctx) - -class RepeatOneMatchResult(MatchResult): +class RepeatOneMatchResult(AbstractMultipleMatchResult): def __init__(self, nextppos, minptr, ptr, marks): self.nextppos = nextppos @@ -180,12 +186,150 @@ self.start_ptr = ptr return True return False + find_next_result = find_first_result - def move_to_next_result(self, ctx): - if self.subresult.move_to_next_result(ctx): - return True + +class MinRepeatOneMatchResult(AbstractMultipleMatchResult): + + def __init__(self, nextppos, ppos3, maxptr, ptr, marks): + self.nextppos = nextppos + self.ppos3 = ppos3 + self.maxptr = maxptr + self.start_ptr = ptr + self.start_marks = marks + + def find_first_result(self, ctx): + ptr = self.start_ptr + while ptr <= self.maxptr: + result = sre_match(ctx, self.nextppos, ptr, self.start_marks) + if result is not None: + self.subresult = result + self.end = result.end + self.marks = result.marks + self.start_ptr = ptr + return True + ptr1 = find_repetition_end(ctx, self.ppos3, ptr, 1) + if ptr1 == ptr: + break + ptr = ptr1 + return False + + def find_next_result(self, ctx): + ptr = self.start_ptr + ptr1 = find_repetition_end(ctx, self.ppos3, ptr, 1) + if ptr1 == ptr: + return False + self.start_ptr = ptr1 return self.find_first_result(ctx) +class AbstractUntilMatchResult(AbstractMultipleMatchResult): + + def __init__(self, ppos, tailppos, ptr, marks): + self.ppos = ppos + self.tailppos = tailppos + self.cur_ptr = ptr + self.cur_marks = marks + self.pending = [] + +class MaxUntilMatchResult(AbstractUntilMatchResult): + + def find_first_result(self, ctx): + enum = sre_match(ctx, self.ppos + 3, self.cur_ptr, self.cur_marks) + return self.search_next(ctx, enum, resume=False) + + def find_next_result(self, ctx): + return self.search_next(ctx, None, resume=True) + + def search_next(self, ctx, enum, resume): + ppos = self.ppos + min = ctx.pat(ppos+1) + max = ctx.pat(ppos+2) + ptr = self.cur_ptr + marks = self.cur_marks + while True: + while True: + if enum is not None: + # matched one more 'item'. record it and continue + self.pending.append((ptr, marks, enum)) + ptr = enum.end + marks = enum.marks + break + else: + # 'item' no longer matches. + if not resume and len(self.pending) >= min: + # try to match 'tail' if we have enough 'item' + result = sre_match(ctx, self.tailppos, ptr, marks) + if result is not None: + self.subresult = result + self.end = result.end + self.marks = result.marks + self.cur_ptr = ptr + self.cur_marks = marks + return True + resume = False + if len(self.pending) == 0: + return False + ptr, marks, enum = self.pending.pop() + if not enum.move_to_next_result(ctx): + enum = None + # + if max == 65535 or len(self.pending) < max: + # try to match one more 'item' + enum = sre_match(ctx, ppos + 3, ptr, marks) + else: + enum = None # 'max' reached, no more matches + +class MinUntilMatchResult(AbstractUntilMatchResult): + + def find_first_result(self, ctx): + return self.search_next(ctx, resume=False) + + def find_next_result(self, ctx): + return self.search_next(ctx, resume=True) + + def search_next(self, ctx, resume): + ppos = self.ppos + min = ctx.pat(ppos+1) + max = ctx.pat(ppos+2) + ptr = self.cur_ptr + marks = self.cur_marks + while True: + # try to match 'tail' if we have enough 'item' + if not resume and len(self.pending) >= min: + result = sre_match(ctx, self.tailppos, ptr, marks) + if result is not None: + self.subresult = result + self.end = result.end + self.marks = result.marks + self.cur_ptr = ptr + self.cur_marks = marks + return True + resume = False + + if max == 65535 or len(self.pending) < max: + # try to match one more 'item' + enum = sre_match(ctx, ppos + 3, ptr, marks) + else: + enum = None # 'max' reached, no more matches + + while True: + if enum is not None: + # matched one more 'item'. record it and continue + self.pending.append((ptr, marks, enum)) + ptr = enum.end + marks = enum.marks + break + else: + # 'item' no longer matches. + if len(self.pending) == 0: + return False + ptr, marks, enum = self.pending.pop() + if not enum.move_to_next_result(ctx): + enum = None + continue + break + +# ____________________________________________________________ def match(pattern, string, start=0, flags=0): ctx = MatchContext(pattern, string, start, flags) @@ -366,13 +510,8 @@ # general repeat. in this version of the re module, all the work # is done here, and not on the later UNTIL operator. # <1=min> <2=max> item tail - itemppos = ppos + 3 # FIXME: we probably need to deal with zero-width matches in here.. - # get the minimum and maximum number of repetitions allowed - min = ctx.pat(ppos+1) - max = ctx.pat(ppos+2) - # decode the later UNTIL operator to see if it is actually # a MAX_UNTIL or MIN_UNTIL untilppos = ppos + ctx.pat(ppos) @@ -383,64 +522,18 @@ # possible, followed by the 'tail'. we do this by # remembering each state for each possible number of # 'item' matching. - pending = [] - # - while True: - if max == 65535 or len(pending) < max: - # try to match one more 'item' - enum = sre_match(ctx, itemppos, ptr, marks) - else: - enum = iter(()) # 'max' reached, no more matches - - while True: - for next in enum: - # matched one more 'item'. record it and continue - pending.append((ptr, marks, enum)) - ptr, marks = next - break - else: - # 'item' no longer matches. - if len(pending) >= min: - # try to match 'tail' if we have enough 'item' - for answer in sre_match(ctx, tailppos, - ptr, marks): - yield_answer - if len(pending) == 0: - return - ptr, marks, enum = pending.pop() - continue - break + result = MaxUntilMatchResult(ppos, tailppos, ptr, marks) + if not result.find_first_result(ctx): + result = None + return result elif op == OPCODE_MIN_UNTIL: # first try to match the 'tail', and if it fails, try # to match one more 'item' and try again - pending = [] - # - while True: - # try to match 'tail' if we have enough 'item' - if len(pending) >= min: - for answer in sre_match(ctx, tailppos, ptr, marks): - yield_answer - - if max == 65535 or len(pending) < max: - # try to match one more 'item' - enum = sre_match(ctx, itemppos, ptr, marks) - else: - enum = iter(()) # 'max' reached, no more matches - - while True: - for next in enum: - # matched one more 'item'. record it and continue - pending.append((ptr, marks, enum)) - ptr, marks = next - break - else: - # 'item' no longer matches. - if len(pending) == 0: - return - ptr, marks, enum = pending.pop() - continue - break + result = MinUntilMatchResult(ppos, tailppos, ptr, marks) + if not result.find_first_result(ctx): + result = None + return result else: raise AssertionError("missing UNTIL after REPEAT") @@ -491,14 +584,11 @@ if maxptr1 <= maxptr: maxptr = maxptr1 nextppos = ppos + ctx.pat(ppos) - while ptr <= maxptr: - for answer in sre_match(ctx, nextppos, ptr, marks): - yield_answer - ptr1 = find_repetition_end(ctx, ppos+3, ptr, 1) - if ptr1 == ptr: - break - ptr = ptr1 - return + result = MinRepeatOneMatchResult(nextppos, ppos+3, maxptr, + ptr, marks) + if not result.find_first_result(ctx): + result = None + return result else: assert 0, "bad pattern code %d" % op From cfbolz at codespeak.net Tue Jul 6 16:01:28 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Jul 2010 16:01:28 +0200 (CEST) Subject: [pypy-svn] r75910 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100706140128.D8128282B9C@codespeak.net> Author: cfbolz Date: Tue Jul 6 16:01:27 2010 New Revision: 75910 Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Log: (wlav, cfbolz, antocuni, arigo): refactor stuff to hopefully be more JIT-friendly Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Tue Jul 6 16:01:27 2010 @@ -9,6 +9,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.rlib.libffi import CDLL +from pypy.rlib import jit, debug from pypy.module.cppyy import converter @@ -89,30 +90,8 @@ return W_CPPLibrary(space, cdll) load_lib.unwrap_spec = [ObjSpace, str] -def prepare_arguments(space, args_w, argtypes): - if len(args_w) != len(argtypes): - raise OperationError(space.w_TypeError, space.wrap("wrong number of args")) - args = lltype.malloc(rffi.CArray(rffi.VOIDP), len(args_w), flavor='raw') - try: - i = 0 # appease RPython: i is used below - for i in range(len(args_w)): - argtype = argtypes[i] - conv = converter.get_converter(argtype) - args[i] = conv.convert_argument(space, args_w[i]) - except: - # fun :-( - for j in range(i): - lltype.free(args[j], flavor='raw') - lltype.free(args, flavor='raw') - raise - return args - -def free_arguments(args, numargs): - for i in range(numargs): - lltype.free(args[i], flavor='raw') - lltype.free(args, flavor='raw') - class W_CPPLibrary(Wrappable): + _immutable_ = True def __init__(self, space, cdll): self.cdll = cdll self.space = space @@ -127,6 +106,8 @@ class CPPMethod(object): """ A concrete function after overloading has been resolved """ + _immutable_ = True + _immutable_fields_ = ["arg_types[*]", "arg_converters[*]"] def __init__(self, cpptype, method_index, result_type, arg_types): self.cpptype = cpptype @@ -134,14 +115,45 @@ self.method_index = method_index self.result_type = result_type self.arg_types = arg_types + self.arg_converters = None def call(self, cppthis, args_w): - args = prepare_arguments(self.space, args_w, self.arg_types) + args = self.prepare_arguments(args_w) result = c_callmethod_l(self.cpptype.name, self.method_index, cppthis, len(args_w), args) - free_arguments(args, len(args_w)) + self.free_arguments(args) return self.space.wrap(result) + def _build_converters(self): + self.arg_converters = [converter.get_converter(arg_type) + for arg_type in self.arg_types] + + def prepare_arguments(self, args_w): + space = self.space + if len(args_w) != len(self.arg_types): + raise OperationError(space.w_TypeError, space.wrap("wrong number of args")) + if self.arg_converters is None: + self._build_converters() + args = lltype.malloc(rffi.CArray(rffi.VOIDP), len(args_w), flavor='raw') + try: + i = 0 # appease RPython: i is used below + for i in range(len(args_w)): + argtype = self.arg_types[i] + conv = self.arg_converters[i] + args[i] = conv.convert_argument(space, args_w[i]) + except: + # fun :-( + for j in range(i): + lltype.free(args[j], flavor='raw') + lltype.free(args, flavor='raw') + raise + return args + + def free_arguments(self, args): + for i in range(len(self.arg_types)): + lltype.free(args[i], flavor='raw') + lltype.free(args, flavor='raw') + def __repr__(self): return "CPPFunction(%s, %s, %s, %s)" % ( self.cpptype, self.method_index, self.result_type, self.arg_types) @@ -149,7 +161,7 @@ class CPPFunction(CPPMethod): def call(self, cppthis, args_w): assert not cppthis - args = prepare_arguments(self.space, args_w, self.arg_types) + args = self.prepare_arguments(args_w) try: if self.result_type == "int": result = c_callstatic_l(self.cpptype.name, self.method_index, len(args_w), args) @@ -160,30 +172,31 @@ else: raise NotImplementedError finally: - free_arguments(args, len(args_w)) + self.free_arguments(args) class CPPConstructor(CPPFunction): def call(self, cppthis, args_w): assert not cppthis - args = prepare_arguments(self.space, args_w, self.arg_types) + args = self.prepare_arguments(args_w) result = c_construct(self.cpptype.name, len(args_w), args) - free_arguments(args, len(args_w)) + self.free_arguments(args) return W_CPPObject(self.cpptype, result) class CPPOverload(object): - def __init__(self, space, func_name): + _immutable_ = True + _immutable_fields_ = ["functions[*]"] + def __init__(self, space, func_name, functions): self.space = space self.func_name = func_name - self.functions = [] - - def add_function(self, cppfunc): - self.functions.append(cppfunc) + self.functions = debug.make_sure_not_resized(functions) + @jit.unroll_safe def call(self, cppthis, args_w): space = self.space - for cppyyfunc in self.functions: + for i in range(len(self.functions)): + cppyyfunc = self.functions[i] try: return cppyyfunc.call(cppthis, args_w) except OperationError, e: @@ -203,6 +216,8 @@ return string class W_CPPType(Wrappable): + _immutable_fields_ = ["cpplib", "name"] + def __init__(self, cpplib, name): self.space = cpplib.space self.cpplib = cpplib @@ -212,14 +227,15 @@ def _find_func_members(self): num_func_members = c_num_methods(self.name) + args_temp = {} for i in range(num_func_members): func_member_name = charp2str(c_method_name(self.name, i)) cppfunction = self._make_cppfunction(i) - overload = self.function_members.get(func_member_name, None) - if overload is None: - overload = CPPOverload(self.space, func_member_name) - self.function_members[func_member_name] = overload - overload.add_function(cppfunction) + overload = args_temp.setdefault(func_member_name, []) + overload.append(cppfunction) + for name, functions in args_temp.iteritems(): + overload = CPPOverload(self.space, name, functions[:]) + self.function_members[name] = overload def _make_cppfunction(self, method_index): result_type = charp2str(c_result_type_method(self.name, method_index)) @@ -236,12 +252,16 @@ cls = CPPMethod return cls(self, method_index, result_type, argtypes) + @jit.purefunction + def get_overload(self, name): + return self.function_members[name] + def invoke(self, name, args_w): - overload = self.function_members[name] + overload = self.get_overload(name) return overload.call(NULL_VOIDP, args_w) def construct(self, args_w): - overload = self.function_members[self.name] + overload = self.get_overload(self.name) return overload.call(NULL_VOIDP, args_w) W_CPPType.typedef = TypeDef( @@ -251,13 +271,14 @@ ) class W_CPPObject(Wrappable): + _immutable_ = True def __init__(self, cppclass, rawobject): self.space = cppclass.space self.cppclass = cppclass self.rawobject = rawobject def invoke(self, method_name, args_w): - overload = self.cppclass.function_members[method_name] + overload = self.cppclass.get_overload(method_name) return overload.call(self.rawobject, args_w) def destruct(self): From arigo at codespeak.net Tue Jul 6 16:10:16 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 16:10:16 +0200 (CEST) Subject: [pypy-svn] r75911 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100706141016.CDFDE282B9C@codespeak.net> Author: arigo Date: Tue Jul 6 16:10:15 2010 New Revision: 75911 Added: pypy/branch/rsre2/pypy/rlib/rsre/test/test_more.py - copied unchanged from r75886, pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py Removed: pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Log: Search, non-fast version. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Tue Jul 6 16:10:15 2010 @@ -331,15 +331,6 @@ # ____________________________________________________________ -def match(pattern, string, start=0, flags=0): - ctx = MatchContext(pattern, string, start, flags) - result = sre_match(ctx, 0, start, None) - if result is not None: - ctx.match_end = result.end - ctx.match_marks = result.marks - return ctx - return None - def sre_match(ctx, ppos, ptr, marks): """Returns either None or a MatchResult object. Usually we only need the first result, but there is the case of REPEAT...UNTIL where we @@ -706,3 +697,26 @@ return ctx.at_non_boundary(ptr, rsre_char.is_uni_word) return False + +# ____________________________________________________________ + +def match(pattern, string, start=0, flags=0): + ctx = MatchContext(pattern, string, start, flags) + result = sre_match(ctx, 0, start, None) + if result is not None: + ctx.match_end = result.end + ctx.match_marks = result.marks + return ctx + return None + +def search(pattern, string, start=0, flags=0): + ctx = MatchContext(pattern, string, start, flags) + while start <= ctx.end: + result = sre_match(ctx, 0, start, None) + if result is not None: + ctx.match_start = start + ctx.match_end = result.end + ctx.match_marks = result.marks + return ctx + start += 1 + return None From benjamin at codespeak.net Tue Jul 6 16:12:28 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 16:12:28 +0200 (CEST) Subject: [pypy-svn] r75912 - in pypy/trunk/pypy/annotation: . test Message-ID: <20100706141228.0B6E0282B9C@codespeak.net> Author: benjamin Date: Tue Jul 6 16:12:27 2010 New Revision: 75912 Modified: pypy/trunk/pypy/annotation/model.py pypy/trunk/pypy/annotation/test/test_model.py Log: make SomeFloats containing different nan's merge correctly Modified: pypy/trunk/pypy/annotation/model.py ============================================================================== --- pypy/trunk/pypy/annotation/model.py (original) +++ pypy/trunk/pypy/annotation/model.py Tue Jul 6 16:12:27 2010 @@ -34,7 +34,7 @@ from pypy.tool.pairtype import pair, extendabletype from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint, r_ulonglong, base_int -from pypy.rlib.rarithmetic import r_singlefloat +from pypy.rlib.rarithmetic import r_singlefloat, isnan import inspect, weakref DEBUG = False # set to False to disable recording of debugging information @@ -162,6 +162,13 @@ # pretend it's a float. immutable = True + def __eq__(self, other): + # NaN unpleasantness. + if (self.is_constant() and other.is_constant() and + isnan(self.const) and isnan(other.const)): + return True + return super(SomeFloat, self).__eq__(other) + def can_be_none(self): return False Modified: pypy/trunk/pypy/annotation/test/test_model.py ============================================================================== --- pypy/trunk/pypy/annotation/test/test_model.py (original) +++ pypy/trunk/pypy/annotation/test/test_model.py Tue Jul 6 16:12:27 2010 @@ -200,6 +200,15 @@ s2.const=cls assert s1.contains(s2) +def test_nan(): + f1 = SomeFloat() + f1.const = float("nan") + f2 = SomeFloat() + f2.const = float("nan") + assert f1.contains(f1) + assert f2.contains(f1) + assert f1.contains(f2) + if __name__ == '__main__': for name, value in globals().items(): if name.startswith('test_'): From cfbolz at codespeak.net Tue Jul 6 16:12:40 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Jul 2010 16:12:40 +0200 (CEST) Subject: [pypy-svn] r75913 - pypy/branch/reflex-support/pypy/jit/tl Message-ID: <20100706141240.CD950282B9C@codespeak.net> Author: cfbolz Date: Tue Jul 6 16:12:39 2010 New Revision: 75913 Modified: pypy/branch/reflex-support/pypy/jit/tl/pypyjit.py pypy/branch/reflex-support/pypy/jit/tl/pypyjit_demo.py Log: the code that we would like to make fast with the JIT Modified: pypy/branch/reflex-support/pypy/jit/tl/pypyjit.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/tl/pypyjit.py (original) +++ pypy/branch/reflex-support/pypy/jit/tl/pypyjit.py Tue Jul 6 16:12:39 2010 @@ -39,6 +39,7 @@ config.objspace.usemodules.pypyjit = True config.objspace.usemodules._weakref = False config.objspace.usemodules._sre = False +config.objspace.usemodules.cppyy = True set_pypy_opt_level(config, level='jit') config.objspace.std.withinlineddict = True Modified: pypy/branch/reflex-support/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/reflex-support/pypy/jit/tl/pypyjit_demo.py Tue Jul 6 16:12:39 2010 @@ -1,38 +1,14 @@ -base = object - -class Number(base): - __slots__ = ('val', ) - def __init__(self, val=0): - self.val = val - - def __add__(self, other): - if not isinstance(other, int): - other = other.val - return Number(val=self.val + other) - - def __cmp__(self, other): - val = self.val - if not isinstance(other, int): - other = other.val - return cmp(val, other) - - def __nonzero__(self): - return bool(self.val) - -def g(x, inc=2): - return x + inc - -def f(n, x, inc): - while x < n: - x = g(x, inc=1) - return x +import cppyy import time -#t1 = time.time() -#f(10000000, Number(), 1) -#t2 = time.time() -#print t2 - t1 +import cppyy +lib = cppyy.load_lib("example01Dict.so") +cls = lib.type_byname("example01") +inst = cls.construct(-17) + t1 = time.time() -f(10000000, 0, 1) +res = 0 +for i in range(1000000): + res += inst.invoke("add", i) t2 = time.time() print t2 - t1 From benjamin at codespeak.net Tue Jul 6 16:18:29 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 16:18:29 +0200 (CEST) Subject: [pypy-svn] r75914 - in pypy/branch/fast-forward: . lib_pypy pypy/annotation pypy/annotation/test pypy/jit/backend/llsupport pypy/jit/backend/llsupport/test pypy/jit/backend/x86 pypy/module/_locale pypy/module/test_lib_pypy/ctypes_tests pypy/rpython/memory Message-ID: <20100706141829.56A5F282B9C@codespeak.net> Author: benjamin Date: Tue Jul 6 16:18:27 2010 New Revision: 75914 Modified: pypy/branch/fast-forward/ (props changed) pypy/branch/fast-forward/lib_pypy/__init__.py pypy/branch/fast-forward/pypy/annotation/model.py pypy/branch/fast-forward/pypy/annotation/test/test_model.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/descr.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/symbolic.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_symbolic.py pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py pypy/branch/fast-forward/pypy/module/_locale/interp_locale.py pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) pypy/branch/fast-forward/pypy/rpython/memory/lltypelayout.py Log: merge from trunk Modified: pypy/branch/fast-forward/lib_pypy/__init__.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/__init__.py (original) +++ pypy/branch/fast-forward/lib_pypy/__init__.py Tue Jul 6 16:18:27 2010 @@ -1,4 +1,5 @@ # This __init__.py shows up in PyPy's app-level standard library. # Let's try to prevent that confusion... +import sys; print sys.path if __name__ != 'lib_pypy': raise ImportError, '__init__' Modified: pypy/branch/fast-forward/pypy/annotation/model.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/model.py (original) +++ pypy/branch/fast-forward/pypy/annotation/model.py Tue Jul 6 16:18:27 2010 @@ -34,7 +34,7 @@ from pypy.tool.pairtype import pair, extendabletype from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint, r_ulonglong, base_int -from pypy.rlib.rarithmetic import r_singlefloat +from pypy.rlib.rarithmetic import r_singlefloat, isnan import inspect, weakref DEBUG = False # set to False to disable recording of debugging information @@ -162,6 +162,13 @@ # pretend it's a float. immutable = True + def __eq__(self, other): + # NaN unpleasantness. + if (self.is_constant() and other.is_constant() and + isnan(self.const) and isnan(other.const)): + return True + return super(SomeFloat, self).__eq__(other) + def can_be_none(self): return False Modified: pypy/branch/fast-forward/pypy/annotation/test/test_model.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/test/test_model.py (original) +++ pypy/branch/fast-forward/pypy/annotation/test/test_model.py Tue Jul 6 16:18:27 2010 @@ -200,6 +200,15 @@ s2.const=cls assert s1.contains(s2) +def test_nan(): + f1 = SomeFloat() + f1.const = float("nan") + f2 = SomeFloat() + f2.const = float("nan") + assert f1.contains(f1) + assert f2.contains(f1) + assert f1.contains(f2) + if __name__ == '__main__': for name, value in globals().items(): if name.startswith('test_'): Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/descr.py Tue Jul 6 16:18:27 2010 @@ -151,7 +151,6 @@ def repr_of_descr(self): return '<%s>' % self._clsname - class NonGcPtrArrayDescr(BaseArrayDescr): _clsname = 'NonGcPtrArrayDescr' def get_item_size(self, translate_support_code): @@ -161,17 +160,45 @@ _clsname = 'GcPtrArrayDescr' _is_array_of_pointers = True +_CA = rffi.CArray(lltype.Signed) + +class BaseArrayNoLengthDescr(BaseArrayDescr): + def get_base_size(self, translate_support_code): + basesize, _, _ = symbolic.get_array_token(_CA, translate_support_code) + return basesize + + def get_ofs_length(self, translate_support_code): + _, _, ofslength = symbolic.get_array_token(_CA, translate_support_code) + return ofslength + +class NonGcPtrArrayNoLengthDescr(BaseArrayNoLengthDescr): + _clsname = 'NonGcPtrArrayNoLengthDescr' + def get_item_size(self, translate_support_code): + return symbolic.get_size_of_ptr(translate_support_code) + +class GcPtrArrayNoLengthDescr(NonGcPtrArrayNoLengthDescr): + _clsname = 'GcPtrArrayNoLengthDescr' + _is_array_of_pointers = True + def getArrayDescrClass(ARRAY): return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr, NonGcPtrArrayDescr, 'Array', 'get_item_size', '_is_array_of_floats') +def getArrayNoLengthDescrClass(ARRAY): + return getDescrClass(ARRAY.OF, BaseArrayNoLengthDescr, GcPtrArrayNoLengthDescr, + NonGcPtrArrayNoLengthDescr, 'ArrayNoLength', 'get_item_size', + '_is_array_of_floats') + def get_array_descr(gccache, ARRAY): cache = gccache._cache_array try: return cache[ARRAY] except KeyError: - arraydescr = getArrayDescrClass(ARRAY)() + if ARRAY._hints.get('nolength', False): + arraydescr = getArrayNoLengthDescrClass(ARRAY)() + else: + arraydescr = getArrayDescrClass(ARRAY)() # verify basic assumption that all arrays' basesize and ofslength # are equal basesize, itemsize, ofslength = symbolic.get_array_token(ARRAY, False) Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py Tue Jul 6 16:18:27 2010 @@ -250,6 +250,7 @@ ofs = arraydescr.get_ofs_length(self.translate_support_code) return rffi.cast(rffi.CArrayPtr(lltype.Signed), array)[ofs/WORD] + @specialize.argtype(2) def bh_getarrayitem_gc_i(self, arraydescr, gcref, itemindex): ofs, size = self.unpack_arraydescr_size(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -272,6 +273,7 @@ # --- end of GC unsafe code --- return pval + @specialize.argtype(2) def bh_getarrayitem_gc_f(self, arraydescr, gcref, itemindex): ofs = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -281,6 +283,7 @@ # --- end of GC unsafe code --- return fval + @specialize.argtype(2) def bh_setarrayitem_gc_i(self, arraydescr, gcref, itemindex, newvalue): ofs, size = self.unpack_arraydescr_size(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -303,6 +306,7 @@ items[itemindex] = self.cast_gcref_to_int(newvalue) # --- end of GC unsafe code --- + @specialize.argtype(2) def bh_setarrayitem_gc_f(self, arraydescr, gcref, itemindex, newvalue): ofs = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -311,6 +315,12 @@ items[itemindex] = newvalue # --- end of GC unsafe code --- + bh_setarrayitem_raw_i = bh_setarrayitem_gc_i + bh_setarrayitem_raw_f = bh_setarrayitem_gc_f + + bh_getarrayitem_raw_i = bh_getarrayitem_gc_i + bh_getarrayitem_raw_f = bh_getarrayitem_gc_f + def bh_strlen(self, string): s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) return len(s.chars) Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/symbolic.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/symbolic.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/symbolic.py Tue Jul 6 16:18:27 2010 @@ -36,8 +36,11 @@ ofs_length = (llmemory.offsetof(T, T._arrayfld) + llmemory.ArrayLengthOffset(SUBARRAY)) else: + if T._hints.get('nolength', None): + ofs_length = -1 + else: + ofs_length = llmemory.ArrayLengthOffset(T) itemsize = llmemory.sizeof(T.OF) - ofs_length = llmemory.ArrayLengthOffset(T) else: if isinstance(T, lltype.Struct): assert T._arrayfld is not None, "%r is not variable-sized" % (T,) @@ -48,8 +51,11 @@ else: before_array_part = 0 carray = ll2ctypes.get_ctypes_type(T) - assert carray.length.size == WORD - ofs_length = before_array_part + carray.length.offset + if T._hints.get('nolength', None): + ofs_length = -1 + else: + assert carray.length.size == WORD + ofs_length = before_array_part + carray.length.offset basesize = before_array_part + carray.items.offset carrayitem = ll2ctypes.get_ctypes_type(T.OF) itemsize = ctypes.sizeof(carrayitem) Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_descr.py Tue Jul 6 16:18:27 2010 @@ -140,7 +140,25 @@ assert isinstance(descr2.get_item_size(True), Symbolic) assert isinstance(descr3.get_item_size(True), Symbolic) assert isinstance(descr4.get_item_size(True), Symbolic) - + CA = rffi.CArray(lltype.Signed) + descr = get_array_descr(c0, CA) + assert not descr.is_array_of_floats() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Ptr(lltype.GcStruct('S'))) + descr = get_array_descr(c0, CA) + assert descr.is_array_of_pointers() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Ptr(lltype.Struct('S'))) + descr = get_array_descr(c0, CA) + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Float) + descr = get_array_descr(c0, CA) + assert descr.is_array_of_floats() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 def test_get_call_descr_not_translated(): c0 = GcCache(False) Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_symbolic.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_symbolic.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_symbolic.py Tue Jul 6 16:18:27 2010 @@ -61,6 +61,12 @@ assert basesize >= WORD # at least the 'length', maybe some gc headers assert itemsize == WORD assert ofs_length == basesize - WORD + A = rffi.CArray(lltype.Signed) + arraytok = get_array_token(A, translate_support_code) + basesize, itemsize, ofs_length = convert(arraytok) + assert basesize == 0 + assert itemsize == WORD + assert ofs_length == -1 def test_varsized_struct_size(): S1 = lltype.GcStruct('S1', ('parent', S), Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py Tue Jul 6 16:18:27 2010 @@ -815,6 +815,7 @@ raise NotImplementedError() genop_getarrayitem_gc_pure = genop_getarrayitem_gc + genop_getarrayitem_raw = genop_getarrayitem_gc def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py Tue Jul 6 16:18:27 2010 @@ -867,6 +867,7 @@ result_loc = self.force_allocate_reg(op.result) self.Perform(op, [base_loc, ofs_loc, imm(scale), imm(ofs)], result_loc) + consider_getarrayitem_raw = consider_getarrayitem_gc consider_getarrayitem_gc_pure = consider_getarrayitem_gc def consider_int_is_true(self, op, guard_op): Modified: pypy/branch/fast-forward/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_locale/interp_locale.py (original) +++ pypy/branch/fast-forward/pypy/module/_locale/interp_locale.py Tue Jul 6 16:18:27 2010 @@ -129,7 +129,13 @@ space.is_true(space.isinstance(w_s2, space.w_str)): s1, s2 = space.str_w(w_s1), space.str_w(w_s2) - return space.wrap(_strcoll(rffi.str2charp(s1), rffi.str2charp(s2))) + s1_c = rffi.str2charp(s1) + s2_c = rffi.str2charp(s2) + try: + return space.wrap(_strcoll(s1_c, s2_c)) + finally: + rffi.free_charp(s1_c) + rffi.free_charp(s2_c) #if not space.is_true(space.isinstance(w_s1, space.w_unicode)) and \ # not space.is_true(space.isinstance(w_s2, space.w_unicode)): @@ -140,7 +146,12 @@ s1_c = rffi.unicode2wcharp(s1) s2_c = rffi.unicode2wcharp(s2) - result = _wcscoll(s1_c, s2_c) + try: + result = _wcscoll(s1_c, s2_c) + finally: + rffi.free_wcharp(s1_c) + rffi.free_wcharp(s2_c) + return space.wrap(result) strcoll.unwrap_spec = [ObjSpace, W_Root, W_Root] @@ -153,13 +164,21 @@ n1 = len(s) + 1 buf = lltype.malloc(rffi.CCHARP.TO, n1, flavor="raw", zero=True) - n2 = _strxfrm(buf, rffi.str2charp(s), n1) + 1 + s_c = rffi.str2charp(s) + try: + n2 = _strxfrm(buf, s_c, n1) + 1 + finally: + rffi.free_charp(s_c) if n2 > n1: # more space needed lltype.free(buf, flavor="raw") buf = lltype.malloc(rffi.CCHARP.TO, intmask(n2), flavor="raw", zero=True) - _strxfrm(buf, rffi.str2charp(s), n2) + s_c = rffi.str2charp(s) + try: + _strxfrm(buf, s_c, n2) + finally: + rffi.free_charp(s_c) val = rffi.charp2str(buf) lltype.free(buf, flavor="raw") @@ -191,7 +210,11 @@ def gettext(space, msg): """gettext(msg) -> string Return translation of msg.""" - return space.wrap(rffi.charp2str(_gettext(rffi.str2charp(msg)))) + msg_c = rffi.str2charp(msg) + try: + return space.wrap(rffi.charp2str(_gettext(msg_c))) + finally: + rffi.free_charp(msg_c) gettext.unwrap_spec = [ObjSpace, str] @@ -202,10 +225,20 @@ Return translation of msg in domain.""" if space.is_w(w_domain, space.w_None): domain = None - result = _dgettext(domain, rffi.str2charp(msg)) + msg_c = rffi.str2charp(msg) + try: + result = _dgettext(domain, msg_c) + finally: + rffi.free_charp(msg_c) else: domain = space.str_w(w_domain) - result = _dgettext(rffi.str2charp(domain), rffi.str2charp(msg)) + domain_c = rffi.str2charp(domain) + msg_c = rffi.str2charp(msg) + try: + result = _dgettext(domain_c, msg_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(msg_c) return space.wrap(rffi.charp2str(result)) @@ -220,12 +253,21 @@ if space.is_w(w_domain, space.w_None): domain = None - result = _dcgettext(domain, rffi.str2charp(msg), - rffi.cast(rffi.INT, category)) + msg_c = rffi.str2charp(msg) + try: + result = _dcgettext(domain, msg_c, rffi.cast(rffi.INT, category)) + finally: + rffi.free_charp(msg_c) else: domain = space.str_w(w_domain) - result = _dcgettext(rffi.str2charp(domain), rffi.str2charp(msg), - rffi.cast(rffi.INT, category)) + domain_c = rffi.str2charp(domain) + msg_c = rffi.str2charp(msg) + try: + result = _dcgettext(domain_c, msg_c, + rffi.cast(rffi.INT, category)) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(msg_c) return space.wrap(rffi.charp2str(result)) @@ -243,7 +285,11 @@ result = _textdomain(domain) else: domain = space.str_w(w_domain) - result = _textdomain(rffi.str2charp(domain)) + domain_c = rffi.str2charp(domain) + try: + result = _textdomain(domain_c) + finally: + rffi.free_charp(domain_c) return space.wrap(rffi.charp2str(result)) @@ -258,11 +304,20 @@ if space.is_w(w_dir, space.w_None): dir = None - dirname = _bindtextdomain(rffi.str2charp(domain), dir) + domain_c = rffi.str2charp(domain) + try: + dirname = _bindtextdomain(domain_c, dir) + finally: + rffi.free_charp(domain_c) else: dir = space.str_w(w_dir) - dirname = _bindtextdomain(rffi.str2charp(domain), - rffi.str2charp(dir)) + domain_c = rffi.str2charp(domain) + dir_c = rffi.str2charp(dir) + try: + dirname = _bindtextdomain(domain_c, dir_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(dir_c) if not dirname: errno = rposix.get_errno() @@ -281,12 +336,20 @@ if space.is_w(w_codeset, space.w_None): codeset = None - result = _bind_textdomain_codeset( - rffi.str2charp(domain), codeset) + domain_c = rffi.str2charp(domain) + try: + result = _bind_textdomain_codeset(domain_c, codeset) + finally: + rffi.free_charp(domain_c) else: codeset = space.str_w(w_codeset) - result = _bind_textdomain_codeset(rffi.str2charp(domain), - rffi.str2charp(codeset)) + domain_c = rffi.str2charp(domain) + codeset_c = rffi.str2charp(codeset) + try: + result = _bind_textdomain_codeset(domain_c, codeset_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(codeset_c) if not result: return space.w_None Modified: pypy/branch/fast-forward/pypy/rpython/memory/lltypelayout.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/lltypelayout.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/lltypelayout.py Tue Jul 6 16:18:27 2010 @@ -111,6 +111,8 @@ elif isinstance(offset, llmemory.ItemOffset): return sizeof(offset.TYPE) * offset.repeat elif isinstance(offset, llmemory.ArrayItemsOffset): + if offset.TYPE._hints.get('nolength', None): + return 0 return get_fixed_size(lltype.Signed) elif isinstance(offset, llmemory.GCHeaderOffset): return sizeof(offset.gcheaderbuilder.HDR) From benjamin at codespeak.net Tue Jul 6 16:20:38 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 16:20:38 +0200 (CEST) Subject: [pypy-svn] r75915 - pypy/trunk/lib_pypy Message-ID: <20100706142038.23EAE282B9C@codespeak.net> Author: benjamin Date: Tue Jul 6 16:20:37 2010 New Revision: 75915 Modified: pypy/trunk/lib_pypy/__init__.py Log: kill old debugging aid Modified: pypy/trunk/lib_pypy/__init__.py ============================================================================== --- pypy/trunk/lib_pypy/__init__.py (original) +++ pypy/trunk/lib_pypy/__init__.py Tue Jul 6 16:20:37 2010 @@ -3,6 +3,5 @@ # Without this check, you would be able to do 'import __init__' from a pypy # prompt -import sys; print sys.path if __name__ != 'lib_pypy': raise ImportError, '__init__' From benjamin at codespeak.net Tue Jul 6 16:30:10 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 16:30:10 +0200 (CEST) Subject: [pypy-svn] r75916 - pypy/trunk/pypy/annotation Message-ID: <20100706143010.96D8E282B9C@codespeak.net> Author: benjamin Date: Tue Jul 6 16:30:09 2010 New Revision: 75916 Modified: pypy/trunk/pypy/annotation/bookkeeper.py Log: avoid invoking SomeObject equality on Ellipsis Modified: pypy/trunk/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/trunk/pypy/annotation/bookkeeper.py (original) +++ pypy/trunk/pypy/annotation/bookkeeper.py Tue Jul 6 16:30:09 2010 @@ -739,7 +739,7 @@ """ w_tuple = SomeTuple def newtuple(self, items_s): - if items_s == [Ellipsis]: + if len(items_s) == 1 and items_s[0] is Ellipsis: res = SomeObject() # hack to get a SomeObject as the *arg res.from_ellipsis = True return res From cfbolz at codespeak.net Tue Jul 6 16:31:33 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Jul 2010 16:31:33 +0200 (CEST) Subject: [pypy-svn] r75917 - pypy/branch/reflex-support/pypy/module/pypyjit Message-ID: <20100706143133.2C4D1282B9C@codespeak.net> Author: cfbolz Date: Tue Jul 6 16:31:31 2010 New Revision: 75917 Modified: pypy/branch/reflex-support/pypy/module/pypyjit/policy.py Log: let the JIT see our new module Modified: pypy/branch/reflex-support/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/reflex-support/pypy/module/pypyjit/policy.py Tue Jul 6 16:31:31 2010 @@ -11,7 +11,7 @@ if '.' in modname: modname, _ = modname.split('.', 1) if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions', - 'imp', 'sys']: + 'imp', 'sys', 'cppyy']: return True return False From benjamin at codespeak.net Tue Jul 6 16:33:06 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 16:33:06 +0200 (CEST) Subject: [pypy-svn] r75918 - in pypy/branch/fast-forward: . lib_pypy pypy/annotation pypy/module/test_lib_pypy/ctypes_tests Message-ID: <20100706143306.DFA99282B9C@codespeak.net> Author: benjamin Date: Tue Jul 6 16:33:05 2010 New Revision: 75918 Modified: pypy/branch/fast-forward/ (props changed) pypy/branch/fast-forward/lib_pypy/__init__.py pypy/branch/fast-forward/pypy/annotation/bookkeeper.py pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) Log: merge from trunk Modified: pypy/branch/fast-forward/lib_pypy/__init__.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/__init__.py (original) +++ pypy/branch/fast-forward/lib_pypy/__init__.py Tue Jul 6 16:33:05 2010 @@ -1,5 +1,4 @@ # This __init__.py shows up in PyPy's app-level standard library. # Let's try to prevent that confusion... -import sys; print sys.path if __name__ != 'lib_pypy': raise ImportError, '__init__' Modified: pypy/branch/fast-forward/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/fast-forward/pypy/annotation/bookkeeper.py Tue Jul 6 16:33:05 2010 @@ -739,7 +739,7 @@ """ w_tuple = SomeTuple def newtuple(self, items_s): - if items_s == [Ellipsis]: + if len(items_s) == 1 and items_s[0] is Ellipsis: res = SomeObject() # hack to get a SomeObject as the *arg res.from_ellipsis = True return res From benjamin at codespeak.net Tue Jul 6 16:54:22 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 16:54:22 +0200 (CEST) Subject: [pypy-svn] r75922 - pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests Message-ID: <20100706145422.32EA7282B9C@codespeak.net> Author: benjamin Date: Tue Jul 6 16:54:20 2010 New Revision: 75922 Modified: pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) Log: kill prop From benjamin at codespeak.net Tue Jul 6 17:10:21 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Jul 2010 17:10:21 +0200 (CEST) Subject: [pypy-svn] r75923 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100706151021.6C871282B9C@codespeak.net> Author: benjamin Date: Tue Jul 6 17:10:19 2010 New Revision: 75923 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py Log: fix untested error case Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Tue Jul 6 17:10:19 2010 @@ -123,8 +123,8 @@ if c == "!": i += 1 if i == end: - raise OperationError(space.w_ValueError, - space.wrap("expected conversion")) + w_msg = self.space.wrap("expected conversion") + raise OperationError(self.space.w_ValueError, w_msg) conversion = s[i] else: conversion = None Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py Tue Jul 6 17:10:19 2010 @@ -73,6 +73,7 @@ def test_invalid_conversion(self): raises(ValueError, self.s("{!x}").format, 3) + raises(ValueError, self.s("{!}").format) def test_recursive(self): assert self.s("{:{}}").format(42, "#o") == self.s("0o52") From hakanardo at codespeak.net Tue Jul 6 17:12:19 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 6 Jul 2010 17:12:19 +0200 (CEST) Subject: [pypy-svn] r75924 - in pypy/branch/interplevel-array: lib_pypy pypy/jit/tl pypy/module/array pypy/module/array/test Message-ID: <20100706151219.A2508282B9C@codespeak.net> Author: hakanardo Date: Tue Jul 6 17:12:17 2010 New Revision: 75924 Added: pypy/branch/interplevel-array/lib_pypy/apparray.py Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: still not compiling Added: pypy/branch/interplevel-array/lib_pypy/apparray.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/lib_pypy/apparray.py Tue Jul 6 17:12:17 2010 @@ -0,0 +1,24 @@ +from array import array as _array + +class array(object): + def __init__(self, typecode, initializer=None): + self._array = _array(typecode) + if initializer is not None: + self.extend(initializer) + + + def append(self ,x): + self._array.append(x) + def __getitem__(self, idx): + return self._array[idx] + def __setitem__(self, idx, val): + self._array[idx]=val + def __len__(self): + return len(self._array) + + + def extend(self, iterable): + for i in iterable: self.append(i) + + + Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py Tue Jul 6 17:12:17 2010 @@ -38,11 +38,11 @@ ## print t2 - t1 from array import array -img=array(4) -def f(): +def f(img): sa=0 for i in xrange(4): sa+=img[i] return sa -print f() +img=array(4) +print f(img) Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Jul 6 17:12:17 2010 @@ -9,25 +9,7 @@ from pypy.rlib.rstruct.runpack import runpack class W_ArrayBase(Wrappable): - def descr_append(self, w_x): - x = self.item_w(w_x) - self.setlen(self.len + 1) - self.buffer[self.len - 1] = x - descr_append.unwrap_spec = ['self', W_Root] - - def descr_extend(self, w_initializer): - space = self.space - w_iterator = space.iter(w_initializer) - while True: - try: - w_item = space.next(w_iterator) - except OperationError, e: - if not e.match(space, space.w_StopIteration): - raise - break - self.descr_append(w_item) - descr_extend.unwrap_spec = ['self', W_Root] - + pass class TypeCode(object): def __init__(self, itemtype, unwrap, canoverflow=False, signed=False): @@ -43,6 +25,11 @@ self.w_class = None + def _freeze_(self): + # hint for the annotator: track individual constant instances + return True + + types = { 'c': TypeCode(lltype.Char, 'str_w'), 'u': TypeCode(lltype.UniChar, 'unicode_w'), @@ -53,7 +40,7 @@ 'i': TypeCode(rffi.INT, 'int_w', True, True), 'I': TypeCode(rffi.UINT, 'int_w', True), 'l': TypeCode(rffi.LONG, 'int_w', True, True), - 'L': TypeCode(rffi.ULONG, 'bigint_w', True), + #'L': TypeCode(rffi.ULONG, 'bigint_w', True), # FIXME: Won't compile 'f': TypeCode(lltype.SingleFloat, 'float_w'), 'd': TypeCode(lltype.Float, 'float_w'), } @@ -66,7 +53,7 @@ def __init__(self, space): self.space = space self.len = 0 - self.buffer = None + self.buffer = lltype.nullptr(self.mytype.arraytype) def item_w(self, w_item): space = self.space @@ -85,14 +72,14 @@ if self.mytype.canoverflow: msg = None if self.mytype.signed: - if item < -2 ** (self.mytype.bytes * 8) / 2: + if item < -1 << (self.mytype.bytes * 8 - 1): msg = 'signed %d-byte integer is less than minimum' % self.mytype.bytes - elif item > 2 ** (self.mytype.bytes * 8) / 2 - 1: + elif item > (1 << (self.mytype.bytes * 8 - 1)) - 1: msg = 'signed %d-byte integer is greater than maximum' % self.mytype.bytes else: if item < 0: msg = 'unsigned %d-byte integer is less than minimum' % self.mytype.bytes - elif item > 2 ** (self.mytype.bytes * 8) - 1: + elif item > (1 << (self.mytype.bytes * 8)) - 1: msg = 'unsigned %d-byte integer is greater than maximum' % self.mytype.bytes if msg is not None: raise OperationError(space.w_OverflowError, space.wrap(msg)) @@ -117,7 +104,8 @@ start, stop, step = space.decode_index(w_idx, self.len) if step==0: item = self.buffer[start] - if self.mytype.typecode in ('b', 'B', 'h', 'H', 'i', 'l'): + tc=self.mytype.typecode + if tc == 'b' or tc == 'B' or tc == 'h' or tc == 'H' or tc == 'i' or tc == 'l': item = rffi.cast(lltype.Signed, item) elif self.mytype.typecode == 'f': item = float(item) @@ -125,7 +113,7 @@ else: size = (stop - start) / step if (stop - start) % step > 0: size += 1 - w_a=W_Array(self.space, self.mytype.typecode) + w_a=self.mytype.w_class(self.space) w_a.setlen(size) j=0 for i in range(start, stop, step): @@ -134,6 +122,54 @@ return w_a descr_getitem.unwrap_spec = ['self', W_Root] + + def descr_append(self, w_x): + x = self.item_w(w_x) + self.setlen(self.len + 1) + self.buffer[self.len - 1] = x + descr_append.unwrap_spec = ['self', W_Root] + + + def descr_extend(self, w_initializer): + space = self.space + w_iterator = space.iter(w_initializer) + while True: + try: + w_item = space.next(w_iterator) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break + self.descr_append(w_item) + descr_extend.unwrap_spec = ['self', W_Root] + + + def descr_setitem(self, w_idx, w_item): + start, stop, step = self.space.decode_index(w_idx, self.len) + if step==0: + item = self.item_w(w_item) + self.buffer[start] = item + else: + if isinstance(w_item, W_Array): + if self.mytype.typecode == w_item.mytype.typecode: + size = (stop - start) / step + if (stop - start) % step > 0: size += 1 + if w_item.len != size: # FIXME: Support for step=1 + msg = ('attempt to assign array of size %d to ' + + 'slice of size %d') % (w_item.len, size) + raise OperationError(self.space.w_ValueError, + self.space.wrap(msg)) + j=0 + for i in range(start, stop, step): + self.buffer[i]=w_item.buffer[j] + j+=1 + return + msg='can only assign array to array slice' + raise OperationError(self.space.w_TypeError, self.space.wrap(msg)) + + descr_setitem.unwrap_spec = ['self', W_Root, W_Root] + + W_Array.__name__ = 'W_ArrayType_'+thetypecode W_Array.typedef = TypeDef( 'ArrayType_'+thetypecode, @@ -141,6 +177,7 @@ extend = interp2app(W_Array.descr_extend), __len__ = interp2app(W_Array.descr_len), __getitem__ = interp2app(W_Array.descr_getitem), + __setitem__ = interp2app(W_Array.descr_setitem), ) thetype.w_class = W_Array @@ -155,22 +192,21 @@ for tc in unroll_typecodes: if typecode == tc: a = types[tc].w_class(space) + if w_initializer is not None: + if not space.is_w(w_initializer, space.w_None): + a.descr_extend(w_initializer) + ## if space.is_w(space.type(w_initializer), space.w_str): + ## a.descr_fromstring(space.str_w(w_initializer)) + ## elif space.is_w(space.type(w_initializer), space.w_unicode): + ## a.descr_fromunicode(space.unicode_w(w_initializer)) + ## elif space.is_w(space.type(w_initializer), space.w_list): + ## a.descr_fromlist(w_initializer) + ## elif not space.is_w(w_initializer, space.w_None): + ## a.descr_extend(w_initializer) break else: msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' raise OperationError(space.w_ValueError, space.wrap(msg)) - - if w_initializer is not None: - if not space.is_w(w_initializer, space.w_None): - a.descr_extend(w_initializer) - ## if space.is_w(space.type(w_initializer), space.w_str): - ## a.descr_fromstring(space.str_w(w_initializer)) - ## elif space.is_w(space.type(w_initializer), space.w_unicode): - ## a.descr_fromunicode(space.unicode_w(w_initializer)) - ## elif space.is_w(space.type(w_initializer), space.w_list): - ## a.descr_fromlist(w_initializer) - ## elif not space.is_w(w_initializer, space.w_None): - ## a.descr_extend(w_initializer) return a array.unwrap_spec = (ObjSpace, str, W_Root) Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Tue Jul 6 17:12:17 2010 @@ -122,7 +122,6 @@ a = self.array(t.upper(), [1, 2, 3]) b = a.itemsize for v in (0, 2 ** (8 * b) - 1): - print b, v a[1] = v assert a[0] == 1 and a[1] == v and a[2] == 3 raises(OverflowError, a.append, -1) @@ -285,3 +284,14 @@ raises(ValueError, self.array('i').tounicode) assert self.array('u', unicode('hello')).tounicode() == unicode('hello') + def test_type(self): + for t in 'bBhHiIlLfdcu': + assert type(self.array(t)) is self.array + +class AppTestAppArray(AppTestArray): + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array',)) + cls.w_array = cls.space.appexec([], """(): + import apparray + return apparray.array + """) From cfbolz at codespeak.net Tue Jul 6 17:17:11 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Jul 2010 17:17:11 +0200 (CEST) Subject: [pypy-svn] r75925 - in pypy/branch/reflex-support/pypy/jit/codewriter: . test Message-ID: <20100706151711.CBE41282B9C@codespeak.net> Author: cfbolz Date: Tue Jul 6 17:17:10 2010 New Revision: 75925 Modified: pypy/branch/reflex-support/pypy/jit/codewriter/jtransform.py pypy/branch/reflex-support/pypy/jit/codewriter/test/test_flatten.py Log: start some support for very simple force_cast cases Modified: pypy/branch/reflex-support/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/reflex-support/pypy/jit/codewriter/jtransform.py Tue Jul 6 17:17:10 2010 @@ -667,7 +667,7 @@ return self._rewrite_symmetric(op) def _is_gc(self, v): - return v.concretetype.TO._gckind == 'gc' + return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc' def _rewrite_cmp_ptrs(self, op): if self._is_gc(op.args[0]): @@ -701,6 +701,17 @@ #return op raise NotImplementedError("cast_ptr_to_int") + def rewrite_op_force_cast(self, op): + from pypy.rpython.lltypesystem.rffi import size_and_sign + from pypy.rlib.rarithmetic import intmask + assert not self._is_gc(op.args[0]) + size1, unsigned1 = size_and_sign(op.args[0].concretetype) + size2, unsigned2 = size_and_sign(op.result.concretetype) + if size1 == size2 and unsigned1 == unsigned2: + return + raise NotImplementedError("cast not supported yet: %s" % (op, )) + + # ---------- # Renames, from the _old opname to the _new one. # The new operation is optionally further processed by rewrite_operation(). Modified: pypy/branch/reflex-support/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/reflex-support/pypy/jit/codewriter/test/test_flatten.py Tue Jul 6 17:17:10 2010 @@ -729,3 +729,32 @@ int_between %i0, %i1, %i2 -> %i3 int_return %i3 """, transform=True) + + def test_force_cast(self): + py.test.skip("later") + from pypy.rpython.lltypesystem import rffi + def f(n): + c = chr(n) + return rffi.cast(rffi.INT, c) + self.encoding_test(f, [42], """ + int_return %i0 + """, transform=True) + def g(n): + return rffi.cast(rffi.UCHAR, n) + self.encoding_test(g, [42], """ + int_and %i0, $255 -> %i1 + int_return %i1 + """, transform=True) + def h(n): + return rffi.cast(rffi.SCHAR, n) + self.encoding_test(h, [42], """ + ... + """, transform=True) + + def test_force_cast_pointer(self): + from pypy.rpython.lltypesystem import rffi + def h(p): + return rffi.cast(rffi.VOIDP, p) + self.encoding_test(h, [lltype.nullptr(rffi.CCHARP.TO)], """ + int_return %i0 + """, transform=True) From arigo at codespeak.net Tue Jul 6 17:35:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 17:35:37 +0200 (CEST) Subject: [pypy-svn] r75927 - pypy/branch/rsre2/pypy/rlib/rsre Message-ID: <20100706153537.A3D8B282C05@codespeak.net> Author: arigo Date: Tue Jul 6 17:35:36 2010 New Revision: 75927 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Log: Fast search. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Tue Jul 6 17:35:36 2010 @@ -711,6 +711,13 @@ def search(pattern, string, start=0, flags=0): ctx = MatchContext(pattern, string, start, flags) + if ctx.pat(0) == OPCODE_INFO: + if ctx.pat(2) & rsre_char.SRE_INFO_PREFIX and ctx.pat(5) > 1: + return fast_search(ctx) + return regular_search(ctx) + +def regular_search(ctx): + start = ctx.match_start while start <= ctx.end: result = sre_match(ctx, 0, start, None) if result is not None: @@ -720,3 +727,48 @@ return ctx start += 1 return None + +def fast_search(ctx): + # skips forward in a string as fast as possible using information from + # an optimization info block + # <1=skip> <2=flags> <3=min> <4=...> + # <5=length> <6=skip> <7=prefix data> + flags = ctx.pat(2) + prefix_len = ctx.pat(5) + assert prefix_len >= 0 + prefix_skip = ctx.pat(6) + assert prefix_skip >= 0 + overlap_offset = 7 + prefix_len - 1 + assert overlap_offset >= 0 + pattern_offset = ctx.pat(1) + 1 + assert pattern_offset >= 0 + i = 0 + string_position = ctx.match_start + end = ctx.end + while string_position < end: + while True: + char_ord = ctx.str(string_position) + if char_ord != ctx.pat(7 + i): + if i == 0: + break + else: + i = ctx.pat(overlap_offset + i) + else: + i += 1 + if i == prefix_len: + # found a potential match + if flags & rsre_char.SRE_INFO_LITERAL: + return True # matched all of pure literal pattern + start = string_position + 1 - prefix_len + ptr = start + prefix_skip + ppos = pattern_offset + 2 * prefix_skip + result = sre_match(ctx, ppos, ptr, None) + if result is not None: + ctx.match_start = start + ctx.match_end = result.end + ctx.match_marks = result.marks + return ctx + i = ctx.pat(overlap_offset + i) + break + string_position += 1 + return None From cfbolz at codespeak.net Tue Jul 6 17:36:03 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Jul 2010 17:36:03 +0200 (CEST) Subject: [pypy-svn] r75928 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100706153603.4B011282C05@codespeak.net> Author: cfbolz Date: Tue Jul 6 17:36:01 2010 New Revision: 75928 Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Log: add some more hints Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Tue Jul 6 17:36:01 2010 @@ -128,6 +128,7 @@ self.arg_converters = [converter.get_converter(arg_type) for arg_type in self.arg_types] + @jit.unroll_safe def prepare_arguments(self, args_w): space = self.space if len(args_w) != len(self.arg_types): @@ -278,7 +279,8 @@ self.rawobject = rawobject def invoke(self, method_name, args_w): - overload = self.cppclass.get_overload(method_name) + cppclass = jit.hint(self.cppclass, promote=True) + overload = cppclass.get_overload(method_name) return overload.call(self.rawobject, args_w) def destruct(self): From getxsick at codespeak.net Tue Jul 6 18:14:54 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 6 Jul 2010 18:14:54 +0200 (CEST) Subject: [pypy-svn] r75930 - pypy/branch/2.6ish Message-ID: <20100706161454.0CB47282C07@codespeak.net> Author: getxsick Date: Tue Jul 6 18:14:53 2010 New Revision: 75930 Removed: pypy/branch/2.6ish/ Log: kill unused and outdated branch. there is fast-forward branch which overwrites this one. From arigo at codespeak.net Tue Jul 6 18:27:01 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 18:27:01 +0200 (CEST) Subject: [pypy-svn] r75931 - in pypy/branch/reflex-support/pypy: annotation annotation/test jit/backend/llsupport jit/backend/llsupport/test jit/backend/x86 module/_locale rlib rpython/lltypesystem rpython/lltypesystem/test rpython/memory Message-ID: <20100706162701.41C74282C05@codespeak.net> Author: arigo Date: Tue Jul 6 18:26:59 2010 New Revision: 75931 Modified: pypy/branch/reflex-support/pypy/annotation/bookkeeper.py pypy/branch/reflex-support/pypy/annotation/model.py pypy/branch/reflex-support/pypy/annotation/test/test_model.py pypy/branch/reflex-support/pypy/jit/backend/llsupport/descr.py pypy/branch/reflex-support/pypy/jit/backend/llsupport/llmodel.py pypy/branch/reflex-support/pypy/jit/backend/llsupport/symbolic.py pypy/branch/reflex-support/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/reflex-support/pypy/jit/backend/llsupport/test/test_symbolic.py pypy/branch/reflex-support/pypy/jit/backend/x86/assembler.py pypy/branch/reflex-support/pypy/jit/backend/x86/regalloc.py pypy/branch/reflex-support/pypy/module/_locale/interp_locale.py pypy/branch/reflex-support/pypy/rlib/rlocale.py pypy/branch/reflex-support/pypy/rpython/lltypesystem/rffi.py pypy/branch/reflex-support/pypy/rpython/lltypesystem/test/test_rffi.py pypy/branch/reflex-support/pypy/rpython/memory/lltypelayout.py Log: Merge trunk, r75828:HEAD. Modified: pypy/branch/reflex-support/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/reflex-support/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/reflex-support/pypy/annotation/bookkeeper.py Tue Jul 6 18:26:59 2010 @@ -739,7 +739,7 @@ """ w_tuple = SomeTuple def newtuple(self, items_s): - if items_s == [Ellipsis]: + if len(items_s) == 1 and items_s[0] is Ellipsis: res = SomeObject() # hack to get a SomeObject as the *arg res.from_ellipsis = True return res Modified: pypy/branch/reflex-support/pypy/annotation/model.py ============================================================================== --- pypy/branch/reflex-support/pypy/annotation/model.py (original) +++ pypy/branch/reflex-support/pypy/annotation/model.py Tue Jul 6 18:26:59 2010 @@ -34,7 +34,7 @@ from pypy.tool.pairtype import pair, extendabletype from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint, r_ulonglong, base_int -from pypy.rlib.rarithmetic import r_singlefloat +from pypy.rlib.rarithmetic import r_singlefloat, isnan import inspect, weakref DEBUG = False # set to False to disable recording of debugging information @@ -162,6 +162,13 @@ # pretend it's a float. immutable = True + def __eq__(self, other): + # NaN unpleasantness. + if (self.is_constant() and other.is_constant() and + isnan(self.const) and isnan(other.const)): + return True + return super(SomeFloat, self).__eq__(other) + def can_be_none(self): return False Modified: pypy/branch/reflex-support/pypy/annotation/test/test_model.py ============================================================================== --- pypy/branch/reflex-support/pypy/annotation/test/test_model.py (original) +++ pypy/branch/reflex-support/pypy/annotation/test/test_model.py Tue Jul 6 18:26:59 2010 @@ -200,6 +200,15 @@ s2.const=cls assert s1.contains(s2) +def test_nan(): + f1 = SomeFloat() + f1.const = float("nan") + f2 = SomeFloat() + f2.const = float("nan") + assert f1.contains(f1) + assert f2.contains(f1) + assert f1.contains(f2) + if __name__ == '__main__': for name, value in globals().items(): if name.startswith('test_'): Modified: pypy/branch/reflex-support/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/reflex-support/pypy/jit/backend/llsupport/descr.py Tue Jul 6 18:26:59 2010 @@ -151,7 +151,6 @@ def repr_of_descr(self): return '<%s>' % self._clsname - class NonGcPtrArrayDescr(BaseArrayDescr): _clsname = 'NonGcPtrArrayDescr' def get_item_size(self, translate_support_code): @@ -161,17 +160,45 @@ _clsname = 'GcPtrArrayDescr' _is_array_of_pointers = True +_CA = rffi.CArray(lltype.Signed) + +class BaseArrayNoLengthDescr(BaseArrayDescr): + def get_base_size(self, translate_support_code): + basesize, _, _ = symbolic.get_array_token(_CA, translate_support_code) + return basesize + + def get_ofs_length(self, translate_support_code): + _, _, ofslength = symbolic.get_array_token(_CA, translate_support_code) + return ofslength + +class NonGcPtrArrayNoLengthDescr(BaseArrayNoLengthDescr): + _clsname = 'NonGcPtrArrayNoLengthDescr' + def get_item_size(self, translate_support_code): + return symbolic.get_size_of_ptr(translate_support_code) + +class GcPtrArrayNoLengthDescr(NonGcPtrArrayNoLengthDescr): + _clsname = 'GcPtrArrayNoLengthDescr' + _is_array_of_pointers = True + def getArrayDescrClass(ARRAY): return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr, NonGcPtrArrayDescr, 'Array', 'get_item_size', '_is_array_of_floats') +def getArrayNoLengthDescrClass(ARRAY): + return getDescrClass(ARRAY.OF, BaseArrayNoLengthDescr, GcPtrArrayNoLengthDescr, + NonGcPtrArrayNoLengthDescr, 'ArrayNoLength', 'get_item_size', + '_is_array_of_floats') + def get_array_descr(gccache, ARRAY): cache = gccache._cache_array try: return cache[ARRAY] except KeyError: - arraydescr = getArrayDescrClass(ARRAY)() + if ARRAY._hints.get('nolength', False): + arraydescr = getArrayNoLengthDescrClass(ARRAY)() + else: + arraydescr = getArrayDescrClass(ARRAY)() # verify basic assumption that all arrays' basesize and ofslength # are equal basesize, itemsize, ofslength = symbolic.get_array_token(ARRAY, False) Modified: pypy/branch/reflex-support/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/reflex-support/pypy/jit/backend/llsupport/llmodel.py Tue Jul 6 18:26:59 2010 @@ -250,6 +250,7 @@ ofs = arraydescr.get_ofs_length(self.translate_support_code) return rffi.cast(rffi.CArrayPtr(lltype.Signed), array)[ofs/WORD] + @specialize.argtype(2) def bh_getarrayitem_gc_i(self, arraydescr, gcref, itemindex): ofs, size = self.unpack_arraydescr_size(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -272,6 +273,7 @@ # --- end of GC unsafe code --- return pval + @specialize.argtype(2) def bh_getarrayitem_gc_f(self, arraydescr, gcref, itemindex): ofs = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -281,6 +283,7 @@ # --- end of GC unsafe code --- return fval + @specialize.argtype(2) def bh_setarrayitem_gc_i(self, arraydescr, gcref, itemindex, newvalue): ofs, size = self.unpack_arraydescr_size(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -303,6 +306,7 @@ items[itemindex] = self.cast_gcref_to_int(newvalue) # --- end of GC unsafe code --- + @specialize.argtype(2) def bh_setarrayitem_gc_f(self, arraydescr, gcref, itemindex, newvalue): ofs = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -311,6 +315,12 @@ items[itemindex] = newvalue # --- end of GC unsafe code --- + bh_setarrayitem_raw_i = bh_setarrayitem_gc_i + bh_setarrayitem_raw_f = bh_setarrayitem_gc_f + + bh_getarrayitem_raw_i = bh_getarrayitem_gc_i + bh_getarrayitem_raw_f = bh_getarrayitem_gc_f + def bh_strlen(self, string): s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) return len(s.chars) Modified: pypy/branch/reflex-support/pypy/jit/backend/llsupport/symbolic.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/backend/llsupport/symbolic.py (original) +++ pypy/branch/reflex-support/pypy/jit/backend/llsupport/symbolic.py Tue Jul 6 18:26:59 2010 @@ -36,8 +36,11 @@ ofs_length = (llmemory.offsetof(T, T._arrayfld) + llmemory.ArrayLengthOffset(SUBARRAY)) else: + if T._hints.get('nolength', None): + ofs_length = -1 + else: + ofs_length = llmemory.ArrayLengthOffset(T) itemsize = llmemory.sizeof(T.OF) - ofs_length = llmemory.ArrayLengthOffset(T) else: if isinstance(T, lltype.Struct): assert T._arrayfld is not None, "%r is not variable-sized" % (T,) @@ -48,8 +51,11 @@ else: before_array_part = 0 carray = ll2ctypes.get_ctypes_type(T) - assert carray.length.size == WORD - ofs_length = before_array_part + carray.length.offset + if T._hints.get('nolength', None): + ofs_length = -1 + else: + assert carray.length.size == WORD + ofs_length = before_array_part + carray.length.offset basesize = before_array_part + carray.items.offset carrayitem = ll2ctypes.get_ctypes_type(T.OF) itemsize = ctypes.sizeof(carrayitem) Modified: pypy/branch/reflex-support/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/reflex-support/pypy/jit/backend/llsupport/test/test_descr.py Tue Jul 6 18:26:59 2010 @@ -140,7 +140,25 @@ assert isinstance(descr2.get_item_size(True), Symbolic) assert isinstance(descr3.get_item_size(True), Symbolic) assert isinstance(descr4.get_item_size(True), Symbolic) - + CA = rffi.CArray(lltype.Signed) + descr = get_array_descr(c0, CA) + assert not descr.is_array_of_floats() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Ptr(lltype.GcStruct('S'))) + descr = get_array_descr(c0, CA) + assert descr.is_array_of_pointers() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Ptr(lltype.Struct('S'))) + descr = get_array_descr(c0, CA) + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Float) + descr = get_array_descr(c0, CA) + assert descr.is_array_of_floats() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 def test_get_call_descr_not_translated(): c0 = GcCache(False) Modified: pypy/branch/reflex-support/pypy/jit/backend/llsupport/test/test_symbolic.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/backend/llsupport/test/test_symbolic.py (original) +++ pypy/branch/reflex-support/pypy/jit/backend/llsupport/test/test_symbolic.py Tue Jul 6 18:26:59 2010 @@ -61,6 +61,12 @@ assert basesize >= WORD # at least the 'length', maybe some gc headers assert itemsize == WORD assert ofs_length == basesize - WORD + A = rffi.CArray(lltype.Signed) + arraytok = get_array_token(A, translate_support_code) + basesize, itemsize, ofs_length = convert(arraytok) + assert basesize == 0 + assert itemsize == WORD + assert ofs_length == -1 def test_varsized_struct_size(): S1 = lltype.GcStruct('S1', ('parent', S), Modified: pypy/branch/reflex-support/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/reflex-support/pypy/jit/backend/x86/assembler.py Tue Jul 6 18:26:59 2010 @@ -815,6 +815,7 @@ raise NotImplementedError() genop_getarrayitem_gc_pure = genop_getarrayitem_gc + genop_getarrayitem_raw = genop_getarrayitem_gc def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs Modified: pypy/branch/reflex-support/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/reflex-support/pypy/jit/backend/x86/regalloc.py Tue Jul 6 18:26:59 2010 @@ -867,6 +867,7 @@ result_loc = self.force_allocate_reg(op.result) self.Perform(op, [base_loc, ofs_loc, imm(scale), imm(ofs)], result_loc) + consider_getarrayitem_raw = consider_getarrayitem_gc consider_getarrayitem_gc_pure = consider_getarrayitem_gc def consider_int_is_true(self, op, guard_op): Modified: pypy/branch/reflex-support/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/_locale/interp_locale.py (original) +++ pypy/branch/reflex-support/pypy/module/_locale/interp_locale.py Tue Jul 6 18:26:59 2010 @@ -132,7 +132,13 @@ space.is_true(space.isinstance(w_s2, space.w_str)): s1, s2 = space.str_w(w_s1), space.str_w(w_s2) - return space.wrap(_strcoll(rffi.str2charp(s1), rffi.str2charp(s2))) + s1_c = rffi.str2charp(s1) + s2_c = rffi.str2charp(s2) + try: + return space.wrap(_strcoll(s1_c, s2_c)) + finally: + rffi.free_charp(s1_c) + rffi.free_charp(s2_c) #if not space.is_true(space.isinstance(w_s1, space.w_unicode)) and \ # not space.is_true(space.isinstance(w_s2, space.w_unicode)): @@ -143,7 +149,12 @@ s1_c = rffi.unicode2wcharp(s1) s2_c = rffi.unicode2wcharp(s2) - result = _wcscoll(s1_c, s2_c) + try: + result = _wcscoll(s1_c, s2_c) + finally: + rffi.free_wcharp(s1_c) + rffi.free_wcharp(s2_c) + return space.wrap(result) strcoll.unwrap_spec = [ObjSpace, W_Root, W_Root] @@ -156,13 +167,21 @@ n1 = len(s) + 1 buf = lltype.malloc(rffi.CCHARP.TO, n1, flavor="raw", zero=True) - n2 = _strxfrm(buf, rffi.str2charp(s), n1) + 1 + s_c = rffi.str2charp(s) + try: + n2 = _strxfrm(buf, s_c, n1) + 1 + finally: + rffi.free_charp(s_c) if n2 > n1: # more space needed lltype.free(buf, flavor="raw") buf = lltype.malloc(rffi.CCHARP.TO, intmask(n2), flavor="raw", zero=True) - _strxfrm(buf, rffi.str2charp(s), n2) + s_c = rffi.str2charp(s) + try: + _strxfrm(buf, s_c, n2) + finally: + rffi.free_charp(s_c) val = rffi.charp2str(buf) lltype.free(buf, flavor="raw") @@ -194,7 +213,11 @@ def gettext(space, msg): """gettext(msg) -> string Return translation of msg.""" - return space.wrap(rffi.charp2str(_gettext(rffi.str2charp(msg)))) + msg_c = rffi.str2charp(msg) + try: + return space.wrap(rffi.charp2str(_gettext(msg_c))) + finally: + rffi.free_charp(msg_c) gettext.unwrap_spec = [ObjSpace, str] @@ -205,10 +228,20 @@ Return translation of msg in domain.""" if space.is_w(w_domain, space.w_None): domain = None - result = _dgettext(domain, rffi.str2charp(msg)) + msg_c = rffi.str2charp(msg) + try: + result = _dgettext(domain, msg_c) + finally: + rffi.free_charp(msg_c) else: domain = space.str_w(w_domain) - result = _dgettext(rffi.str2charp(domain), rffi.str2charp(msg)) + domain_c = rffi.str2charp(domain) + msg_c = rffi.str2charp(msg) + try: + result = _dgettext(domain_c, msg_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(msg_c) return space.wrap(rffi.charp2str(result)) @@ -223,12 +256,21 @@ if space.is_w(w_domain, space.w_None): domain = None - result = _dcgettext(domain, rffi.str2charp(msg), - rffi.cast(rffi.INT, category)) + msg_c = rffi.str2charp(msg) + try: + result = _dcgettext(domain, msg_c, rffi.cast(rffi.INT, category)) + finally: + rffi.free_charp(msg_c) else: domain = space.str_w(w_domain) - result = _dcgettext(rffi.str2charp(domain), rffi.str2charp(msg), - rffi.cast(rffi.INT, category)) + domain_c = rffi.str2charp(domain) + msg_c = rffi.str2charp(msg) + try: + result = _dcgettext(domain_c, msg_c, + rffi.cast(rffi.INT, category)) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(msg_c) return space.wrap(rffi.charp2str(result)) @@ -246,7 +288,11 @@ result = _textdomain(domain) else: domain = space.str_w(w_domain) - result = _textdomain(rffi.str2charp(domain)) + domain_c = rffi.str2charp(domain) + try: + result = _textdomain(domain_c) + finally: + rffi.free_charp(domain_c) return space.wrap(rffi.charp2str(result)) @@ -261,11 +307,20 @@ if space.is_w(w_dir, space.w_None): dir = None - dirname = _bindtextdomain(rffi.str2charp(domain), dir) + domain_c = rffi.str2charp(domain) + try: + dirname = _bindtextdomain(domain_c, dir) + finally: + rffi.free_charp(domain_c) else: dir = space.str_w(w_dir) - dirname = _bindtextdomain(rffi.str2charp(domain), - rffi.str2charp(dir)) + domain_c = rffi.str2charp(domain) + dir_c = rffi.str2charp(dir) + try: + dirname = _bindtextdomain(domain_c, dir_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(dir_c) if not dirname: errno = rposix.get_errno() @@ -284,12 +339,20 @@ if space.is_w(w_codeset, space.w_None): codeset = None - result = _bind_textdomain_codeset( - rffi.str2charp(domain), codeset) + domain_c = rffi.str2charp(domain) + try: + result = _bind_textdomain_codeset(domain_c, codeset) + finally: + rffi.free_charp(domain_c) else: codeset = space.str_w(w_codeset) - result = _bind_textdomain_codeset(rffi.str2charp(domain), - rffi.str2charp(codeset)) + domain_c = rffi.str2charp(domain) + codeset_c = rffi.str2charp(codeset) + try: + result = _bind_textdomain_codeset(domain_c, codeset_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(codeset_c) if not result: return space.w_None Modified: pypy/branch/reflex-support/pypy/rlib/rlocale.py ============================================================================== --- pypy/branch/reflex-support/pypy/rlib/rlocale.py (original) +++ pypy/branch/reflex-support/pypy/rlib/rlocale.py Tue Jul 6 18:26:59 2010 @@ -13,15 +13,15 @@ self.message = message HAVE_LANGINFO = sys.platform != 'win32' -#HAVE_LIBINTL = sys.platform != 'win32' +HAVE_LIBINTL = sys.platform != 'win32' class CConfig: includes = ['locale.h', 'limits.h'] if HAVE_LANGINFO: includes += ['langinfo.h'] - #if HAVE_LIBINTL: - # includes += ['libintl.h'] + if HAVE_LIBINTL: + includes += ['libintl.h'] if sys.platform == 'win32': includes += ['windows.h'] _compilation_info_ = ExternalCompilationInfo( Modified: pypy/branch/reflex-support/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/reflex-support/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/reflex-support/pypy/rpython/lltypesystem/rffi.py Tue Jul 6 18:26:59 2010 @@ -786,6 +786,8 @@ return r_wchar_t.BITS/8 if tp is lltype.Float: return 8 + if tp is lltype.SingleFloat: + return 4 assert isinstance(tp, lltype.Number) if tp is lltype.Signed: return ULONG._type.BITS/8 Modified: pypy/branch/reflex-support/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/reflex-support/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/reflex-support/pypy/rpython/lltypesystem/test/test_rffi.py Tue Jul 6 18:26:59 2010 @@ -707,6 +707,7 @@ lltype.UniChar: ctypes.c_wchar, lltype.Char: ctypes.c_ubyte, DOUBLE: ctypes.c_double, + FLOAT: ctypes.c_float, SIGNEDCHAR: ctypes.c_byte, UCHAR: ctypes.c_ubyte, SHORT: ctypes.c_short, Modified: pypy/branch/reflex-support/pypy/rpython/memory/lltypelayout.py ============================================================================== --- pypy/branch/reflex-support/pypy/rpython/memory/lltypelayout.py (original) +++ pypy/branch/reflex-support/pypy/rpython/memory/lltypelayout.py Tue Jul 6 18:26:59 2010 @@ -111,6 +111,8 @@ elif isinstance(offset, llmemory.ItemOffset): return sizeof(offset.TYPE) * offset.repeat elif isinstance(offset, llmemory.ArrayItemsOffset): + if offset.TYPE._hints.get('nolength', None): + return 0 return get_fixed_size(lltype.Signed) elif isinstance(offset, llmemory.GCHeaderOffset): return sizeof(offset.gcheaderbuilder.HDR) From wlav at codespeak.net Tue Jul 6 18:45:12 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Tue, 6 Jul 2010 18:45:12 +0200 (CEST) Subject: [pypy-svn] r75932 - in pypy/branch/reflex-support/pypy/module/cppyy: . src test Message-ID: <20100706164512.AC8D2282C05@codespeak.net> Author: wlav Date: Tue Jul 6 18:45:10 2010 New Revision: 75932 Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Log: (wlav, cfbolz) Add const char* argument passing and refactor converter lookup. Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Tue Jul 6 18:45:10 2010 @@ -1,9 +1,14 @@ from pypy.rpython.lltypesystem import rffi, lltype +_converters = {} + class TypeConverter(object): def convert_argument(self, space, w_obj): raise NotImplementedError("abstract base class") + def free_argument(self, arg): + lltype.free(arg, flavor='raw') + class IntConverter(TypeConverter): def convert_argument(self, space, w_obj): @@ -19,9 +24,20 @@ x[0] = arg return rffi.cast(rffi.VOIDP, x) +class CStringConverter(TypeConverter): + def convert_argument(self, space, w_obj): + arg = space.str_w(w_obj) + x = rffi.str2charp(arg) + return rffi.cast(rffi.VOIDP, x) + def get_converter(name): - if name == "int": - return IntConverter() - if name == "double": - return DoubleConverter() + try: + return _converters[name] + except KeyError: + pass + raise TypeError("no clue what %s is" % name) + +_converters["int"] = IntConverter() +_converters["double"] = DoubleConverter() +_converters["const char*"] = CStringConverter() Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Tue Jul 6 18:45:10 2010 @@ -145,14 +145,16 @@ except: # fun :-( for j in range(i): - lltype.free(args[j], flavor='raw') + conv = self.arg_converters[j] + conv.free_argument(args[j]) lltype.free(args, flavor='raw') raise return args def free_arguments(self, args): for i in range(len(self.arg_types)): - lltype.free(args[i], flavor='raw') + conv = self.arg_converters[i] + conv.free_argument(args[i]) lltype.free(args, flavor='raw') def __repr__(self): Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Tue Jul 6 18:45:10 2010 @@ -52,10 +52,10 @@ int num_methods(const char* class_name) { Reflex::Type t = Reflex::Type::ByName(class_name); - for (int i = 0; i < t.FunctionMemberSize(); i++) { + for (int i = 0; i < (int)t.FunctionMemberSize(); i++) { Reflex::Member m = t.FunctionMemberAt(i); std::cout << i << " " << m.Name() << std::endl; - for (int j = 0; j < m.FunctionParameterSize(); j++) { + for (int j = 0; j < (int)m.FunctionParameterSize(); j++) { Reflex::Type at = m.TypeOf().FunctionParameterAt(j); std::cout << " " << j << " " << at.Name() << std::endl; } @@ -76,7 +76,7 @@ Reflex::Type t = Reflex::Type::ByName(class_name); Reflex::Member m = t.FunctionMemberAt(method_index); Reflex::Type rt = m.TypeOf().ReturnType(); - std::string name = rt.Name(Reflex::FINAL); + std::string name = rt.Name(Reflex::FINAL|Reflex::SCOPED|Reflex::QUALIFIED); char* name_char = (char*)malloc(name.size() + 1); strcpy(name_char, name.c_str()); return name_char; @@ -93,7 +93,7 @@ Reflex::Type t = Reflex::Type::ByName(class_name); Reflex::Member m = t.FunctionMemberAt(method_index); Reflex::Type at = m.TypeOf().FunctionParameterAt(arg_index); - std::string name = at.Name(Reflex::FINAL); + std::string name = at.Name(Reflex::FINAL|Reflex::SCOPED|Reflex::QUALIFIED); char* name_char = (char*)malloc(name.size() + 1); strcpy(name_char, name.c_str()); return name_char; Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx Tue Jul 6 18:45:10 2010 @@ -1,4 +1,5 @@ #include +#include class example01 { public: @@ -35,6 +36,9 @@ static double adddouble(double a) { return a + 0.01; } + static int atoi(const char* str) { + return ::atoi(str); + } static int getcount() { std::cout << "getcount called" << std::endl; return count; Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Tue Jul 6 18:45:10 2010 @@ -44,6 +44,11 @@ res = t.invoke("adddouble", 0.09) assert res == 0.09 + 0.01 + def test_example01static_constcharp(self): + t = self.example01.type_byname("example01") + res = t.invoke("atoi", "1") + assert res == 1 + def test_example01method(self): t = self.example01.type_byname("example01") count = t.invoke("getcount") @@ -57,3 +62,4 @@ count = t.invoke("getcount") assert count == 0 + From cfbolz at codespeak.net Tue Jul 6 19:03:24 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Jul 2010 19:03:24 +0200 (CEST) Subject: [pypy-svn] r75934 - in pypy/branch/reflex-support/pypy/jit/backend/llgraph: . test Message-ID: <20100706170324.CAFC5282C05@codespeak.net> Author: cfbolz Date: Tue Jul 6 19:03:23 2010 New Revision: 75934 Modified: pypy/branch/reflex-support/pypy/jit/backend/llgraph/llimpl.py pypy/branch/reflex-support/pypy/jit/backend/llgraph/test/test_llgraph.py Log: (cfbolz, arigo): slightly obscure: need an unsafe cast in setarrayitem_raw_int Modified: pypy/branch/reflex-support/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/reflex-support/pypy/jit/backend/llgraph/llimpl.py Tue Jul 6 19:03:23 2010 @@ -10,7 +10,7 @@ BoxInt, BoxPtr, BoxObj, BoxFloat, REF, INT, FLOAT) from pypy.jit.codewriter import heaptracker -from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr +from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi from pypy.rpython.ootypesystem import ootype from pypy.rpython.module.support import LLSupport, OOSupport from pypy.rpython.llinterp import LLException @@ -972,10 +972,12 @@ return heaptracker.adr2int(x) return lltype.cast_primitive(lltype.Signed, x) -def cast_from_int(TYPE, x): +def cast_from_int(TYPE, x, unsafe=False): if isinstance(TYPE, lltype.Ptr): if isinstance(x, (int, long, llmemory.AddressAsInt)): x = llmemory.cast_int_to_adr(x) + if unsafe: + return rffi.cast(TYPE, x) return llmemory.cast_adr_to_ptr(x, TYPE) elif TYPE == llmemory.Address: if isinstance(x, (int, long, llmemory.AddressAsInt)): @@ -1262,7 +1264,7 @@ def do_setarrayitem_raw_int(array, index, newvalue): array = array.adr.ptr ITEMTYPE = lltype.typeOf(array).TO.OF - newvalue = cast_from_int(ITEMTYPE, newvalue) + newvalue = cast_from_int(ITEMTYPE, newvalue, unsafe=True) array._obj.setitem(index, newvalue) def do_setarrayitem_gc_float(array, index, newvalue): Modified: pypy/branch/reflex-support/pypy/jit/backend/llgraph/test/test_llgraph.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/backend/llgraph/test/test_llgraph.py (original) +++ pypy/branch/reflex-support/pypy/jit/backend/llgraph/test/test_llgraph.py Tue Jul 6 19:03:23 2010 @@ -32,6 +32,19 @@ assert heaptracker.adr2int(llmemory.NULL) == 0 assert heaptracker.int2adr(0) == llmemory.NULL +def test_force_cast_ptr_to_other_ptr(): + from pypy.jit.backend.llgraph import llimpl + from pypy.rpython.lltypesystem import rffi, llmemory + x = lltype.malloc(rffi.DOUBLEP.TO, 1, flavor='raw') + x[0] = 0.1 + args = lltype.malloc(rffi.CArray(rffi.VOIDP), 1, flavor='raw', zero=True) + assert not args[0] + arrayasint = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(args), mode="symbolic") + ptrasint = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(x), mode="symbolic") + llimpl.do_setarrayitem_raw_int(arrayasint, 0, ptrasint) + assert args[0] + + ## these tests never worked ## class TestOOTypeLLGraph(LLGraphTest): ## from pypy.jit.backend.llgraph.runner import OOtypeCPU as cpu_type From wlav at codespeak.net Tue Jul 6 19:15:44 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Tue, 6 Jul 2010 19:15:44 +0200 (CEST) Subject: [pypy-svn] r75935 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100706171544.34323282C05@codespeak.net> Author: wlav Date: Tue Jul 6 19:15:42 2010 New Revision: 75935 Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Log: (wlav) Add handling for char* return type on static functions. Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Tue Jul 6 19:15:42 2010 @@ -172,6 +172,11 @@ if self.result_type == "double": result = c_callstatic_d(self.cpptype.name, self.method_index, len(args_w), args) return self.space.wrap(result) + if self.result_type == "char*": + lresult = c_callstatic_l(self.cpptype.name, self.method_index, len(args_w), args) + ccpresult = rffi.cast(rffi.CCHARP, lresult) + result = charp2str_free(ccpresult) + return self.space.wrap(result) else: raise NotImplementedError finally: @@ -213,7 +218,7 @@ def __repr__(self): return "CPPOverload(%s, %s)" % (self.func_name, self.functions) -def charp2str(charp): +def charp2str_free(charp): string = rffi.charp2str(charp) c_myfree(charp) return string @@ -232,7 +237,7 @@ num_func_members = c_num_methods(self.name) args_temp = {} for i in range(num_func_members): - func_member_name = charp2str(c_method_name(self.name, i)) + func_member_name = charp2str_free(c_method_name(self.name, i)) cppfunction = self._make_cppfunction(i) overload = args_temp.setdefault(func_member_name, []) overload.append(cppfunction) @@ -241,11 +246,11 @@ self.function_members[name] = overload def _make_cppfunction(self, method_index): - result_type = charp2str(c_result_type_method(self.name, method_index)) + result_type = charp2str_free(c_result_type_method(self.name, method_index)) num_args = c_num_args_method(self.name, method_index) argtypes = [] for i in range(num_args): - argtype = charp2str(c_arg_type_method(self.name, method_index, i)) + argtype = charp2str_free(c_arg_type_method(self.name, method_index, i)) argtypes.append(argtype) if c_is_constructor(self.name, method_index): cls = CPPConstructor Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx Tue Jul 6 19:15:42 2010 @@ -1,5 +1,6 @@ #include #include +#include class example01 { public: @@ -39,6 +40,12 @@ static int atoi(const char* str) { return ::atoi(str); } + static char* strcpy(const char* strin) { + char* strout = (char*)malloc(::strlen(strin + 1)); + ::strcpy(strout, strin); + return strout; + } + static int getcount() { std::cout << "getcount called" << std::endl; return count; Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Tue Jul 6 19:15:42 2010 @@ -49,6 +49,9 @@ res = t.invoke("atoi", "1") assert res == 1 + res = t.invoke("strcpy", "aap") + assert res == "aap" + def test_example01method(self): t = self.example01.type_byname("example01") count = t.invoke("getcount") From getxsick at codespeak.net Tue Jul 6 19:23:11 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 6 Jul 2010 19:23:11 +0200 (CEST) Subject: [pypy-svn] r75937 - pypy/trunk/pypy/jit/backend/llsupport Message-ID: <20100706172311.60A59282C05@codespeak.net> Author: getxsick Date: Tue Jul 6 19:23:09 2010 New Revision: 75937 Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py Log: hack to avoid globals() as it's not RPython Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/gc.py Tue Jul 6 19:23:09 2010 @@ -1,3 +1,4 @@ +import sys from pypy.rlib import rgc from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import fatalerror @@ -618,7 +619,7 @@ else: name = "boehm" try: - cls = globals()['GcLLDescr_' + name] + cls = getattr(sys.modules[__name__], 'GcLLDescr_' + name) except KeyError: raise NotImplementedError("GC transformer %r not supported by " "the JIT backend" % (name,)) From arigo at codespeak.net Tue Jul 6 19:27:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 19:27:37 +0200 (CEST) Subject: [pypy-svn] r75938 - in pypy/trunk/pypy/jit/codewriter: . test Message-ID: <20100706172737.87C9E282C05@codespeak.net> Author: arigo Date: Tue Jul 6 19:27:36 2010 New Revision: 75938 Modified: pypy/trunk/pypy/jit/codewriter/jtransform.py pypy/trunk/pypy/jit/codewriter/test/test_flatten.py Log: Merge r75925 from branch/reflex-support: start some support for very simple force_cast cases Modified: pypy/trunk/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/jtransform.py (original) +++ pypy/trunk/pypy/jit/codewriter/jtransform.py Tue Jul 6 19:27:36 2010 @@ -667,7 +667,7 @@ return self._rewrite_symmetric(op) def _is_gc(self, v): - return v.concretetype.TO._gckind == 'gc' + return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc' def _rewrite_cmp_ptrs(self, op): if self._is_gc(op.args[0]): @@ -701,6 +701,17 @@ #return op raise NotImplementedError("cast_ptr_to_int") + def rewrite_op_force_cast(self, op): + from pypy.rpython.lltypesystem.rffi import size_and_sign + from pypy.rlib.rarithmetic import intmask + assert not self._is_gc(op.args[0]) + size1, unsigned1 = size_and_sign(op.args[0].concretetype) + size2, unsigned2 = size_and_sign(op.result.concretetype) + if size1 == size2 and unsigned1 == unsigned2: + return + raise NotImplementedError("cast not supported yet: %s" % (op, )) + + # ---------- # Renames, from the _old opname to the _new one. # The new operation is optionally further processed by rewrite_operation(). Modified: pypy/trunk/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/trunk/pypy/jit/codewriter/test/test_flatten.py Tue Jul 6 19:27:36 2010 @@ -729,3 +729,32 @@ int_between %i0, %i1, %i2 -> %i3 int_return %i3 """, transform=True) + + def test_force_cast(self): + py.test.skip("later") + from pypy.rpython.lltypesystem import rffi + def f(n): + c = chr(n) + return rffi.cast(rffi.INT, c) + self.encoding_test(f, [42], """ + int_return %i0 + """, transform=True) + def g(n): + return rffi.cast(rffi.UCHAR, n) + self.encoding_test(g, [42], """ + int_and %i0, $255 -> %i1 + int_return %i1 + """, transform=True) + def h(n): + return rffi.cast(rffi.SCHAR, n) + self.encoding_test(h, [42], """ + ... + """, transform=True) + + def test_force_cast_pointer(self): + from pypy.rpython.lltypesystem import rffi + def h(p): + return rffi.cast(rffi.VOIDP, p) + self.encoding_test(h, [lltype.nullptr(rffi.CCHARP.TO)], """ + int_return %i0 + """, transform=True) From getxsick at codespeak.net Tue Jul 6 19:36:38 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 6 Jul 2010 19:36:38 +0200 (CEST) Subject: [pypy-svn] r75939 - in pypy/branch/fast-ctypes/pypy/rlib: . test Message-ID: <20100706173638.4E281282C05@codespeak.net> Author: getxsick Date: Tue Jul 6 19:36:36 2010 New Revision: 75939 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: merge from trunk Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Tue Jul 6 19:36:36 2010 @@ -54,10 +54,10 @@ raise ValueError(self.res_type) try: - addr = rffi.cast(lltype.Signed, rdynload.dlsym(self.lib, func)) + self.funcaddr = rffi.cast(lltype.Signed, rdynload.dlsym(self.lib, func)) except KeyError: raise ValueError("Cannot find symbol %s", func) - self.bfuncaddr = BoxInt(addr) + self.bfuncaddr = BoxInt(self.funcaddr) args = [] for arg in self.args_type: @@ -74,19 +74,25 @@ FUNC = deref(FPTR) self.calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) - def call(self): - inputargs = [self.bfuncaddr] + self.bargs + self.looptoken = LoopToken() + self.inputargs = [ BoxInt(), BoxInt(), BoxInt() ] + + self.oplist = [ResOperation(rop.CALL, self.inputargs, self.bres, + descr=self.calldescr), + ResOperation(rop.FINISH, [self.bres], None, + descr=BasicFailDescr(0))] + self.cpu.compile_loop(self.inputargs, self.oplist, self.looptoken) - oplist = [ResOperation(rop.CALL, inputargs, self.bres, - descr=self.calldescr), - ResOperation(rop.FINISH, [self.bres], None, - descr=BasicFailDescr(0))] - looptoken = LoopToken() - self.cpu.compile_loop(inputargs, oplist, looptoken) - self.cpu.set_future_value_int(0, self.bfuncaddr.getint()) + def call(self): + self.inputargs[0].value = self.funcaddr + self.cpu.set_future_value_int(0, self.funcaddr) + self.inputargs[1].value = 1 + self.cpu.set_future_value_int(1, 1) + self.inputargs[2].value = 2 + self.cpu.set_future_value_int(2, 2) - res = self.cpu.execute_token(looptoken) - if res is oplist[-1].descr: + res = self.cpu.execute_token(self.looptoken) + if res is self.oplist[-1].descr: self.guard_failed = False else: self.guard_failed = True @@ -107,7 +113,12 @@ def setup_stack(self): self.bargs = [] - self.esp = 1 # 0 is a func addr + self.esp = 0 + + def push_funcaddr(self, value): + self.cpu.set_future_value_int(self.esp, value) + self.bargs.append(BoxInt(value)) # insert(0, )? + self.esp += 1 def push_int(self, value): self.cpu.set_future_value_int(self.esp, value) Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Tue Jul 6 19:36:36 2010 @@ -69,30 +69,30 @@ lib = rjitffi.CDLL(self.lib_name) func = lib.get('add_integers', ['i', 'i'], 'i') - func.push_int(1) - func.push_int(2) + #func.push_int(1) + #func.push_int(2) assert func.call() == 3 - func = lib.get('add_integers', ['i', 'i'], 'i') - func.push_int(-1) - func.push_int(2) - assert func.call() == 1 - - func = lib.get('add_integers', ['i', 'i'], 'i') - func.push_int(0) - func.push_int(0) - assert func.call() == 0 - - func = lib.get('max3', ['i', 'i', 'i'], 'i') - func.push_int(2) - func.push_int(8) - func.push_int(3) - assert func.call() == 8 - - func = lib.get('add_floats', ['f', 'f'], 'f') - func.push_float(1.2) - func.push_float(1.5) - assert func.call() == 2.7 + #func = lib.get('add_integers', ['i', 'i'], 'i') + #func.push_int(-1) + #func.push_int(2) + #assert func.call() == 1 + + #func = lib.get('add_integers', ['i', 'i'], 'i') + #func.push_int(0) + #func.push_int(0) + #assert func.call() == 0 + + #func = lib.get('max3', ['i', 'i', 'i'], 'i') + #func.push_int(2) + #func.push_int(8) + #func.push_int(3) + #assert func.call() == 8 + + #func = lib.get('add_floats', ['f', 'f'], 'f') + #func.push_float(1.2) + #func.push_float(1.5) + #assert func.call() == 2.7 def test_get_void(self): lib = rjitffi.CDLL(self.lib_name) From cfbolz at codespeak.net Tue Jul 6 19:49:45 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Jul 2010 19:49:45 +0200 (CEST) Subject: [pypy-svn] r75940 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100706174945.46D8C282C09@codespeak.net> Author: cfbolz Date: Tue Jul 6 19:49:36 2010 New Revision: 75940 Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Log: (cfbolz, arigo, antocuni): refactor exception handling to help the JIT for completely unobvious reasons :-( Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Tue Jul 6 19:49:36 2010 @@ -136,19 +136,20 @@ if self.arg_converters is None: self._build_converters() args = lltype.malloc(rffi.CArray(rffi.VOIDP), len(args_w), flavor='raw') - try: - i = 0 # appease RPython: i is used below - for i in range(len(args_w)): - argtype = self.arg_types[i] - conv = self.arg_converters[i] - args[i] = conv.convert_argument(space, args_w[i]) - except: - # fun :-( - for j in range(i): - conv = self.arg_converters[j] - conv.free_argument(args[j]) - lltype.free(args, flavor='raw') - raise + for i in range(len(args_w)): + argtype = self.arg_types[i] + conv = self.arg_converters[i] + w_arg = args_w[i] + try: + arg = conv.convert_argument(space, w_arg) + except: + # fun :-( + for j in range(i): + conv = self.arg_converters[j] + conv.free_argument(args[j]) + lltype.free(args, flavor='raw') + raise + args[i] = arg return args def free_arguments(self, args): From arigo at codespeak.net Tue Jul 6 19:53:48 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Jul 2010 19:53:48 +0200 (CEST) Subject: [pypy-svn] r75941 - pypy/trunk/pypy/jit/backend/llsupport Message-ID: <20100706175348.0C26E282C09@codespeak.net> Author: arigo Date: Tue Jul 6 19:53:46 2010 New Revision: 75941 Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py Log: Revert r75937. That's nonsense :-( Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/gc.py Tue Jul 6 19:53:46 2010 @@ -1,4 +1,3 @@ -import sys from pypy.rlib import rgc from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import fatalerror @@ -619,7 +618,7 @@ else: name = "boehm" try: - cls = getattr(sys.modules[__name__], 'GcLLDescr_' + name) + cls = globals()['GcLLDescr_' + name] except KeyError: raise NotImplementedError("GC transformer %r not supported by " "the JIT backend" % (name,)) From getxsick at codespeak.net Tue Jul 6 19:59:03 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 6 Jul 2010 19:59:03 +0200 (CEST) Subject: [pypy-svn] r75942 - in pypy/branch/fast-ctypes/pypy/rlib: . test Message-ID: <20100706175903.62348282C09@codespeak.net> Author: getxsick Date: Tue Jul 6 19:59:01 2010 New Revision: 75942 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: roll back r75939 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Tue Jul 6 19:59:01 2010 @@ -54,10 +54,10 @@ raise ValueError(self.res_type) try: - self.funcaddr = rffi.cast(lltype.Signed, rdynload.dlsym(self.lib, func)) + addr = rffi.cast(lltype.Signed, rdynload.dlsym(self.lib, func)) except KeyError: raise ValueError("Cannot find symbol %s", func) - self.bfuncaddr = BoxInt(self.funcaddr) + self.bfuncaddr = BoxInt(addr) args = [] for arg in self.args_type: @@ -74,25 +74,19 @@ FUNC = deref(FPTR) self.calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) - self.looptoken = LoopToken() - self.inputargs = [ BoxInt(), BoxInt(), BoxInt() ] - - self.oplist = [ResOperation(rop.CALL, self.inputargs, self.bres, - descr=self.calldescr), - ResOperation(rop.FINISH, [self.bres], None, - descr=BasicFailDescr(0))] - self.cpu.compile_loop(self.inputargs, self.oplist, self.looptoken) - def call(self): - self.inputargs[0].value = self.funcaddr - self.cpu.set_future_value_int(0, self.funcaddr) - self.inputargs[1].value = 1 - self.cpu.set_future_value_int(1, 1) - self.inputargs[2].value = 2 - self.cpu.set_future_value_int(2, 2) + inputargs = [self.bfuncaddr] + self.bargs + + oplist = [ResOperation(rop.CALL, inputargs, self.bres, + descr=self.calldescr), + ResOperation(rop.FINISH, [self.bres], None, + descr=BasicFailDescr(0))] + looptoken = LoopToken() + self.cpu.compile_loop(inputargs, oplist, looptoken) + self.cpu.set_future_value_int(0, self.bfuncaddr.getint()) - res = self.cpu.execute_token(self.looptoken) - if res is self.oplist[-1].descr: + res = self.cpu.execute_token(looptoken) + if res is oplist[-1].descr: self.guard_failed = False else: self.guard_failed = True @@ -113,12 +107,7 @@ def setup_stack(self): self.bargs = [] - self.esp = 0 - - def push_funcaddr(self, value): - self.cpu.set_future_value_int(self.esp, value) - self.bargs.append(BoxInt(value)) # insert(0, )? - self.esp += 1 + self.esp = 1 # 0 is a func addr def push_int(self, value): self.cpu.set_future_value_int(self.esp, value) Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Tue Jul 6 19:59:01 2010 @@ -69,30 +69,30 @@ lib = rjitffi.CDLL(self.lib_name) func = lib.get('add_integers', ['i', 'i'], 'i') - #func.push_int(1) - #func.push_int(2) + func.push_int(1) + func.push_int(2) assert func.call() == 3 - #func = lib.get('add_integers', ['i', 'i'], 'i') - #func.push_int(-1) - #func.push_int(2) - #assert func.call() == 1 - - #func = lib.get('add_integers', ['i', 'i'], 'i') - #func.push_int(0) - #func.push_int(0) - #assert func.call() == 0 - - #func = lib.get('max3', ['i', 'i', 'i'], 'i') - #func.push_int(2) - #func.push_int(8) - #func.push_int(3) - #assert func.call() == 8 - - #func = lib.get('add_floats', ['f', 'f'], 'f') - #func.push_float(1.2) - #func.push_float(1.5) - #assert func.call() == 2.7 + func = lib.get('add_integers', ['i', 'i'], 'i') + func.push_int(-1) + func.push_int(2) + assert func.call() == 1 + + func = lib.get('add_integers', ['i', 'i'], 'i') + func.push_int(0) + func.push_int(0) + assert func.call() == 0 + + func = lib.get('max3', ['i', 'i', 'i'], 'i') + func.push_int(2) + func.push_int(8) + func.push_int(3) + assert func.call() == 8 + + func = lib.get('add_floats', ['f', 'f'], 'f') + func.push_float(1.2) + func.push_float(1.5) + assert func.call() == 2.7 def test_get_void(self): lib = rjitffi.CDLL(self.lib_name) From hakanardo at codespeak.net Tue Jul 6 21:16:36 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 6 Jul 2010 21:16:36 +0200 (CEST) Subject: [pypy-svn] r75943 - in pypy/branch/interplevel-array: . lib-python lib_pypy lib_pypy/pypy_test pypy/annotation pypy/annotation/test pypy/interpreter pypy/jit/backend/llgraph pypy/jit/backend/llsupport pypy/jit/backend/llsupport/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/test pypy/module/_locale pypy/module/array pypy/module/cpyext pypy/module/marshal pypy/module/termios pypy/module/test_lib_pypy/ctypes_tests pypy/objspace/std pypy/rlib pypy/rpython pypy/rpython/lltypesystem pypy/rpython/lltypesystem/test pypy/rpython/memory pypy/rpython/module pypy/rpython/test Message-ID: <20100706191636.63EA0282C09@codespeak.net> Author: hakanardo Date: Tue Jul 6 21:16:31 2010 New Revision: 75943 Modified: pypy/branch/interplevel-array/ (props changed) pypy/branch/interplevel-array/lib-python/ (props changed) pypy/branch/interplevel-array/lib_pypy/ (props changed) pypy/branch/interplevel-array/lib_pypy/dbm.py (props changed) pypy/branch/interplevel-array/lib_pypy/pypy_test/test_functools.py (props changed) pypy/branch/interplevel-array/pypy/annotation/bookkeeper.py pypy/branch/interplevel-array/pypy/annotation/model.py pypy/branch/interplevel-array/pypy/annotation/test/test_model.py pypy/branch/interplevel-array/pypy/annotation/unaryop.py pypy/branch/interplevel-array/pypy/interpreter/pyopcode.py pypy/branch/interplevel-array/pypy/jit/backend/llgraph/llimpl.py pypy/branch/interplevel-array/pypy/jit/backend/llgraph/runner.py pypy/branch/interplevel-array/pypy/jit/backend/llsupport/descr.py pypy/branch/interplevel-array/pypy/jit/backend/llsupport/llmodel.py pypy/branch/interplevel-array/pypy/jit/backend/llsupport/symbolic.py pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_symbolic.py pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py pypy/branch/interplevel-array/pypy/jit/backend/x86/regalloc.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/interplevel-array/pypy/jit/codewriter/jtransform.py pypy/branch/interplevel-array/pypy/jit/codewriter/support.py pypy/branch/interplevel-array/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/interplevel-array/pypy/jit/codewriter/test/test_flatten.py pypy/branch/interplevel-array/pypy/jit/metainterp/blackhole.py pypy/branch/interplevel-array/pypy/jit/metainterp/executor.py pypy/branch/interplevel-array/pypy/jit/metainterp/pyjitpl.py pypy/branch/interplevel-array/pypy/jit/metainterp/resoperation.py pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_basic.py pypy/branch/interplevel-array/pypy/module/_locale/interp_locale.py pypy/branch/interplevel-array/pypy/module/array/interp_array_try1.py (props changed) pypy/branch/interplevel-array/pypy/module/cpyext/api.py pypy/branch/interplevel-array/pypy/module/marshal/interp_marshal.py pypy/branch/interplevel-array/pypy/module/termios/interp_termios.py pypy/branch/interplevel-array/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) pypy/branch/interplevel-array/pypy/objspace/std/stringobject.py pypy/branch/interplevel-array/pypy/rlib/libffi.py pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py pypy/branch/interplevel-array/pypy/rlib/runicode.py pypy/branch/interplevel-array/pypy/rpython/lltypesystem/rffi.py pypy/branch/interplevel-array/pypy/rpython/lltypesystem/test/test_rffi.py pypy/branch/interplevel-array/pypy/rpython/memory/lltypelayout.py pypy/branch/interplevel-array/pypy/rpython/module/ll_termios.py pypy/branch/interplevel-array/pypy/rpython/rstr.py pypy/branch/interplevel-array/pypy/rpython/test/test_rstr.py Log: Merge: svn merge -r75756:75941 svn+ssh://hakanardo at codespeak.net/svn/pypy/trunk Modified: pypy/branch/interplevel-array/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/interplevel-array/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/interplevel-array/pypy/annotation/bookkeeper.py Tue Jul 6 21:16:31 2010 @@ -739,7 +739,7 @@ """ w_tuple = SomeTuple def newtuple(self, items_s): - if items_s == [Ellipsis]: + if len(items_s) == 1 and items_s[0] is Ellipsis: res = SomeObject() # hack to get a SomeObject as the *arg res.from_ellipsis = True return res Modified: pypy/branch/interplevel-array/pypy/annotation/model.py ============================================================================== --- pypy/branch/interplevel-array/pypy/annotation/model.py (original) +++ pypy/branch/interplevel-array/pypy/annotation/model.py Tue Jul 6 21:16:31 2010 @@ -34,7 +34,7 @@ from pypy.tool.pairtype import pair, extendabletype from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint, r_ulonglong, base_int -from pypy.rlib.rarithmetic import r_singlefloat +from pypy.rlib.rarithmetic import r_singlefloat, isnan import inspect, weakref DEBUG = False # set to False to disable recording of debugging information @@ -162,6 +162,13 @@ # pretend it's a float. immutable = True + def __eq__(self, other): + # NaN unpleasantness. + if (self.is_constant() and other.is_constant() and + isnan(self.const) and isnan(other.const)): + return True + return super(SomeFloat, self).__eq__(other) + def can_be_none(self): return False @@ -229,11 +236,15 @@ class SomeChar(SomeString): "Stands for an object known to be a string of length 1." + can_be_None = False + def __init__(self): # no 'can_be_None' argument here + pass class SomeUnicodeCodePoint(SomeUnicodeString): "Stands for an object known to be a unicode codepoint." - def can_be_none(self): - return False + can_be_None = False + def __init__(self): # no 'can_be_None' argument here + pass SomeString.basestringclass = SomeString SomeString.basecharclass = SomeChar Modified: pypy/branch/interplevel-array/pypy/annotation/test/test_model.py ============================================================================== --- pypy/branch/interplevel-array/pypy/annotation/test/test_model.py (original) +++ pypy/branch/interplevel-array/pypy/annotation/test/test_model.py Tue Jul 6 21:16:31 2010 @@ -200,6 +200,15 @@ s2.const=cls assert s1.contains(s2) +def test_nan(): + f1 = SomeFloat() + f1.const = float("nan") + f2 = SomeFloat() + f2.const = float("nan") + assert f1.contains(f1) + assert f2.contains(f1) + assert f1.contains(f2) + if __name__ == '__main__': for name, value in globals().items(): if name.startswith('test_'): Modified: pypy/branch/interplevel-array/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/interplevel-array/pypy/annotation/unaryop.py (original) +++ pypy/branch/interplevel-array/pypy/annotation/unaryop.py Tue Jul 6 21:16:31 2010 @@ -5,7 +5,7 @@ from types import MethodType from pypy.annotation.model import \ SomeObject, SomeInteger, SomeBool, SomeString, SomeChar, SomeList, \ - SomeDict, SomeTuple, SomeImpossibleValue, \ + SomeDict, SomeTuple, SomeImpossibleValue, SomeUnicodeCodePoint, \ SomeInstance, SomeBuiltin, SomeFloat, SomeIterator, SomePBC, \ SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \ s_ImpossibleValue, s_Bool, s_None, \ @@ -494,9 +494,6 @@ def getanyitem(str): return str.basecharclass() - def ord(str): - return SomeInteger(nonneg=True) - def method_split(str, patt): # XXX getbookkeeper().count("str_split", str, patt) return getbookkeeper().newlist(str.basestringclass()) @@ -541,11 +538,16 @@ return SomeUnicodeString() method_decode.can_only_throw = [UnicodeDecodeError] -class __extend__(SomeChar): +class __extend__(SomeChar, SomeUnicodeCodePoint): def len(chr): return immutablevalue(1) + def ord(str): + return SomeInteger(nonneg=True) + +class __extend__(SomeChar): + def method_isspace(chr): return s_Bool Modified: pypy/branch/interplevel-array/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/pyopcode.py Tue Jul 6 21:16:31 2010 @@ -14,6 +14,7 @@ from pypy.rlib import jit, rstackovf, rstack from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.debug import check_nonneg from pypy.tool.stdlib_opcode import (bytecode_spec, host_bytecode_spec, unrolling_all_opcode_descs, opmap, host_opmap) @@ -186,7 +187,7 @@ lo = ord(co_code[next_instr]) hi = ord(co_code[next_instr+1]) next_instr += 2 - oparg = (hi << 8) | lo + oparg = (hi * 256) | lo else: oparg = 0 @@ -197,7 +198,7 @@ lo = ord(co_code[next_instr+1]) hi = ord(co_code[next_instr+2]) next_instr += 3 - oparg = (oparg << 16) | (hi << 8) | lo + oparg = (oparg * 65536) | (hi * 256) | lo if opcode == self.opcodedesc.RETURN_VALUE.index: w_returnvalue = self.popvalue() @@ -323,7 +324,9 @@ ## def NOP(self, oparg, next_instr): - pass + # annotation-time check: if it fails, it means that the decoding + # of oparg failed to produce an integer which is annotated as non-neg + check_nonneg(oparg) def LOAD_FAST(self, varindex, next_instr): # access a local variable directly Modified: pypy/branch/interplevel-array/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llgraph/llimpl.py Tue Jul 6 21:16:31 2010 @@ -1199,10 +1199,18 @@ array = array._obj.container return cast_to_int(array.getitem(index)) +def do_getarrayitem_raw_int(array, index): + array = array.adr.ptr._obj + return cast_to_int(array.getitem(index)) + def do_getarrayitem_gc_float(array, index): array = array._obj.container return cast_to_float(array.getitem(index)) +def do_getarrayitem_raw_float(array, index): + array = array.adr.ptr._obj + return cast_to_float(array.getitem(index)) + def do_getarrayitem_gc_ptr(array, index): array = array._obj.container return cast_to_ptr(array.getitem(index)) @@ -1251,12 +1259,24 @@ newvalue = cast_from_int(ITEMTYPE, newvalue) array.setitem(index, newvalue) +def do_setarrayitem_raw_int(array, index, newvalue): + array = array.adr.ptr + ITEMTYPE = lltype.typeOf(array).TO.OF + newvalue = cast_from_int(ITEMTYPE, newvalue) + array._obj.setitem(index, newvalue) + def do_setarrayitem_gc_float(array, index, newvalue): array = array._obj.container ITEMTYPE = lltype.typeOf(array).OF newvalue = cast_from_float(ITEMTYPE, newvalue) array.setitem(index, newvalue) +def do_setarrayitem_raw_float(array, index, newvalue): + array = array.adr.ptr + ITEMTYPE = lltype.typeOf(array).TO.OF + newvalue = cast_from_int(ITEMTYPE, newvalue) + array._obj.setitem(index, newvalue) + def do_setarrayitem_gc_ptr(array, index, newvalue): array = array._obj.container ITEMTYPE = lltype.typeOf(array).OF Modified: pypy/branch/interplevel-array/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llgraph/runner.py Tue Jul 6 21:16:31 2010 @@ -294,7 +294,6 @@ return llimpl.grab_exc_value() def arraydescrof(self, A): - assert isinstance(A, lltype.GcArray) assert A.OF != lltype.Void size = symbolic.get_size(A) token = history.getkind(A.OF) @@ -317,12 +316,18 @@ def bh_getarrayitem_gc_i(self, arraydescr, array, index): assert isinstance(arraydescr, Descr) return llimpl.do_getarrayitem_gc_int(array, index) + def bh_getarrayitem_raw_i(self, arraydescr, array, index): + assert isinstance(arraydescr, Descr) + return llimpl.do_getarrayitem_raw_int(array, index) def bh_getarrayitem_gc_r(self, arraydescr, array, index): assert isinstance(arraydescr, Descr) return llimpl.do_getarrayitem_gc_ptr(array, index) def bh_getarrayitem_gc_f(self, arraydescr, array, index): assert isinstance(arraydescr, Descr) return llimpl.do_getarrayitem_gc_float(array, index) + def bh_getarrayitem_raw_f(self, arraydescr, array, index): + assert isinstance(arraydescr, Descr) + return llimpl.do_getarrayitem_raw_float(array, index) def bh_getfield_gc_i(self, struct, fielddescr): assert isinstance(fielddescr, Descr) @@ -372,6 +377,10 @@ assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_int(array, index, newvalue) + def bh_setarrayitem_raw_i(self, arraydescr, array, index, newvalue): + assert isinstance(arraydescr, Descr) + llimpl.do_setarrayitem_raw_int(array, index, newvalue) + def bh_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_ptr(array, index, newvalue) @@ -380,6 +389,10 @@ assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_float(array, index, newvalue) + def bh_setarrayitem_raw_f(self, arraydescr, array, index, newvalue): + assert isinstance(arraydescr, Descr) + llimpl.do_setarrayitem_raw_float(array, index, newvalue) + def bh_setfield_gc_i(self, struct, fielddescr, newvalue): assert isinstance(fielddescr, Descr) llimpl.do_setfield_gc_int(struct, fielddescr.ofs, newvalue) Modified: pypy/branch/interplevel-array/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llsupport/descr.py Tue Jul 6 21:16:31 2010 @@ -151,7 +151,6 @@ def repr_of_descr(self): return '<%s>' % self._clsname - class NonGcPtrArrayDescr(BaseArrayDescr): _clsname = 'NonGcPtrArrayDescr' def get_item_size(self, translate_support_code): @@ -161,17 +160,45 @@ _clsname = 'GcPtrArrayDescr' _is_array_of_pointers = True +_CA = rffi.CArray(lltype.Signed) + +class BaseArrayNoLengthDescr(BaseArrayDescr): + def get_base_size(self, translate_support_code): + basesize, _, _ = symbolic.get_array_token(_CA, translate_support_code) + return basesize + + def get_ofs_length(self, translate_support_code): + _, _, ofslength = symbolic.get_array_token(_CA, translate_support_code) + return ofslength + +class NonGcPtrArrayNoLengthDescr(BaseArrayNoLengthDescr): + _clsname = 'NonGcPtrArrayNoLengthDescr' + def get_item_size(self, translate_support_code): + return symbolic.get_size_of_ptr(translate_support_code) + +class GcPtrArrayNoLengthDescr(NonGcPtrArrayNoLengthDescr): + _clsname = 'GcPtrArrayNoLengthDescr' + _is_array_of_pointers = True + def getArrayDescrClass(ARRAY): return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr, NonGcPtrArrayDescr, 'Array', 'get_item_size', '_is_array_of_floats') +def getArrayNoLengthDescrClass(ARRAY): + return getDescrClass(ARRAY.OF, BaseArrayNoLengthDescr, GcPtrArrayNoLengthDescr, + NonGcPtrArrayNoLengthDescr, 'ArrayNoLength', 'get_item_size', + '_is_array_of_floats') + def get_array_descr(gccache, ARRAY): cache = gccache._cache_array try: return cache[ARRAY] except KeyError: - arraydescr = getArrayDescrClass(ARRAY)() + if ARRAY._hints.get('nolength', False): + arraydescr = getArrayNoLengthDescrClass(ARRAY)() + else: + arraydescr = getArrayDescrClass(ARRAY)() # verify basic assumption that all arrays' basesize and ofslength # are equal basesize, itemsize, ofslength = symbolic.get_array_token(ARRAY, False) Modified: pypy/branch/interplevel-array/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llsupport/llmodel.py Tue Jul 6 21:16:31 2010 @@ -250,6 +250,7 @@ ofs = arraydescr.get_ofs_length(self.translate_support_code) return rffi.cast(rffi.CArrayPtr(lltype.Signed), array)[ofs/WORD] + @specialize.argtype(2) def bh_getarrayitem_gc_i(self, arraydescr, gcref, itemindex): ofs, size = self.unpack_arraydescr_size(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -272,6 +273,7 @@ # --- end of GC unsafe code --- return pval + @specialize.argtype(2) def bh_getarrayitem_gc_f(self, arraydescr, gcref, itemindex): ofs = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -281,6 +283,7 @@ # --- end of GC unsafe code --- return fval + @specialize.argtype(2) def bh_setarrayitem_gc_i(self, arraydescr, gcref, itemindex, newvalue): ofs, size = self.unpack_arraydescr_size(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -303,6 +306,7 @@ items[itemindex] = self.cast_gcref_to_int(newvalue) # --- end of GC unsafe code --- + @specialize.argtype(2) def bh_setarrayitem_gc_f(self, arraydescr, gcref, itemindex, newvalue): ofs = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -311,6 +315,12 @@ items[itemindex] = newvalue # --- end of GC unsafe code --- + bh_setarrayitem_raw_i = bh_setarrayitem_gc_i + bh_setarrayitem_raw_f = bh_setarrayitem_gc_f + + bh_getarrayitem_raw_i = bh_getarrayitem_gc_i + bh_getarrayitem_raw_f = bh_getarrayitem_gc_f + def bh_strlen(self, string): s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) return len(s.chars) Modified: pypy/branch/interplevel-array/pypy/jit/backend/llsupport/symbolic.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llsupport/symbolic.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llsupport/symbolic.py Tue Jul 6 21:16:31 2010 @@ -36,8 +36,11 @@ ofs_length = (llmemory.offsetof(T, T._arrayfld) + llmemory.ArrayLengthOffset(SUBARRAY)) else: + if T._hints.get('nolength', None): + ofs_length = -1 + else: + ofs_length = llmemory.ArrayLengthOffset(T) itemsize = llmemory.sizeof(T.OF) - ofs_length = llmemory.ArrayLengthOffset(T) else: if isinstance(T, lltype.Struct): assert T._arrayfld is not None, "%r is not variable-sized" % (T,) @@ -48,8 +51,11 @@ else: before_array_part = 0 carray = ll2ctypes.get_ctypes_type(T) - assert carray.length.size == WORD - ofs_length = before_array_part + carray.length.offset + if T._hints.get('nolength', None): + ofs_length = -1 + else: + assert carray.length.size == WORD + ofs_length = before_array_part + carray.length.offset basesize = before_array_part + carray.items.offset carrayitem = ll2ctypes.get_ctypes_type(T.OF) itemsize = ctypes.sizeof(carrayitem) Modified: pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_descr.py Tue Jul 6 21:16:31 2010 @@ -140,7 +140,25 @@ assert isinstance(descr2.get_item_size(True), Symbolic) assert isinstance(descr3.get_item_size(True), Symbolic) assert isinstance(descr4.get_item_size(True), Symbolic) - + CA = rffi.CArray(lltype.Signed) + descr = get_array_descr(c0, CA) + assert not descr.is_array_of_floats() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Ptr(lltype.GcStruct('S'))) + descr = get_array_descr(c0, CA) + assert descr.is_array_of_pointers() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Ptr(lltype.Struct('S'))) + descr = get_array_descr(c0, CA) + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Float) + descr = get_array_descr(c0, CA) + assert descr.is_array_of_floats() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 def test_get_call_descr_not_translated(): c0 = GcCache(False) Modified: pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_symbolic.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_symbolic.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_symbolic.py Tue Jul 6 21:16:31 2010 @@ -61,6 +61,12 @@ assert basesize >= WORD # at least the 'length', maybe some gc headers assert itemsize == WORD assert ofs_length == basesize - WORD + A = rffi.CArray(lltype.Signed) + arraytok = get_array_token(A, translate_support_code) + basesize, itemsize, ofs_length = convert(arraytok) + assert basesize == 0 + assert itemsize == WORD + assert ofs_length == -1 def test_varsized_struct_size(): S1 = lltype.GcStruct('S1', ('parent', S), Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py Tue Jul 6 21:16:31 2010 @@ -815,6 +815,7 @@ raise NotImplementedError() genop_getarrayitem_gc_pure = genop_getarrayitem_gc + genop_getarrayitem_raw = genop_getarrayitem_gc def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/regalloc.py Tue Jul 6 21:16:31 2010 @@ -783,12 +783,14 @@ arglocs.append(self.loc(op.args[0])) return self._call(op, arglocs) # boehm GC (XXX kill the following code at some point) - scale_of_field, basesize, _ = self._unpack_arraydescr(op.descr) - return self._malloc_varsize(basesize, 0, scale_of_field, op.args[0], - op.result) + scale_of_field, basesize, ofs_length, _ = ( + self._unpack_arraydescr(op.descr)) + return self._malloc_varsize(basesize, ofs_length, scale_of_field, + op.args[0], op.result) def _unpack_arraydescr(self, arraydescr): assert isinstance(arraydescr, BaseArrayDescr) + ofs_length = arraydescr.get_ofs_length(self.translate_support_code) ofs = arraydescr.get_base_size(self.translate_support_code) size = arraydescr.get_item_size(self.translate_support_code) ptr = arraydescr.is_array_of_pointers() @@ -796,7 +798,7 @@ while (1 << scale) < size: scale += 1 assert (1 << scale) == size - return scale, ofs, ptr + return scale, ofs, ofs_length, ptr def _unpack_fielddescr(self, fielddescr): assert isinstance(fielddescr, BaseFieldDescr) @@ -831,7 +833,7 @@ consider_unicodesetitem = consider_strsetitem def consider_setarrayitem_gc(self, op): - scale, ofs, ptr = self._unpack_arraydescr(op.descr) + scale, ofs, _, ptr = self._unpack_arraydescr(op.descr) base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) if scale == 0: need_lower_byte = True @@ -858,13 +860,14 @@ consider_getfield_gc_pure = consider_getfield_gc def consider_getarrayitem_gc(self, op): - scale, ofs, _ = self._unpack_arraydescr(op.descr) + scale, ofs, _, _ = self._unpack_arraydescr(op.descr) base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) ofs_loc = self.rm.make_sure_var_in_reg(op.args[1], op.args) self.rm.possibly_free_vars(op.args) result_loc = self.force_allocate_reg(op.result) self.Perform(op, [base_loc, ofs_loc, imm(scale), imm(ofs)], result_loc) + consider_getarrayitem_raw = consider_getarrayitem_gc consider_getarrayitem_gc_pure = consider_getarrayitem_gc def consider_int_is_true(self, op, guard_op): Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_zrpy_gc.py Tue Jul 6 21:16:31 2010 @@ -105,6 +105,12 @@ def test_compile_boehm(): myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) + @dont_look_inside + def see(lst, n): + assert len(lst) == 3 + assert lst[0] == n+10 + assert lst[1] == n+20 + assert lst[2] == n+30 def main(n, x): while n > 0: myjitdriver.can_enter_jit(n=n, x=x) @@ -112,6 +118,7 @@ y = X() y.foo = x.foo n -= y.foo + see([n+10, n+20, n+30], n) res = compile_and_run(get_entry(get_g(main)), "boehm", jit=True) assert int(res) >= 16 Modified: pypy/branch/interplevel-array/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/interplevel-array/pypy/jit/codewriter/jtransform.py Tue Jul 6 21:16:31 2010 @@ -403,7 +403,12 @@ log.WARNING('ignoring hint %r at %r' % (hints, self.graph)) def rewrite_op_malloc_varsize(self, op): - assert op.args[1].value == {'flavor': 'gc'} + if op.args[1].value['flavor'] == 'raw': + ARRAY = op.args[0].value + return self._do_builtin_call(op, 'raw_malloc', + [op.args[2]], + extra = (ARRAY,), + extrakey = ARRAY) if op.args[0].value == rstr.STR: return SpaceOperation('newstr', [op.args[2]], op.result) elif op.args[0].value == rstr.UNICODE: @@ -415,9 +420,14 @@ return SpaceOperation('new_array', [arraydescr, op.args[2]], op.result) + def rewrite_op_free(self, op): + assert op.args[1].value == 'raw' + ARRAY = op.args[0].concretetype.TO + return self._do_builtin_call(op, 'raw_free', [op.args[0]], + extra = (ARRAY,), extrakey = ARRAY) + def rewrite_op_getarrayitem(self, op): ARRAY = op.args[0].concretetype.TO - assert ARRAY._gckind == 'gc' if self._array_of_voids(ARRAY): return [] if op.args[0] in self.vable_array_vars: # for virtualizables @@ -431,13 +441,12 @@ # normal case follows arraydescr = self.cpu.arraydescrof(ARRAY) kind = getkind(op.result.concretetype) - return SpaceOperation('getarrayitem_gc_%s' % kind[0], + return SpaceOperation('getarrayitem_%s_%s' % (ARRAY._gckind, kind[0]), [op.args[0], arraydescr, op.args[1]], op.result) def rewrite_op_setarrayitem(self, op): ARRAY = op.args[0].concretetype.TO - assert ARRAY._gckind == 'gc' if self._array_of_voids(ARRAY): return [] if op.args[0] in self.vable_array_vars: # for virtualizables @@ -450,7 +459,7 @@ op.args[1], op.args[2]], None)] arraydescr = self.cpu.arraydescrof(ARRAY) kind = getkind(op.args[2].concretetype) - return SpaceOperation('setarrayitem_gc_%s' % kind[0], + return SpaceOperation('setarrayitem_%s_%s' % (ARRAY._gckind, kind[0]), [op.args[0], arraydescr, op.args[1], op.args[2]], None) @@ -658,7 +667,7 @@ return self._rewrite_symmetric(op) def _is_gc(self, v): - return v.concretetype.TO._gckind == 'gc' + return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc' def _rewrite_cmp_ptrs(self, op): if self._is_gc(op.args[0]): @@ -692,6 +701,17 @@ #return op raise NotImplementedError("cast_ptr_to_int") + def rewrite_op_force_cast(self, op): + from pypy.rpython.lltypesystem.rffi import size_and_sign + from pypy.rlib.rarithmetic import intmask + assert not self._is_gc(op.args[0]) + size1, unsigned1 = size_and_sign(op.args[0].concretetype) + size2, unsigned2 = size_and_sign(op.result.concretetype) + if size1 == size2 and unsigned1 == unsigned2: + return + raise NotImplementedError("cast not supported yet: %s" % (op, )) + + # ---------- # Renames, from the _old opname to the _new one. # The new operation is optionally further processed by rewrite_operation(). Modified: pypy/branch/interplevel-array/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/codewriter/support.py (original) +++ pypy/branch/interplevel-array/pypy/jit/codewriter/support.py Tue Jul 6 21:16:31 2010 @@ -282,6 +282,9 @@ # ---------- malloc with del ---------- + def _ll_2_raw_malloc(TP, size): + return lltype.malloc(TP, size, flavor='raw') + def build_ll_0_alloc_with_del(RESULT, vtable): def _ll_0_alloc_with_del(): p = lltype.malloc(RESULT.TO) @@ -289,6 +292,15 @@ return p return _ll_0_alloc_with_del + def build_ll_1_raw_malloc(ARRAY): + def _ll_1_raw_malloc(n): + return lltype.malloc(ARRAY, n, flavor='raw') + return _ll_1_raw_malloc + + def build_ll_1_raw_free(ARRAY): + def _ll_1_raw_free(p): + lltype.free(p, flavor='raw') + return _ll_1_raw_free class OOtypeHelpers: Modified: pypy/branch/interplevel-array/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/interplevel-array/pypy/jit/codewriter/test/test_codewriter.py Tue Jul 6 21:16:31 2010 @@ -2,7 +2,7 @@ from pypy.jit.codewriter.codewriter import CodeWriter from pypy.jit.codewriter import support from pypy.jit.metainterp.history import AbstractDescr -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rffi class FakeCallDescr(AbstractDescr): def __init__(self, FUNC, ARGS, RESULT, effectinfo=None): @@ -24,12 +24,17 @@ def as_vtable_size_descr(self): return self +class FakeArrayDescr(AbstractDescr): + def __init__(self, ARRAY): + self.ARRAY = ARRAY + class FakeCPU: def __init__(self, rtyper): self.rtyper = rtyper calldescrof = FakeCallDescr fielddescrof = FakeFieldDescr sizeof = FakeSizeDescr + arraydescrof = FakeArrayDescr class FakePolicy: def look_inside_graph(self, graph): @@ -157,3 +162,25 @@ # s = jitdriver_sd.mainjitcode.dump() assert "inline_call_ir_i " in s + +def test_raw_malloc_and_access(): + TP = rffi.CArray(lltype.Signed) + + def f(n): + a = lltype.malloc(TP, n, flavor='raw') + a[0] = n + res = a[0] + lltype.free(a, flavor='raw') + return res + + rtyper = support.annotate(f, [35]) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cw = CodeWriter(FakeCPU(rtyper), [jitdriver_sd]) + cw.find_all_graphs(FakePolicy()) + cw.make_jitcodes(verbose=True) + # + s = jitdriver_sd.mainjitcode.dump() + assert 'residual_call_ir_i $<* fn _ll_1_raw_malloc__Signed>' in s + assert 'setarrayitem_raw_i' in s + assert 'getarrayitem_raw_i' in s + assert 'residual_call_ir_v $<* fn _ll_1_raw_free__arrayPtr>' in s Modified: pypy/branch/interplevel-array/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/interplevel-array/pypy/jit/codewriter/test/test_flatten.py Tue Jul 6 21:16:31 2010 @@ -729,3 +729,32 @@ int_between %i0, %i1, %i2 -> %i3 int_return %i3 """, transform=True) + + def test_force_cast(self): + py.test.skip("later") + from pypy.rpython.lltypesystem import rffi + def f(n): + c = chr(n) + return rffi.cast(rffi.INT, c) + self.encoding_test(f, [42], """ + int_return %i0 + """, transform=True) + def g(n): + return rffi.cast(rffi.UCHAR, n) + self.encoding_test(g, [42], """ + int_and %i0, $255 -> %i1 + int_return %i1 + """, transform=True) + def h(n): + return rffi.cast(rffi.SCHAR, n) + self.encoding_test(h, [42], """ + ... + """, transform=True) + + def test_force_cast_pointer(self): + from pypy.rpython.lltypesystem import rffi + def h(p): + return rffi.cast(rffi.VOIDP, p) + self.encoding_test(h, [lltype.nullptr(rffi.CCHARP.TO)], """ + int_return %i0 + """, transform=True) Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/blackhole.py Tue Jul 6 21:16:31 2010 @@ -990,6 +990,13 @@ bhimpl_getarrayitem_gc_pure_r = bhimpl_getarrayitem_gc_r bhimpl_getarrayitem_gc_pure_f = bhimpl_getarrayitem_gc_f + @arguments("cpu", "i", "d", "i", returns="i") + def bhimpl_getarrayitem_raw_i(cpu, array, arraydescr, index): + return cpu.bh_getarrayitem_raw_i(arraydescr, array, index) + @arguments("cpu", "i", "d", "i", returns="f") + def bhimpl_getarrayitem_raw_f(cpu, array, arraydescr, index): + return cpu.bh_getarrayitem_raw_f(arraydescr, array, index) + @arguments("cpu", "r", "d", "i", "i") def bhimpl_setarrayitem_gc_i(cpu, array, arraydescr, index, newvalue): cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue) @@ -1000,6 +1007,15 @@ def bhimpl_setarrayitem_gc_f(cpu, array, arraydescr, index, newvalue): cpu.bh_setarrayitem_gc_f(arraydescr, array, index, newvalue) + @arguments("cpu", "i", "d", "i", "i") + def bhimpl_setarrayitem_raw_i(cpu, array, arraydescr, index, newvalue): + cpu.bh_setarrayitem_raw_i(arraydescr, array, index, newvalue) + @arguments("cpu", "i", "d", "i", "f") + def bhimpl_setarrayitem_raw_f(cpu, array, arraydescr, index, newvalue): + cpu.bh_setarrayitem_raw_f(arraydescr, array, index, newvalue) + + # note, there is no 'r' here, since it can't happen + @arguments("cpu", "r", "d", returns="i") def bhimpl_arraylen_gc(cpu, array, arraydescr): return cpu.bh_arraylen_gc(arraydescr, array) Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/executor.py Tue Jul 6 21:16:31 2010 @@ -90,6 +90,15 @@ else: return BoxInt(cpu.bh_getarrayitem_gc_i(arraydescr, array, index)) +def do_getarrayitem_raw(cpu, _, arraybox, indexbox, arraydescr): + array = arraybox.getint() + index = indexbox.getint() + assert not arraydescr.is_array_of_pointers() + if arraydescr.is_array_of_floats(): + return BoxFloat(cpu.bh_getarrayitem_raw_f(arraydescr, array, index)) + else: + return BoxInt(cpu.bh_getarrayitem_raw_i(arraydescr, array, index)) + def do_setarrayitem_gc(cpu, _, arraybox, indexbox, itembox, arraydescr): array = arraybox.getref_base() index = indexbox.getint() @@ -101,6 +110,15 @@ else: cpu.bh_setarrayitem_gc_i(arraydescr, array, index, itembox.getint()) +def do_setarrayitem_raw(cpu, _, arraybox, indexbox, itembox, arraydescr): + array = arraybox.getint() + index = indexbox.getint() + assert not arraydescr.is_array_of_pointers() + if arraydescr.is_array_of_floats(): + cpu.bh_setarrayitem_raw_f(arraydescr, array, index, itembox.getfloat()) + else: + cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint()) + def do_getfield_gc(cpu, _, structbox, fielddescr): struct = structbox.getref_base() if fielddescr.is_pointer_field(): Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/pyjitpl.py Tue Jul 6 21:16:31 2010 @@ -378,6 +378,14 @@ opimpl_getarrayitem_gc_f = _opimpl_getarrayitem_gc_any @arguments("box", "descr", "box") + def _opimpl_getarrayitem_raw_any(self, arraybox, arraydescr, indexbox): + return self.execute_with_descr(rop.GETARRAYITEM_RAW, + arraydescr, arraybox, indexbox) + + opimpl_getarrayitem_raw_i = _opimpl_getarrayitem_raw_any + opimpl_getarrayitem_raw_f = _opimpl_getarrayitem_raw_any + + @arguments("box", "descr", "box") def _opimpl_getarrayitem_gc_pure_any(self, arraybox, arraydescr, indexbox): return self.execute_with_descr(rop.GETARRAYITEM_GC_PURE, arraydescr, arraybox, indexbox) @@ -396,6 +404,15 @@ opimpl_setarrayitem_gc_r = _opimpl_setarrayitem_gc_any opimpl_setarrayitem_gc_f = _opimpl_setarrayitem_gc_any + @arguments("box", "descr", "box", "box") + def _opimpl_setarrayitem_raw_any(self, arraybox, arraydescr, + indexbox, itembox): + self.execute_with_descr(rop.SETARRAYITEM_RAW, arraydescr, arraybox, + indexbox, itembox) + + opimpl_setarrayitem_raw_i = _opimpl_setarrayitem_raw_any + opimpl_setarrayitem_raw_f = _opimpl_setarrayitem_raw_any + @arguments("box", "descr") def opimpl_arraylen_gc(self, arraybox, arraydescr): return self.execute_with_descr(rop.ARRAYLEN_GC, arraydescr, arraybox) Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/resoperation.py Tue Jul 6 21:16:31 2010 @@ -199,6 +199,7 @@ '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- 'GETARRAYITEM_GC/2d', + 'GETARRAYITEM_RAW/2d', 'GETFIELD_GC/1d', 'GETFIELD_RAW/1d', 'NEW/0d', @@ -209,7 +210,7 @@ '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'SETARRAYITEM_GC/3d', - 'SETARRAYITEM_RAW/3d',#only added by backend.llsupport.gc.rewrite_assembler + 'SETARRAYITEM_RAW/3d', 'SETFIELD_GC/2d', 'SETFIELD_RAW/2d', 'ARRAYCOPY/7d', # removed before it's passed to the backend Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_basic.py Tue Jul 6 21:16:31 2010 @@ -1562,6 +1562,35 @@ res = self.interp_operations(f, [3, 2]) assert res == 1 + def test_raw_malloc_and_access(self): + from pypy.rpython.lltypesystem import rffi + + TP = rffi.CArray(lltype.Signed) + + def f(n): + a = lltype.malloc(TP, n, flavor='raw') + a[0] = n + res = a[0] + lltype.free(a, flavor='raw') + return res + + res = self.interp_operations(f, [10]) + assert res == 10 + + def test_raw_malloc_and_access_float(self): + from pypy.rpython.lltypesystem import rffi + + TP = rffi.CArray(lltype.Float) + + def f(n, f): + a = lltype.malloc(TP, n, flavor='raw') + a[0] = f + res = a[0] + lltype.free(a, flavor='raw') + return res + + res = self.interp_operations(f, [10, 3.5]) + assert res == 3.5 class TestOOtype(BasicTests, OOJitMixin): Modified: pypy/branch/interplevel-array/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_locale/interp_locale.py (original) +++ pypy/branch/interplevel-array/pypy/module/_locale/interp_locale.py Tue Jul 6 21:16:31 2010 @@ -132,7 +132,13 @@ space.is_true(space.isinstance(w_s2, space.w_str)): s1, s2 = space.str_w(w_s1), space.str_w(w_s2) - return space.wrap(_strcoll(rffi.str2charp(s1), rffi.str2charp(s2))) + s1_c = rffi.str2charp(s1) + s2_c = rffi.str2charp(s2) + try: + return space.wrap(_strcoll(s1_c, s2_c)) + finally: + rffi.free_charp(s1_c) + rffi.free_charp(s2_c) #if not space.is_true(space.isinstance(w_s1, space.w_unicode)) and \ # not space.is_true(space.isinstance(w_s2, space.w_unicode)): @@ -143,7 +149,12 @@ s1_c = rffi.unicode2wcharp(s1) s2_c = rffi.unicode2wcharp(s2) - result = _wcscoll(s1_c, s2_c) + try: + result = _wcscoll(s1_c, s2_c) + finally: + rffi.free_wcharp(s1_c) + rffi.free_wcharp(s2_c) + return space.wrap(result) strcoll.unwrap_spec = [ObjSpace, W_Root, W_Root] @@ -156,13 +167,21 @@ n1 = len(s) + 1 buf = lltype.malloc(rffi.CCHARP.TO, n1, flavor="raw", zero=True) - n2 = _strxfrm(buf, rffi.str2charp(s), n1) + 1 + s_c = rffi.str2charp(s) + try: + n2 = _strxfrm(buf, s_c, n1) + 1 + finally: + rffi.free_charp(s_c) if n2 > n1: # more space needed lltype.free(buf, flavor="raw") buf = lltype.malloc(rffi.CCHARP.TO, intmask(n2), flavor="raw", zero=True) - _strxfrm(buf, rffi.str2charp(s), n2) + s_c = rffi.str2charp(s) + try: + _strxfrm(buf, s_c, n2) + finally: + rffi.free_charp(s_c) val = rffi.charp2str(buf) lltype.free(buf, flavor="raw") @@ -194,7 +213,11 @@ def gettext(space, msg): """gettext(msg) -> string Return translation of msg.""" - return space.wrap(rffi.charp2str(_gettext(rffi.str2charp(msg)))) + msg_c = rffi.str2charp(msg) + try: + return space.wrap(rffi.charp2str(_gettext(msg_c))) + finally: + rffi.free_charp(msg_c) gettext.unwrap_spec = [ObjSpace, str] @@ -205,10 +228,20 @@ Return translation of msg in domain.""" if space.is_w(w_domain, space.w_None): domain = None - result = _dgettext(domain, rffi.str2charp(msg)) + msg_c = rffi.str2charp(msg) + try: + result = _dgettext(domain, msg_c) + finally: + rffi.free_charp(msg_c) else: domain = space.str_w(w_domain) - result = _dgettext(rffi.str2charp(domain), rffi.str2charp(msg)) + domain_c = rffi.str2charp(domain) + msg_c = rffi.str2charp(msg) + try: + result = _dgettext(domain_c, msg_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(msg_c) return space.wrap(rffi.charp2str(result)) @@ -223,12 +256,21 @@ if space.is_w(w_domain, space.w_None): domain = None - result = _dcgettext(domain, rffi.str2charp(msg), - rffi.cast(rffi.INT, category)) + msg_c = rffi.str2charp(msg) + try: + result = _dcgettext(domain, msg_c, rffi.cast(rffi.INT, category)) + finally: + rffi.free_charp(msg_c) else: domain = space.str_w(w_domain) - result = _dcgettext(rffi.str2charp(domain), rffi.str2charp(msg), - rffi.cast(rffi.INT, category)) + domain_c = rffi.str2charp(domain) + msg_c = rffi.str2charp(msg) + try: + result = _dcgettext(domain_c, msg_c, + rffi.cast(rffi.INT, category)) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(msg_c) return space.wrap(rffi.charp2str(result)) @@ -246,7 +288,11 @@ result = _textdomain(domain) else: domain = space.str_w(w_domain) - result = _textdomain(rffi.str2charp(domain)) + domain_c = rffi.str2charp(domain) + try: + result = _textdomain(domain_c) + finally: + rffi.free_charp(domain_c) return space.wrap(rffi.charp2str(result)) @@ -261,11 +307,20 @@ if space.is_w(w_dir, space.w_None): dir = None - dirname = _bindtextdomain(rffi.str2charp(domain), dir) + domain_c = rffi.str2charp(domain) + try: + dirname = _bindtextdomain(domain_c, dir) + finally: + rffi.free_charp(domain_c) else: dir = space.str_w(w_dir) - dirname = _bindtextdomain(rffi.str2charp(domain), - rffi.str2charp(dir)) + domain_c = rffi.str2charp(domain) + dir_c = rffi.str2charp(dir) + try: + dirname = _bindtextdomain(domain_c, dir_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(dir_c) if not dirname: errno = rposix.get_errno() @@ -284,12 +339,20 @@ if space.is_w(w_codeset, space.w_None): codeset = None - result = _bind_textdomain_codeset( - rffi.str2charp(domain), codeset) + domain_c = rffi.str2charp(domain) + try: + result = _bind_textdomain_codeset(domain_c, codeset) + finally: + rffi.free_charp(domain_c) else: codeset = space.str_w(w_codeset) - result = _bind_textdomain_codeset(rffi.str2charp(domain), - rffi.str2charp(codeset)) + domain_c = rffi.str2charp(domain) + codeset_c = rffi.str2charp(codeset) + try: + result = _bind_textdomain_codeset(domain_c, codeset_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(codeset_c) if not result: return space.w_None Modified: pypy/branch/interplevel-array/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/cpyext/api.py (original) +++ pypy/branch/interplevel-array/pypy/module/cpyext/api.py Tue Jul 6 21:16:31 2010 @@ -104,8 +104,12 @@ for name in ("pypy_decl.h", "pypy_macros.h"): headers.append(udir.join(name)) for header in headers: - header.copy(dstdir) target = dstdir.join(header.basename) + try: + header.copy(dstdir) + except py.error.EACCES: + target.remove() # maybe it was a read-only file + header.copy(dstdir) target.chmod(0444) # make the file read-only, to make sure that nobody # edits it by mistake Modified: pypy/branch/interplevel-array/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/marshal/interp_marshal.py (original) +++ pypy/branch/interplevel-array/pypy/module/marshal/interp_marshal.py Tue Jul 6 21:16:31 2010 @@ -365,8 +365,8 @@ return self.reader.read(n) def get1(self): - # convince typer to use a char - return chr(ord(self.get(1))) + # the [0] is used to convince the annotator to return a char + return self.get(1)[0] def atom_str(self, typecode): self.start(typecode) Modified: pypy/branch/interplevel-array/pypy/module/termios/interp_termios.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/termios/interp_termios.py (original) +++ pypy/branch/interplevel-array/pypy/module/termios/interp_termios.py Tue Jul 6 21:16:31 2010 @@ -60,8 +60,8 @@ # last one need to be chosen carefully cc_w = [space.wrap(i) for i in cc] if lflag & termios.ICANON: - cc_w[termios.VMIN] = space.wrap(ord(cc[termios.VMIN])) - cc_w[termios.VTIME] = space.wrap(ord(cc[termios.VTIME])) + cc_w[termios.VMIN] = space.wrap(ord(cc[termios.VMIN][0])) + cc_w[termios.VTIME] = space.wrap(ord(cc[termios.VTIME][0])) w_cc = space.newlist(cc_w) l_w.append(w_cc) return space.newlist(l_w) Modified: pypy/branch/interplevel-array/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/interplevel-array/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/interplevel-array/pypy/objspace/std/stringobject.py Tue Jul 6 21:16:31 2010 @@ -863,7 +863,7 @@ space.w_TypeError, "ord() expected a character, but string " "of length %d found", len(u_str)) - return space.wrap(ord(u_str)) + return space.wrap(ord(u_str[0])) def getnewargs__String(space, w_str): return space.newtuple([wrapstr(space, w_str._value)]) Modified: pypy/branch/interplevel-array/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/libffi.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/libffi.py Tue Jul 6 21:16:31 2010 @@ -402,7 +402,7 @@ restype = ffi_type_sint32 elif restype.c_size <= 8: restype = ffi_type_sint64 - + res = c_ffi_prep_cif(self.ll_cif, cc, rffi.cast(rffi.UINT, argnum), restype, self.ll_argtypes) Modified: pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py Tue Jul 6 21:16:31 2010 @@ -54,13 +54,10 @@ NAN = INFINITY / INFINITY def isinf(x): - return x != 0.0 and x / 2 == x + return x == INFINITY or x == -INFINITY -# To get isnan, working x-platform and both on 2.3 and 2.4, is a -# horror. I think this works (for reasons I don't really want to talk -# about), and probably when implemented on top of pypy, too. def isnan(v): - return v != v*1.0 or (v == 1.0 and v == 2.0) + return v != v def intmask(n): if isinstance(n, int): Modified: pypy/branch/interplevel-array/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/runicode.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/runicode.py Tue Jul 6 21:16:31 2010 @@ -822,13 +822,13 @@ else: # when we get here, chr is a 32-bit unicode character if chr <= MAXUNICODE: - builder.append(unichr(chr)) + builder.append(UNICHR(chr)) pos += digits elif chr <= 0x10ffff: chr -= 0x10000L builder.append(unichr(0xD800 + (chr >> 10))) - builder.append(unichr(0xDC00 + (chr & 0x03FF))) + builder.append(unichr(0xDC00 + (chr & 0x03FF))) pos += digits else: message = "illegal Unicode character" @@ -943,7 +943,7 @@ continue pos = look + 1 if code <= MAXUNICODE: - builder.append(unichr(code)) + builder.append(UNICHR(code)) else: code -= 0x10000L builder.append(unichr(0xD800 + (code >> 10))) Modified: pypy/branch/interplevel-array/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/lltypesystem/rffi.py Tue Jul 6 21:16:31 2010 @@ -786,6 +786,8 @@ return r_wchar_t.BITS/8 if tp is lltype.Float: return 8 + if tp is lltype.SingleFloat: + return 4 assert isinstance(tp, lltype.Number) if tp is lltype.Signed: return ULONG._type.BITS/8 Modified: pypy/branch/interplevel-array/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/lltypesystem/test/test_rffi.py Tue Jul 6 21:16:31 2010 @@ -707,6 +707,7 @@ lltype.UniChar: ctypes.c_wchar, lltype.Char: ctypes.c_ubyte, DOUBLE: ctypes.c_double, + FLOAT: ctypes.c_float, SIGNEDCHAR: ctypes.c_byte, UCHAR: ctypes.c_ubyte, SHORT: ctypes.c_short, Modified: pypy/branch/interplevel-array/pypy/rpython/memory/lltypelayout.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/memory/lltypelayout.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/memory/lltypelayout.py Tue Jul 6 21:16:31 2010 @@ -111,6 +111,8 @@ elif isinstance(offset, llmemory.ItemOffset): return sizeof(offset.TYPE) * offset.repeat elif isinstance(offset, llmemory.ArrayItemsOffset): + if offset.TYPE._hints.get('nolength', None): + return 0 return get_fixed_size(lltype.Signed) elif isinstance(offset, llmemory.GCHeaderOffset): return sizeof(offset.gcheaderbuilder.HDR) Modified: pypy/branch/interplevel-array/pypy/rpython/module/ll_termios.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/module/ll_termios.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/module/ll_termios.py Tue Jul 6 21:16:31 2010 @@ -85,7 +85,7 @@ c_struct.c_c_lflag, ispeed, ospeed, cc = attributes try: for i in range(NCCS): - c_struct.c_c_cc[i] = rffi.r_uchar(ord(cc[i])) + c_struct.c_c_cc[i] = rffi.r_uchar(ord(cc[i][0])) error = c_cfsetispeed(c_struct, ispeed) if error == -1: raise termios.error(error, 'tcsetattr failed') Modified: pypy/branch/interplevel-array/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/rstr.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/rstr.py Tue Jul 6 21:16:31 2010 @@ -80,17 +80,6 @@ # defaults to checking the length return super(AbstractStringRepr, self).rtype_is_true(hop) - def rtype_ord(self, hop): - string_repr = hop.args_r[0].repr - v_str, = hop.inputargs(string_repr) - c_zero = inputconst(Signed, 0) - v_chr = hop.gendirectcall(self.ll.ll_stritem_nonneg, v_str, c_zero) - if string_repr is hop.rtyper.type_system.rstr.string_repr: - return hop.genop('cast_char_to_int', [v_chr], resulttype=Signed) - else: - assert string_repr is hop.rtyper.type_system.rstr.unicode_repr - return hop.genop('cast_unichar_to_int', [v_chr], resulttype=Signed) - def rtype_method_startswith(self, hop): str1_repr, str2_repr = self._str_reprs(hop) v_str, v_value = hop.inputargs(str1_repr, str2_repr) Modified: pypy/branch/interplevel-array/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/test/test_rstr.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/test/test_rstr.py Tue Jul 6 21:16:31 2010 @@ -131,7 +131,7 @@ s = c * mul res = 0 for i in range(len(s)): - res = res*10 + ord(const(s[i])) - ord(const('0')) + res = res*10 + ord(const(s[i])[0]) - ord(const('0')[0]) c2 = c c2 *= mul res = 10 * res + (c2 == s) @@ -577,7 +577,7 @@ sum = 0 for num in l: if len(num): - sum += ord(num) - ord(const('0')) + sum += ord(num[0]) - ord(const('0')[0]) return sum + len(l) * 100 for i in range(5): res = self.interpret(fn, [i]) From benjamin at codespeak.net Wed Jul 7 00:59:09 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 00:59:09 +0200 (CEST) Subject: [pypy-svn] r75944 - in pypy/branch/fast-forward/pypy: module/sys objspace/std rlib Message-ID: <20100706225909.19455282C0A@codespeak.net> Author: benjamin Date: Wed Jul 7 00:59:07 2010 New Revision: 75944 Added: pypy/branch/fast-forward/pypy/rlib/rfloat.py (contents, props changed) Modified: pypy/branch/fast-forward/pypy/module/sys/system.py pypy/branch/fast-forward/pypy/objspace/std/floatobject.py pypy/branch/fast-forward/pypy/objspace/std/floattype.py Log: move float constants to rfloat.py Modified: pypy/branch/fast-forward/pypy/module/sys/system.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/sys/system.py (original) +++ pypy/branch/fast-forward/pypy/module/sys/system.py Wed Jul 7 00:59:07 2010 @@ -1,26 +1,6 @@ """Information about the current system.""" - from pypy.interpreter import gateway - -from pypy.rpython.tool import rffi_platform -from pypy.translator.tool.cbuild import ExternalCompilationInfo - - -class CConfig: - _compilation_info_ = ExternalCompilationInfo(includes=["float.h"]) - -float_constants = ["DBL_MAX", "DBL_MIN", "DBL_EPSILON"] -int_constants = ["DBL_MAX_EXP", "DBL_MAX_10_EXP", - "DBL_MIN_EXP", "DBL_MIN_10_EXP", - "DBL_DIG", "DBL_MANT_DIG", - "FLT_RADIX", "FLT_ROUNDS"] -for const in float_constants: - setattr(CConfig, const, rffi_platform.DefinedConstantDouble(const)) -for const in int_constants: - setattr(CConfig, const, rffi_platform.DefinedConstantInteger(const)) -del float_constants, int_constants, const - -globals().update(rffi_platform.configure(CConfig)) +from pypy.rlib import rfloat app = gateway.applevel(""" @@ -45,17 +25,17 @@ def get_float_info(space): info_w = [ - space.wrap(DBL_MAX), - space.wrap(DBL_MAX_EXP), - space.wrap(DBL_MAX_10_EXP), - space.wrap(DBL_MIN), - space.wrap(DBL_MIN_EXP), - space.wrap(DBL_MIN_10_EXP), - space.wrap(DBL_DIG), - space.wrap(DBL_MANT_DIG), - space.wrap(DBL_EPSILON), - space.wrap(FLT_RADIX), - space.wrap(FLT_ROUNDS), + space.wrap(rfloat.DBL_MAX), + space.wrap(rfloat.DBL_MAX_EXP), + space.wrap(rfloat.DBL_MAX_10_EXP), + space.wrap(rfloat.DBL_MIN), + space.wrap(rfloat.DBL_MIN_EXP), + space.wrap(rfloat.DBL_MIN_10_EXP), + space.wrap(rfloat.DBL_DIG), + space.wrap(rfloat.DBL_MANT_DIG), + space.wrap(rfloat.DBL_EPSILON), + space.wrap(rfloat.FLT_RADIX), + space.wrap(rfloat.FLT_ROUNDS), ] w_float_info = app.wget(space, "float_info") return space.call_function(w_float_info, space.newtuple(info_w)) Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Wed Jul 7 00:59:07 2010 @@ -8,11 +8,11 @@ from pypy.objspace.std.register_all import register_all from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.longobject import W_LongObject -from pypy.module.sys import system from pypy.rlib.rarithmetic import ovfcheck_float_to_int, intmask, isinf, isnan from pypy.rlib.rarithmetic import formatd, LONG_BIT, FL_MAXINT, FL_MININT from pypy.rlib.rbigint import rbigint from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib import rfloat from pypy.tool.sourcetools import func_with_new_name import math @@ -92,7 +92,7 @@ def _char_from_hex(number): return "0123456789abcdef"[number] -TOHEX_NBITS = system.DBL_MANT_DIG + 3 - (system.DBL_MANT_DIG + 2) % 4 +TOHEX_NBITS = rfloat.DBL_MANT_DIG + 3 - (rfloat.DBL_MANT_DIG + 2) % 4 def float_hex__Float(space, w_float): value = w_float.floatval @@ -104,7 +104,7 @@ else: return space.wrap("0x0.0p+0") mant, exp = math.frexp(value) - shift = 1 - max(system.DBL_MIN_EXP - exp, 0) + shift = 1 - max(rfloat.DBL_MIN_EXP - exp, 0) mant = math.ldexp(mant, shift) mant = abs(mant) exp -= shift Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floattype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floattype.py Wed Jul 7 00:59:07 2010 @@ -1,13 +1,13 @@ import math import sys from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib import rfloat from pypy.interpreter import gateway from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.error import OperationError from pypy.objspace.std.stdtypedef import StdTypeDef, SMM from pypy.objspace.std.strutil import ParseStringError from pypy.objspace.std.strutil import interp_string_to_float -from pypy.module.sys import system float_as_integer_ratio = SMM("as_integer_ratio", 1) @@ -127,8 +127,8 @@ if not total_digits: raise OperationError(space.w_ValueError, space.wrap("invalid hex string")) - const_one = system.DBL_MIN_EXP - system.DBL_MANT_DIG + sys.maxint // 2 - const_two = sys.maxint // 2 + 1 - system.DBL_MAX_EXP + const_one = rfloat.DBL_MIN_EXP - rfloat.DBL_MANT_DIG + sys.maxint // 2 + const_two = sys.maxint // 2 + 1 - rfloat.DBL_MAX_EXP if total_digits > min(const_one, const_two) // 4: raise OperationError(space.w_ValueError, space.wrap("way too long")) if i < length and (s[i] == "p" or s[i] == "P"): @@ -165,13 +165,13 @@ while digit: top_exp += 1 digit //= 2 - if top_exp < system.DBL_MIN_EXP - system.DBL_MANT_DIG: + if top_exp < rfloat.DBL_MIN_EXP - rfloat.DBL_MANT_DIG: value = 0.0 - elif top_exp > system.DBL_MAX_EXP: + elif top_exp > rfloat.DBL_MAX_EXP: raise OperationError(space.w_OverflowError, space.wrap("too large")) else: - lsb = max(top_exp, system.DBL_MIN_EXP) - system.DBL_MANT_DIG + lsb = max(top_exp, rfloat.DBL_MIN_EXP) - rfloat.DBL_MANT_DIG value = 0 if exp >= lsb: for j in range(total_digits - 1, -1, -1): @@ -199,8 +199,8 @@ break if round_up: value += 2 * half_eps - mant_dig = system.DBL_MANT_DIG - if (top_exp == system.DBL_MAX_EXP and + mant_dig = rfloat.DBL_MANT_DIG + if (top_exp == rfloat.DBL_MAX_EXP and value == math.ldexp(2 * half_eps, mant_dig)): raise OperationError(space.w_OverflowError, space.wrap("too large")) Added: pypy/branch/fast-forward/pypy/rlib/rfloat.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/pypy/rlib/rfloat.py Wed Jul 7 00:59:07 2010 @@ -0,0 +1,21 @@ +"""Float constants""" + +from pypy.rpython.tool import rffi_platform +from pypy.translator.tool.cbuild import ExternalCompilationInfo + + +class CConfig: + _compilation_info_ = ExternalCompilationInfo(includes=["float.h"]) + +float_constants = ["DBL_MAX", "DBL_MIN", "DBL_EPSILON"] +int_constants = ["DBL_MAX_EXP", "DBL_MAX_10_EXP", + "DBL_MIN_EXP", "DBL_MIN_10_EXP", + "DBL_DIG", "DBL_MANT_DIG", + "FLT_RADIX", "FLT_ROUNDS"] +for const in float_constants: + setattr(CConfig, const, rffi_platform.DefinedConstantDouble(const)) +for const in int_constants: + setattr(CConfig, const, rffi_platform.DefinedConstantInteger(const)) +del float_constants, int_constants, const + +globals().update(rffi_platform.configure(CConfig)) From benjamin at codespeak.net Wed Jul 7 01:20:10 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 01:20:10 +0200 (CEST) Subject: [pypy-svn] r75945 - in pypy/branch/fast-forward/pypy: module/math module/math/test rlib rpython/lltypesystem/module Message-ID: <20100706232010.8F07C282C0A@codespeak.net> Author: benjamin Date: Wed Jul 7 01:20:08 2010 New Revision: 75945 Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py pypy/branch/fast-forward/pypy/module/math/interp_math.py pypy/branch/fast-forward/pypy/module/math/test/test_math.py pypy/branch/fast-forward/pypy/rlib/rarithmetic.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Log: add log1p, asinh, acosh, and atanh Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/math/__init__.py Wed Jul 7 01:20:08 2010 @@ -40,5 +40,9 @@ 'trunc' : 'interp_math.trunc', 'fsum' : 'interp_math.fsum', 'factorial' : 'interp_math.factorial', + 'asinh' : 'interp_math.asinh', + 'acosh' : 'interp_math.acosh', + 'atanh' : 'interp_math.atanh', + 'log1p' : 'interp_math.log1p', } Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Wed Jul 7 01:20:08 2010 @@ -396,3 +396,23 @@ for i in range(1, x + 1): w_res = space.mul(w_res, space.wrap(i)) return w_res + +def log1p(space, x): + """Find log(x + 1).""" + return math1(space, rarithmetic.log1p, x) +log1p.unwrap_spec = [ObjSpace, float] + +def acosh(space, x): + """Inverse hyperbolic cosine""" + return math1(space, rarithmetic.acosh, x) +acosh.unwrap_spec = [ObjSpace, float] + +def asinh(space, x): + """Inverse hyperbolic sine""" + return math1(space, rarithmetic.asinh, x) +asinh.unwrap_spec = [ObjSpace, float] + +def atanh(space, x): + """Inverse hyperbolic tangent""" + return math1(space, rarithmetic.atanh, x) +atanh.unwrap_spec = [ObjSpace, float] Modified: pypy/branch/fast-forward/pypy/module/math/test/test_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/test/test_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/test/test_math.py Wed Jul 7 01:20:08 2010 @@ -8,6 +8,10 @@ cls.space = gettestobjspace(usemodules=['math']) cls.w_cases = cls.space.wrap(test_direct.MathTests.TESTCASES) cls.w_consistent_host = cls.space.wrap(test_direct.consistent_host) + cls.w_ftest = cls.space.appexec((), """(): + def ftest(actual, expected): + assert abs(actual - expected) < 10E-5 + return ftest""") def test_all_cases(self): if not self.consistent_host: @@ -86,6 +90,35 @@ raises(ValueError, math.factorial, -1.) raises(ValueError, math.factorial, 1.1) + def test_log1p(self): + import math + self.ftest(math.log1p(1/math.e-1), -1) + self.ftest(math.log1p(0), 0) + self.ftest(math.log1p(math.e-1), 1) + self.ftest(math.log1p(1), math.log(2)) + + def test_acosh(self): + import math + self.ftest(math.acosh(1), 0) + self.ftest(math.acosh(2), 1.3169578969248168) + assert math.isinf(math.asinh(float("inf"))) + raises(ValueError, math.acosh, 0) + + def test_asinh(self): + import math + self.ftest(math.asinh(0), 0) + self.ftest(math.asinh(1), 0.88137358701954305) + self.ftest(math.asinh(-1), -0.88137358701954305) + assert math.isinf(math.asinh(float("inf"))) + + def test_atanh(self): + import math + self.ftest(math.atanh(0), 0) + self.ftest(math.atanh(0.5), 0.54930614433405489) + self.ftest(math.atanh(-0.5), -0.54930614433405489) + raises(ValueError, math.atanh, 1.) + assert math.isnan(math.atanh(float("nan"))) + def test_mtestfile(self): import math import abc Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Wed Jul 7 01:20:08 2010 @@ -35,8 +35,8 @@ """ import math from pypy.rpython import extregistry - from pypy.rlib import objectmodel + # set up of machine internals _bits = 0 _itest = 1 @@ -54,7 +54,8 @@ NAN = INFINITY / INFINITY try: - from math import isinf, isnan, copysign + # Try to get math functions added in 2.6. + from math import isinf, isnan, copysign, acosh, asinh, atanh, log1p except ImportError: def isinf(x): return x == INFINITY or x == -INFINITY @@ -69,6 +70,67 @@ else: return -math.fabs(x) + _2_to_m28 = 3.7252902984619141E-09; # 2**-28 + _2_to_p28 = 268435456.0; # 2**28 + _ln2 = 6.93147180559945286227E-01 + + def acosh(x): + if isnan(x): + return NAN + if x < 1.: + raise ValueError("math domain error") + if x >= _2_to_p28: + if isinf(x): + return x + else: + return math.log(x) + _ln2 + if x == 1.: + return 0. + if x >= 2.: + t = x * x + return math.log(2. * x - 1. / (x + math.sqrt(t - 1.0))) + t = x - 1.0 + return log1p(t + math.sqrt(2. * t + t * t)) + + def asinh(x): + absx = abs(x) + if isnan(x) or isinf(x): + return x + if absx < _2_to_m28: + return x + if absx > _2_to_p28: + w = math.log(absx) + _ln2 + elif absx > 2.: + w = math.log(2. * absx + 1. // (math.sqrt(x * x + 1.) + absx)) + else: + t = x * x + w = log1p(absx + t // (1. + math.sqrt(1. + t))) + return copysign(w, x) + + def atanh(x): + if isnan(x): + return x + absx = abs(x) + if absx >= 1.: + raise ValueError("math domain error") + if absx < _2_to_m28: + return x + if absx < .5: + t = absx + absx + t = .5 * log1p(t + t * absx // (1. - absx)) + else: + t = .5 * log1p((absx + absx) // (1. - absx)) + return copysign(t, x) + + def log1p(x): + from pypy.rlib import rfloat + if abs(x) < rfloat.DBL_EPSILON // 2.: + return x + elif -.5 <= x <= 1.: + y = 1. + x + return math.log(y) - ((y - 1.) - x) // y + else: + return math.log(1. + x) def intmask(n): if isinstance(n, int): Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Wed Jul 7 01:20:08 2010 @@ -316,7 +316,7 @@ 'acos', 'asin', 'atan', 'ceil', 'cos', 'cosh', 'exp', 'fabs', 'floor', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', 'log', 'log10', - # 'log1p', 'acosh', 'asinh', 'atanh', -- added in Python 2.6 + 'acosh', 'asinh', 'atanh', # -- added in Python 2.6 ] unary_math_functions_can_overflow = [ 'cosh', 'exp', 'log1p', 'sinh', # why log1p? CPython does it From benjamin at codespeak.net Wed Jul 7 01:20:47 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 01:20:47 +0200 (CEST) Subject: [pypy-svn] r75946 - pypy/branch/fast-forward/pypy/rlib Message-ID: <20100706232047.76962282C0A@codespeak.net> Author: benjamin Date: Wed Jul 7 01:20:46 2010 New Revision: 75946 Modified: pypy/branch/fast-forward/pypy/rlib/rstring.py Log: these are technically correct, but it's very annoying to prove to the annotator _every_ time Modified: pypy/branch/fast-forward/pypy/rlib/rstring.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstring.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstring.py Wed Jul 7 01:20:46 2010 @@ -105,14 +105,11 @@ assert isinstance(s_str, SomeString) assert isinstance(s_start, SomeInteger) assert isinstance(s_end, SomeInteger) - assert s_start.nonneg - assert s_end.nonneg return s_None def method_append_multiple_char(self, s_char, s_times): assert isinstance(s_char, SomeChar) assert isinstance(s_times, SomeInteger) - assert s_times.nonneg return s_None def method_build(self): @@ -130,14 +127,11 @@ assert isinstance(s_str, SomeUnicodeString) assert isinstance(s_start, SomeInteger) assert isinstance(s_end, SomeInteger) - assert s_start.nonneg - assert s_end.nonneg return s_None def method_append_multiple_char(self, s_char, s_times): assert isinstance(s_char, SomeUnicodeCodePoint) assert isinstance(s_times, SomeInteger) - assert s_times.nonneg return s_None def method_build(self): From benjamin at codespeak.net Wed Jul 7 01:22:07 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 01:22:07 +0200 (CEST) Subject: [pypy-svn] r75947 - pypy/branch/fast-forward/pypy/rpython/lltypesystem/module Message-ID: <20100706232207.40C25282C0A@codespeak.net> Author: benjamin Date: Wed Jul 7 01:22:05 2010 New Revision: 75947 Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Log: log1p is just a normal overflow Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Wed Jul 7 01:22:05 2010 @@ -319,7 +319,7 @@ 'acosh', 'asinh', 'atanh', # -- added in Python 2.6 ] unary_math_functions_can_overflow = [ - 'cosh', 'exp', 'log1p', 'sinh', # why log1p? CPython does it + 'cosh', 'exp', 'log1p', 'sinh', ] for name in unary_math_functions: From benjamin at codespeak.net Wed Jul 7 01:49:00 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 01:49:00 +0200 (CEST) Subject: [pypy-svn] r75948 - pypy/branch/fast-forward/lib_pypy Message-ID: <20100706234900.16304282C0A@codespeak.net> Author: benjamin Date: Wed Jul 7 01:48:59 2010 New Revision: 75948 Modified: pypy/branch/fast-forward/lib_pypy/struct.py Log: fix float packing/unpacking Modified: pypy/branch/fast-forward/lib_pypy/struct.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/struct.py (original) +++ pypy/branch/fast-forward/lib_pypy/struct.py Wed Jul 7 01:48:59 2010 @@ -66,42 +66,6 @@ INFINITY = 1e200 * 1e200 NAN = INFINITY / INFINITY -def unpack_float(data,index,size,le): - bytes = [ord(b) for b in data[index:index+size]] - if len(bytes) != size: - raise StructError,"Not enough data to unpack" - if max(bytes) == 0: - return 0.0 - if le == 'big': - bytes.reverse() - if size == 4: - bias = 127 - exp = 8 - prec = 23 - else: - bias = 1023 - exp = 11 - prec = 52 - mantissa = long(bytes[size-2] & (2**(15-exp)-1)) - for b in bytes[size-3::-1]: - mantissa = mantissa << 8 | b - mantissa = 1 + (1.0*mantissa)/(2**(prec)) - mantissa /= 2 - e = (bytes[-1] & 0x7f) << (exp - 7) - e += (bytes[size-2] >> (15 - exp)) & (2**(exp - 7) -1) - e -= bias - e += 1 - sign = bytes[-1] & 0x80 - if e == bias + 2: - if mantissa == 0.5: - number = INFINITY - else: - return NAN - else: - number = math.ldexp(mantissa,e) - if sign : number *= -1 - return number - def unpack_char(data,index,size,le): return data[index:index+size] @@ -139,54 +103,138 @@ def isnan(v): return v != v*1.0 or (v == 1.0 and v == 2.0) -def pack_float(number, size, le): - if size == 4: - bias = 127 - exp = 8 - prec = 23 +def pack_float(x, size, le): + unsigned = float_pack(x, size) + result = [] + for i in range(8): + result.append(chr((unsigned >> (i * 8)) & 0xFF)) + if le == "big": + result.reverse() + return ''.join(result) + +def unpack_float(data, index, size, le): + binary = data[index:index + 8] + if le == "big": + binary.reverse() + unsigned = 0 + for i in range(8): + unsigned |= ord(binary[i]) << (i * 8) + return float_unpack(unsigned, size) + +def round_to_nearest(x): + """Python 3 style round: round a float x to the nearest int, but + unlike the builtin Python 2.x round function: + + - return an int, not a float + - do round-half-to-even, not round-half-away-from-zero. + + We assume that x is finite and nonnegative; except wrong results + if you use this for negative x. + + """ + int_part = int(x) + frac_part = x - int_part + if frac_part > 0.5 or frac_part == 0.5 and int_part & 1 == 1: + int_part += 1 + return int_part + +def float_unpack(Q, size, le): + """Convert a 32-bit or 64-bit integer created + by float_pack into a Python float.""" + + if size == 8: + MIN_EXP = -1021 # = sys.float_info.min_exp + MAX_EXP = 1024 # = sys.float_info.max_exp + MANT_DIG = 53 # = sys.float_info.mant_dig + BITS = 64 + elif size == 4: + MIN_EXP = -125 # C's FLT_MIN_EXP + MAX_EXP = 128 # FLT_MAX_EXP + MANT_DIG = 24 # FLT_MANT_DIG + BITS = 32 else: - bias = 1023 - exp = 11 - prec = 52 - - if isnan(number): - sign = 0x80 - man, e = 1.5, bias + 1 + raise ValueError("invalid size value") + + if Q >> BITS: + raise ValueError("input out of range") + + # extract pieces + sign = Q >> BITS - 1 + exp = (Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1 + mant = Q & ((1 << MANT_DIG - 1) - 1) + + if exp == MAX_EXP - MIN_EXP + 2: + # nan or infinity + result = float('nan') if mant else float('inf') + elif exp == 0: + # subnormal or zero + result = math.ldexp(float(mant), MIN_EXP - MANT_DIG) else: - if number < 0: - sign = 0x80 - number *= -1 - elif number == 0.0: - return '\x00' * size - else: - sign = 0x00 - if isinf(number): - man, e = 1.0, bias + 1 + # normal + mant += 1 << MANT_DIG - 1 + result = math.ldexp(float(mant), exp + MIN_EXP - MANT_DIG - 1) + return -result if sign else result + + +def float_pack(x, size): + """Convert a Python float x into a 64-bit unsigned integer + with the same byte representation.""" + + if size == 8: + MIN_EXP = -1021 # = sys.float_info.min_exp + MAX_EXP = 1024 # = sys.float_info.max_exp + MANT_DIG = 53 # = sys.float_info.mant_dig + BITS = 64 + elif size == 4: + MIN_EXP = -125 # C's FLT_MIN_EXP + MAX_EXP = 128 # FLT_MAX_EXP + MANT_DIG = 24 # FLT_MANT_DIG + BITS = 32 + else: + raise ValueError("invalid size value") + + sign = math.copysign(1.0, x) < 0.0 + if math.isinf(x): + mant = 0 + exp = MAX_EXP - MIN_EXP + 2 + elif math.isnan(x): + mant = 1 << (MANT_DIG-2) # other values possible + exp = MAX_EXP - MIN_EXP + 2 + elif x == 0.0: + mant = 0 + exp = 0 + else: + m, e = math.frexp(abs(x)) # abs(x) == m * 2**e + exp = e - (MIN_EXP - 1) + if exp > 0: + # Normal case. + mant = round_to_nearest(m * (1 << MANT_DIG)) + mant -= 1 << MANT_DIG - 1 else: - man, e = math.frexp(number) + # Subnormal case. + if exp + MANT_DIG - 1 >= 0: + mant = round_to_nearest(m * (1 << exp + MANT_DIG - 1)) + else: + mant = 0 + exp = 0 + + # Special case: rounding produced a MANT_DIG-bit mantissa. + assert 0 <= mant <= 1 << MANT_DIG - 1 + if mant == 1 << MANT_DIG - 1: + mant = 0 + exp += 1 + + # Raise on overflow (in some circumstances, may want to return + # infinity instead). + if exp >= MAX_EXP - MIN_EXP + 2: + raise OverflowError("float too large to pack in this format") + + # check constraints + assert 0 <= mant < 1 << MANT_DIG - 1 + assert 0 <= exp <= MAX_EXP - MIN_EXP + 2 + assert 0 <= sign <= 1 + return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant - result = [] - if 0.5 <= man and man < 1.0: - man *= 2 - e -= 1 - man -= 1 - e += bias - power_of_two = 1 << prec - mantissa = int(power_of_two * man + 0.5) - if mantissa >> prec : - mantissa = 0 - e += 1 - - for i in range(size-2): - result.append(chr(mantissa & 0xff)) - mantissa >>= 8 - x = (mantissa & ((1<<(15-exp))-1)) | ((e & ((1<<(exp-7))-1))<<(15-exp)) - result.append(chr(x)) - x = sign | e >> (exp - 7) - result.append(chr(x)) - if le == 'big': - result.reverse() - return ''.join(result) big_endian_format = { 'x':{ 'size' : 1, 'alignment' : 0, 'pack' : None, 'unpack' : None}, From benjamin at codespeak.net Wed Jul 7 01:56:22 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 01:56:22 +0200 (CEST) Subject: [pypy-svn] r75949 - in pypy/branch/fast-forward/pypy: module/math module/math/test rlib rpython/lltypesystem/module Message-ID: <20100706235622.492EB282C0A@codespeak.net> Author: benjamin Date: Wed Jul 7 01:56:20 2010 New Revision: 75949 Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py pypy/branch/fast-forward/pypy/module/math/interp_math.py pypy/branch/fast-forward/pypy/module/math/test/test_math.py pypy/branch/fast-forward/pypy/rlib/rarithmetic.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Log: add math.expm1 Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/math/__init__.py Wed Jul 7 01:56:20 2010 @@ -44,5 +44,6 @@ 'acosh' : 'interp_math.acosh', 'atanh' : 'interp_math.atanh', 'log1p' : 'interp_math.log1p', + 'expm1' : 'interp_math.expm1', } Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Wed Jul 7 01:56:20 2010 @@ -416,3 +416,8 @@ """Inverse hyperbolic tangent""" return math1(space, rarithmetic.atanh, x) atanh.unwrap_spec = [ObjSpace, float] + +def expm1(space, x): + """exp(x) - 1""" + return math1(space, rarithmetic.expm1, x) +expm1.unwrap_spec = [ObjSpace, float] Modified: pypy/branch/fast-forward/pypy/module/math/test/test_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/test/test_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/test/test_math.py Wed Jul 7 01:56:20 2010 @@ -123,6 +123,7 @@ import math import abc import os + import struct def _parse_mtestfile(fname): """Parse a file with test values Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Wed Jul 7 01:56:20 2010 @@ -132,6 +132,17 @@ else: return math.log(1. + x) +try: + from math import expm1 # Added in Python 2.7. +except ImportError: + def expm1(x): + if abs(x) < .7: + u = math.exp(x) + if u == 1.: + return x + return (u - 1.) * x / math.log(u) + return math.exp(x) - 1. + def intmask(n): if isinstance(n, int): return int(n) # possibly bool->int Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Wed Jul 7 01:56:20 2010 @@ -319,7 +319,7 @@ 'acosh', 'asinh', 'atanh', # -- added in Python 2.6 ] unary_math_functions_can_overflow = [ - 'cosh', 'exp', 'log1p', 'sinh', + 'cosh', 'exp', 'log1p', 'sinh', 'expm1', ] for name in unary_math_functions: From benjamin at codespeak.net Wed Jul 7 02:07:03 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 02:07:03 +0200 (CEST) Subject: [pypy-svn] r75950 - pypy/branch/fast-forward/pypy/rlib Message-ID: <20100707000703.3F922282C0A@codespeak.net> Author: benjamin Date: Wed Jul 7 02:07:01 2010 New Revision: 75950 Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Log: flooring is quite incorrect Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Wed Jul 7 02:07:01 2010 @@ -101,10 +101,10 @@ if absx > _2_to_p28: w = math.log(absx) + _ln2 elif absx > 2.: - w = math.log(2. * absx + 1. // (math.sqrt(x * x + 1.) + absx)) + w = math.log(2. * absx + 1. / (math.sqrt(x * x + 1.) + absx)) else: t = x * x - w = log1p(absx + t // (1. + math.sqrt(1. + t))) + w = log1p(absx + t / (1. + math.sqrt(1. + t))) return copysign(w, x) def atanh(x): @@ -117,9 +117,9 @@ return x if absx < .5: t = absx + absx - t = .5 * log1p(t + t * absx // (1. - absx)) + t = .5 * log1p(t + t * absx / (1. - absx)) else: - t = .5 * log1p((absx + absx) // (1. - absx)) + t = .5 * log1p((absx + absx) / (1. - absx)) return copysign(t, x) def log1p(x): @@ -128,7 +128,7 @@ return x elif -.5 <= x <= 1.: y = 1. + x - return math.log(y) - ((y - 1.) - x) // y + return math.log(y) - ((y - 1.) - x) / y else: return math.log(1. + x) From benjamin at codespeak.net Wed Jul 7 05:39:45 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 05:39:45 +0200 (CEST) Subject: [pypy-svn] r75951 - pypy/branch/fast-forward/pypy/module/math Message-ID: <20100707033945.5DBD7282BAD@codespeak.net> Author: benjamin Date: Wed Jul 7 05:39:42 2010 New Revision: 75951 Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py pypy/branch/fast-forward/pypy/module/math/interp_math.py Log: add erf and erfc Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/math/__init__.py Wed Jul 7 05:39:42 2010 @@ -45,5 +45,7 @@ 'atanh' : 'interp_math.atanh', 'log1p' : 'interp_math.log1p', 'expm1' : 'interp_math.expm1', + 'erf' : 'interp_math.erf', + 'erfc' : 'interp_math.erfc', } Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Wed Jul 7 05:39:42 2010 @@ -421,3 +421,74 @@ """exp(x) - 1""" return math1(space, rarithmetic.expm1, x) expm1.unwrap_spec = [ObjSpace, float] + +def erf(space, x): + """The error function""" + return math1(space, _erf, x) +erf.unwrap_spec = [ObjSpace, float] + +def erfc(space, x): + """The complementary error function""" + return math1(space, _erfc, x) +erfc.unwrap_spec = [ObjSpace, float] + +# Implementation of the error function, the complimentary error function, the +# gamma function, and the natural log of the gamma function. This exist in +# libm, but I hear those implementations are horrible. + +ERF_SERIES_CUTOFF = 1.5 +ERF_SERIES_TERMS = 25 +ERFC_CONTFRAC_CUTOFF = 30. +ERFC_CONTFRAC_TERMS = 50 +_sqrtpi = 1.772453850905516027298167483341145182798; + +def _erf_series(x): + x2 = x * x + acc = 0. + fk = ERF_SERIES_TERMS * + .5 + for i in range(ERF_SERIES_TERMS): + acc = 2.0 * x2 * acc / fk + fk -= 1. + return acc * x * math.exp(-x2) / _sqrtpi + +def _erfc_contfrac(x): + if x >= ERFC_CONTFRAC_CUTOFF: + return 0. + x2 = x * x + a = 0. + da = 0. + p = 1. + p_last = 0. + q = da + x2 + q_last = 1. + for i in range(ERFC_CONTFRAC_TERMS): + a += da + da += 2. + b = da + x2 + temp = p + p = b * p - a * p_last + p_last = temp + temp = q + q = b * q - a * q_last + q_last = temp + return p / q * x * math.exp(-x2) / _sqrtpi + +def _erf(x): + if rarithmetic.isnan(x): + return x + absx = abs(x) + if absx < ERF_SERIES_CUTOFF: + return _erf_series(x) + else: + cf = _erfc_contfrac(absx) + return 1. - cf if x > 0. else cf - 1. + +def _erfc(x): + if rarithmetic.isnan(x): + return x + absx = abs(x) + if absx < ERF_SERIES_CUTOFF: + return 1. - _erf_series(x) + else: + cf = _erfc_contfrac(absx) + return cf if x > 0. else 2. - cf From hakanardo at codespeak.net Wed Jul 7 08:36:10 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 7 Jul 2010 08:36:10 +0200 (CEST) Subject: [pypy-svn] r75952 - in pypy/branch/interplevel-array/pypy: jit/codewriter module/array module/array/test Message-ID: <20100707063610.B45EC282BD6@codespeak.net> Author: hakanardo Date: Wed Jul 7 08:36:07 2010 New Revision: 75952 Modified: pypy/branch/interplevel-array/pypy/jit/codewriter/jtransform.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: compilation now raises LLException: Modified: pypy/branch/interplevel-array/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/interplevel-array/pypy/jit/codewriter/jtransform.py Wed Jul 7 08:36:07 2010 @@ -709,8 +709,9 @@ size2, unsigned2 = size_and_sign(op.result.concretetype) if size1 == size2 and unsigned1 == unsigned2: return - raise NotImplementedError("cast not supported yet: %s" % (op, )) - + raise NotImplementedError("cast not supported yet: %s (%s->%s)" % + (op, op.args[0].concretetype, + op.result.concretetype)) # ---------- # Renames, from the _old opname to the _new one. Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Wed Jul 7 08:36:07 2010 @@ -33,16 +33,16 @@ types = { 'c': TypeCode(lltype.Char, 'str_w'), 'u': TypeCode(lltype.UniChar, 'unicode_w'), - 'b': TypeCode(rffi.SIGNEDCHAR, 'int_w', True, True), - 'B': TypeCode(rffi.UCHAR, 'int_w', True), - 'h': TypeCode(rffi.SHORT, 'int_w', True, True), - 'H': TypeCode(rffi.USHORT, 'int_w', True), - 'i': TypeCode(rffi.INT, 'int_w', True, True), - 'I': TypeCode(rffi.UINT, 'int_w', True), - 'l': TypeCode(rffi.LONG, 'int_w', True, True), + #'b': TypeCode(rffi.SIGNEDCHAR, 'int_w', True, True), + #'B': TypeCode(rffi.UCHAR, 'int_w', True), + #'h': TypeCode(rffi.SHORT, 'int_w', True, True), + #'H': TypeCode(rffi.USHORT, 'int_w', True), + #'i': TypeCode(rffi.INT, 'int_w', True, True), + #'I': TypeCode(rffi.UINT, 'int_w', True), + #'l': TypeCode(rffi.LONG, 'int_w', True, True), #'L': TypeCode(rffi.ULONG, 'bigint_w', True), # FIXME: Won't compile - 'f': TypeCode(lltype.SingleFloat, 'float_w'), - 'd': TypeCode(lltype.Float, 'float_w'), + #'f': TypeCode(lltype.SingleFloat, 'float_w'), + #'d': TypeCode(lltype.Float, 'float_w'), } for k, v in types.items(): v.typecode=k unroll_typecodes = unrolling_iterable(types.keys()) @@ -59,7 +59,7 @@ space = self.space unwrap = getattr(space, self.mytype.unwrap) item = unwrap(w_item) - if self.mytype.unwrap == 'bigint_w': + if self.mytype.unwrap == 'bigint_w': try: if self.mytype.signed: item = item.tolonglong() @@ -68,6 +68,11 @@ except (ValueError, OverflowError): msg = 'unsigned %d-byte integer out of range' % self.mytype.bytes raise OperationError(space.w_OverflowError, space.wrap(msg)) + elif self.mytype.unwrap == 'str_w' or self.mytype.unwrap == 'unicode_w': + if len(item) != 1: + msg = 'array item must be char' + raise OperationError(space.w_TypeError, space.wrap(msg)) + item=item[0] if self.mytype.canoverflow: msg = None Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Wed Jul 7 08:36:07 2010 @@ -30,6 +30,7 @@ a = self.array('c') raises(TypeError, a.append, 7) + raises(TypeError, a.append, 'hi') a.append('h') assert a[0] == 'h' assert type(a[0]) is str @@ -37,6 +38,7 @@ a = self.array('u') raises(TypeError, a.append, 7) + raises(TypeError, a.append, u'hi') a.append(unicode('h')) assert a[0] == unicode('h') assert type(a[0]) is unicode From cfbolz at codespeak.net Wed Jul 7 10:25:40 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Jul 2010 10:25:40 +0200 (CEST) Subject: [pypy-svn] r75953 - pypy/branch/reflex-support/pypy/jit/backend/llsupport Message-ID: <20100707082540.3E038282BD6@codespeak.net> Author: cfbolz Date: Wed Jul 7 10:25:38 2010 New Revision: 75953 Modified: pypy/branch/reflex-support/pypy/jit/backend/llsupport/descr.py Log: (arigo, cfbolz): add some sanity asserts to check that C arrays don't need a typeid. don't try to get a typeid for raw arrays. Modified: pypy/branch/reflex-support/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/reflex-support/pypy/jit/backend/llsupport/descr.py Wed Jul 7 10:25:38 2010 @@ -23,10 +23,10 @@ self._cache_call = {} def init_size_descr(self, STRUCT, sizedescr): - pass + assert isinstance(STRUCT, lltype.GcStruct) def init_array_descr(self, ARRAY, arraydescr): - pass + assert isinstance(ARRAY, lltype.GcArray) # ____________________________________________________________ @@ -205,7 +205,8 @@ assert basesize == arraydescr.get_base_size(False) assert itemsize == arraydescr.get_item_size(False) assert ofslength == arraydescr.get_ofs_length(False) - gccache.init_array_descr(ARRAY, arraydescr) + if isinstance(ARRAY, lltype.GcArray): + gccache.init_array_descr(ARRAY, arraydescr) cache[ARRAY] = arraydescr return arraydescr From arigo at codespeak.net Wed Jul 7 10:36:42 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 10:36:42 +0200 (CEST) Subject: [pypy-svn] r75954 - pypy/branch/reflex-support/pypy/jit/backend/llsupport Message-ID: <20100707083642.90A69282BD6@codespeak.net> Author: arigo Date: Wed Jul 7 10:36:41 2010 New Revision: 75954 Modified: pypy/branch/reflex-support/pypy/jit/backend/llsupport/support.py Log: Fix for x86/test/test_basic.test_raw_malloc_and_access Modified: pypy/branch/reflex-support/pypy/jit/backend/llsupport/support.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/backend/llsupport/support.py (original) +++ pypy/branch/reflex-support/pypy/jit/backend/llsupport/support.py Wed Jul 7 10:36:41 2010 @@ -30,7 +30,13 @@ funcobj = get_funcobj(fnptr) if hasattr(funcobj, 'graph'): - llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) + # cache the llinterp; otherwise the remember_malloc/remember_free + # done on the LLInterpreter don't match + try: + llinterp = rtyper._on_top_of_llinterp_llinterp + except AttributeError: + llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) + rtyper._on_top_of_llinterp_llinterp = llinterp def on_top_of_llinterp(*args): real_args = process_args(args) return llinterp.eval_graph(funcobj.graph, real_args) From arigo at codespeak.net Wed Jul 7 10:43:51 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 10:43:51 +0200 (CEST) Subject: [pypy-svn] r75955 - pypy/branch/reflex-support/pypy/jit/backend/llgraph Message-ID: <20100707084351.6E362282BD6@codespeak.net> Author: arigo Date: Wed Jul 7 10:43:49 2010 New Revision: 75955 Modified: pypy/branch/reflex-support/pypy/jit/backend/llgraph/llimpl.py Log: Fix for metainterp/test/test_ztranslation. Modified: pypy/branch/reflex-support/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/reflex-support/pypy/jit/backend/llgraph/llimpl.py Wed Jul 7 10:43:49 2010 @@ -1551,6 +1551,8 @@ setannotation(do_getarrayitem_gc_int, annmodel.SomeInteger()) setannotation(do_getarrayitem_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) setannotation(do_getarrayitem_gc_float, annmodel.SomeFloat()) +setannotation(do_getarrayitem_raw_int, annmodel.SomeInteger()) +setannotation(do_getarrayitem_raw_float, annmodel.SomeFloat()) setannotation(do_getfield_gc_int, annmodel.SomeInteger()) setannotation(do_getfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) setannotation(do_getfield_gc_float, annmodel.SomeFloat()) @@ -1562,6 +1564,8 @@ setannotation(do_setarrayitem_gc_int, annmodel.s_None) setannotation(do_setarrayitem_gc_ptr, annmodel.s_None) setannotation(do_setarrayitem_gc_float, annmodel.s_None) +setannotation(do_setarrayitem_raw_int, annmodel.s_None) +setannotation(do_setarrayitem_raw_float, annmodel.s_None) setannotation(do_setfield_gc_int, annmodel.s_None) setannotation(do_setfield_gc_ptr, annmodel.s_None) setannotation(do_setfield_gc_float, annmodel.s_None) From arigo at codespeak.net Wed Jul 7 10:46:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 10:46:59 +0200 (CEST) Subject: [pypy-svn] r75956 - in pypy/trunk/pypy/jit/backend: llgraph llsupport Message-ID: <20100707084659.9203D282BD6@codespeak.net> Author: arigo Date: Wed Jul 7 10:46:57 2010 New Revision: 75956 Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py pypy/trunk/pypy/jit/backend/llsupport/descr.py pypy/trunk/pypy/jit/backend/llsupport/support.py Log: Merge r75952:75955 from branch/reflex-support: fix failing tests. Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Wed Jul 7 10:46:57 2010 @@ -1549,6 +1549,8 @@ setannotation(do_getarrayitem_gc_int, annmodel.SomeInteger()) setannotation(do_getarrayitem_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) setannotation(do_getarrayitem_gc_float, annmodel.SomeFloat()) +setannotation(do_getarrayitem_raw_int, annmodel.SomeInteger()) +setannotation(do_getarrayitem_raw_float, annmodel.SomeFloat()) setannotation(do_getfield_gc_int, annmodel.SomeInteger()) setannotation(do_getfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) setannotation(do_getfield_gc_float, annmodel.SomeFloat()) @@ -1560,6 +1562,8 @@ setannotation(do_setarrayitem_gc_int, annmodel.s_None) setannotation(do_setarrayitem_gc_ptr, annmodel.s_None) setannotation(do_setarrayitem_gc_float, annmodel.s_None) +setannotation(do_setarrayitem_raw_int, annmodel.s_None) +setannotation(do_setarrayitem_raw_float, annmodel.s_None) setannotation(do_setfield_gc_int, annmodel.s_None) setannotation(do_setfield_gc_ptr, annmodel.s_None) setannotation(do_setfield_gc_float, annmodel.s_None) Modified: pypy/trunk/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/descr.py Wed Jul 7 10:46:57 2010 @@ -23,10 +23,10 @@ self._cache_call = {} def init_size_descr(self, STRUCT, sizedescr): - pass + assert isinstance(STRUCT, lltype.GcStruct) def init_array_descr(self, ARRAY, arraydescr): - pass + assert isinstance(ARRAY, lltype.GcArray) # ____________________________________________________________ @@ -205,7 +205,8 @@ assert basesize == arraydescr.get_base_size(False) assert itemsize == arraydescr.get_item_size(False) assert ofslength == arraydescr.get_ofs_length(False) - gccache.init_array_descr(ARRAY, arraydescr) + if isinstance(ARRAY, lltype.GcArray): + gccache.init_array_descr(ARRAY, arraydescr) cache[ARRAY] = arraydescr return arraydescr Modified: pypy/trunk/pypy/jit/backend/llsupport/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/support.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/support.py Wed Jul 7 10:46:57 2010 @@ -30,7 +30,13 @@ funcobj = get_funcobj(fnptr) if hasattr(funcobj, 'graph'): - llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) + # cache the llinterp; otherwise the remember_malloc/remember_free + # done on the LLInterpreter don't match + try: + llinterp = rtyper._on_top_of_llinterp_llinterp + except AttributeError: + llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) + rtyper._on_top_of_llinterp_llinterp = llinterp def on_top_of_llinterp(*args): real_args = process_args(args) return llinterp.eval_graph(funcobj.graph, real_args) From arigo at codespeak.net Wed Jul 7 11:21:40 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 11:21:40 +0200 (CEST) Subject: [pypy-svn] r75957 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100707092140.96376282BD6@codespeak.net> Author: arigo Date: Wed Jul 7 11:21:39 2010 New Revision: 75957 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_zexternal.py Log: Fix fast_search(), and test search extensively. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Wed Jul 7 11:21:39 2010 @@ -81,8 +81,11 @@ def span(self, groupnum=0): # compatibility - lst = self.flatten_marks()[groupnum*2:groupnum*2+2] - return tuple(lst) or (-1, -1) + fmarks = self.flatten_marks() + groupnum *= 2 + if groupnum >= len(fmarks): + return (-1, -1) + return (fmarks[groupnum], fmarks[groupnum+1]) def group(self, group=0): # compatibility @@ -757,10 +760,14 @@ i += 1 if i == prefix_len: # found a potential match - if flags & rsre_char.SRE_INFO_LITERAL: - return True # matched all of pure literal pattern start = string_position + 1 - prefix_len ptr = start + prefix_skip + if flags & rsre_char.SRE_INFO_LITERAL: + # matched all of pure literal pattern + ctx.match_start = start + ctx.match_end = ptr + ctx.match_marks = None + return ctx ppos = pattern_offset + 2 * prefix_skip result = sre_match(ctx, ppos, ptr, None) if result is not None: Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_zexternal.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_zexternal.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_zexternal.py Wed Jul 7 11:21:39 2010 @@ -3,12 +3,17 @@ from pypy.rlib.rsre import rsre -def test_external(): +def test_external_match(): from pypy.rlib.rsre.test.re_tests import tests for t in tests: - yield run_external, t + yield run_external, t, False -def run_external(t): +def test_external_search(): + from pypy.rlib.rsre.test.re_tests import tests + for t in tests: + yield run_external, t, True + +def run_external(t, use_search): from pypy.rlib.rsre.test.re_tests import SUCCEED, FAIL, SYNTAX_ERROR pattern, s, outcome = t[:3] if len(t) == 5: @@ -25,11 +30,14 @@ if outcome == SYNTAX_ERROR: raise Exception("this should have been a syntax error") # - # Emulate a poor man's search() with repeated match()s - for i in range(len(s)+1): - result = rsre.match(obj, s, start=i) - if result: - break + if use_search: + result = rsre.search(obj, s) + else: + # Emulate a poor man's search() with repeated match()s + for i in range(len(s)+1): + result = rsre.match(obj, s, start=i) + if result: + break # if outcome == FAIL: if result is not None: From arigo at codespeak.net Wed Jul 7 11:24:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 11:24:34 +0200 (CEST) Subject: [pypy-svn] r75958 - pypy/branch/rsre2/pypy/rlib Message-ID: <20100707092434.195D7282BD6@codespeak.net> Author: arigo Date: Wed Jul 7 11:24:32 2010 New Revision: 75958 Modified: pypy/branch/rsre2/pypy/rlib/debug.py Log: Rewrite check_nonneg() to give better positioning information when it fails. Modified: pypy/branch/rsre2/pypy/rlib/debug.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/debug.py (original) +++ pypy/branch/rsre2/pypy/rlib/debug.py Wed Jul 7 11:24:32 2010 @@ -255,11 +255,20 @@ class IntegerCanBeNegative(Exception): pass -def _check_nonneg(ann, bk): - from pypy.annotation.model import SomeInteger - s_nonneg = SomeInteger(nonneg=True) - if not s_nonneg.contains(ann): - raise IntegerCanBeNegative - def check_nonneg(x): - check_annotation(x, _check_nonneg) + assert x >= 0 + return x + +class Entry(ExtRegistryEntry): + _about_ = check_nonneg + + def compute_result_annotation(self, s_arg): + from pypy.annotation.model import SomeInteger + s_nonneg = SomeInteger(nonneg=True) + if not s_nonneg.contains(s_arg): + raise IntegerCanBeNegative + return s_arg + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputarg(hop.args_r[0], arg=0) From arigo at codespeak.net Wed Jul 7 11:29:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 11:29:46 +0200 (CEST) Subject: [pypy-svn] r75959 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100707092946.84417282BD6@codespeak.net> Author: arigo Date: Wed Jul 7 11:29:43 2010 New Revision: 75959 Added: pypy/branch/rsre2/pypy/rlib/rsre/test/targetrsre.py - copied, changed from r75953, pypy/trunk/pypy/rlib/rsre/test/targetrsre.py Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Log: Copy targetrsre.py from trunk. Translation fixes. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Wed Jul 7 11:29:43 2010 @@ -43,6 +43,7 @@ match_marks_flat = None def __init__(self, pattern, string, match_start, flags): + assert match_start >= 0 self.pattern = pattern self.string = string self.end = len(string) @@ -761,6 +762,7 @@ if i == prefix_len: # found a potential match start = string_position + 1 - prefix_len + assert start >= 0 ptr = start + prefix_skip if flags & rsre_char.SRE_INFO_LITERAL: # matched all of pure literal pattern Copied: pypy/branch/rsre2/pypy/rlib/rsre/test/targetrsre.py (from r75953, pypy/trunk/pypy/rlib/rsre/test/targetrsre.py) ============================================================================== --- pypy/trunk/pypy/rlib/rsre/test/targetrsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/targetrsre.py Wed Jul 7 11:29:43 2010 @@ -26,15 +26,13 @@ data = read(filename) p = 0 while True: - state = rsre.SimpleStringState(data, p) - res = state.search(r_code1) - if not res: + res = rsre.search(r_code1, data, p) + if res is None: break - groups = state.create_regs(1) - matchstart, matchstop = groups[1] + matchstart, matchstop = res.span(1) assert 0 <= matchstart <= matchstop print '%s: %s' % (filename, data[matchstart:matchstop]) - p = groups[0][1] + p = res.span(0)[1] # __________ Entry point __________ From wlav at codespeak.net Wed Jul 7 11:44:58 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Wed, 7 Jul 2010 11:44:58 +0200 (CEST) Subject: [pypy-svn] r75960 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100707094458.3ADBF282BE0@codespeak.net> Author: wlav Date: Wed Jul 7 11:44:56 2010 New Revision: 75960 Added: pypy/branch/reflex-support/pypy/module/cppyy/capi.py (contents, props changed) pypy/branch/reflex-support/pypy/module/cppyy/executor.py (contents, props changed) Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Log: (cfbolz, wlav) Make use of the new executor module and moved all Reflex-C interface code to capi.py. Added: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Wed Jul 7 11:44:56 2010 @@ -0,0 +1,82 @@ +import py, os + +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rpython.lltypesystem import rffi, lltype + +srcpath = py.path.local(__file__).dirpath().join("src") +incpath = py.path.local(__file__).dirpath().join("include") + +try: + rootincpath = os.path.join(os.environ["ROOTSYS"], "include") + rootlibpath = os.path.join(os.environ["ROOTSYS"], "lib") +except KeyError: + print 'please set ROOTSYS envar to the location of the ROOT installation' + raise + +eci = ExternalCompilationInfo( + separate_module_files=[srcpath.join("reflexcwrapper.cxx")], + include_dirs=[incpath, rootincpath], + library_dirs=[rootlibpath], + libraries=["Reflex"], + use_cpp_linker=True, +) + +c_callstatic_l = rffi.llexternal( + "callstatic_l", + [rffi.CCHARP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.LONG, + compilation_info=eci) +c_callstatic_d = rffi.llexternal( + "callstatic_d", + [rffi.CCHARP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.DOUBLE, + compilation_info=eci) +c_construct = rffi.llexternal( + "construct", + [rffi.CCHARP, rffi.INT, rffi.VOIDPP], rffi.VOIDP, + compilation_info=eci) +c_callmethod_l = rffi.llexternal( + "callmethod_l", + [rffi.CCHARP, rffi.INT, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.LONG, + compilation_info=eci) +c_destruct = rffi.llexternal( + "destruct", + [rffi.CCHARP, rffi.VOIDP], lltype.Void, + compilation_info=eci) + + +c_num_methods = rffi.llexternal( + "num_methods", + [rffi.CCHARP], rffi.INT, + compilation_info=eci) +c_method_name = rffi.llexternal( + "method_name", + [rffi.CCHARP, rffi.INT], rffi.CCHARP, + compilation_info=eci) +c_result_type_method = rffi.llexternal( + "result_type_method", + [rffi.CCHARP, rffi.INT], rffi.CCHARP, + compilation_info=eci) +c_num_args_method = rffi.llexternal( + "num_args_method", + [rffi.CCHARP, rffi.INT], rffi.INT, + compilation_info=eci) +c_arg_type_method = rffi.llexternal( + "arg_type_method", + [rffi.CCHARP, rffi.INT, rffi.INT], rffi.CCHARP, + compilation_info=eci) +c_is_constructor = rffi.llexternal( + "is_constructor", + [rffi.CCHARP, rffi.INT], rffi.INT, + compilation_info=eci) +c_is_static = rffi.llexternal( + "is_static", + [rffi.CCHARP, rffi.INT], rffi.INT, + compilation_info=eci) +c_myfree = rffi.llexternal( + "myfree", + [rffi.VOIDP], lltype.Void, + compilation_info=eci) + +def charp2str_free(charp): + string = rffi.charp2str(charp) + c_myfree(charp) + return string Added: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Wed Jul 7 11:44:56 2010 @@ -0,0 +1,43 @@ +import pypy.module.cppyy.capi as capi + +from pypy.rpython.lltypesystem import rffi, lltype + +_executors = {} + + +class FunctionExecutor(object): + def execute(self, space, func, num_args, args): + raise NotImplementedError("abstract base class") + + +class LongExecutor(FunctionExecutor): + def execute(self, space, func, num_args, args): + result = capi.c_callstatic_l(func.cpptype.name, func.method_index, num_args, args) + return space.wrap(result) + +class DoubleExecutor(FunctionExecutor): + def execute(self, space, func, num_args, args): + result = capi.c_callstatic_d(func.cpptype.name, func.method_index, num_args, args) + return space.wrap(result) + +class CStringExecutor(FunctionExecutor): + def execute(self, space, func, num_args, args): + lresult = capi.c_callstatic_l(func.cpptype.name, func.method_index, num_args, args) + ccpresult = rffi.cast(rffi.CCHARP, lresult) + result = capi.charp2str_free(ccpresult) + return space.wrap(result) + +def get_executor(name): + try: + return _executors[name] + except KeyError: + pass + + return None # currently used until proper lazy instantiation available in interp_cppyy + + # raise TypeError("no clue what %s is" % name) + +_executors["int"] = LongExecutor() +_executors["long"] = LongExecutor() +_executors["double"] = DoubleExecutor() +_executors["char*"] = CStringExecutor() Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Wed Jul 7 11:44:56 2010 @@ -1,86 +1,16 @@ -import py, os +import pypy.module.cppyy.capi as capi from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, interp2app from pypy.interpreter.typedef import TypeDef from pypy.interpreter.baseobjspace import Wrappable -from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.lltypesystem import rffi, lltype from pypy.rlib.libffi import CDLL from pypy.rlib import jit, debug -from pypy.module.cppyy import converter - - -srcpath = py.path.local(__file__).dirpath().join("src") -incpath = py.path.local(__file__).dirpath().join("include") -rootincpath = os.path.join(os.environ["ROOTSYS"], "include") -rootlibpath = os.path.join(os.environ["ROOTSYS"], "lib") - -eci = ExternalCompilationInfo( - separate_module_files=[srcpath.join("reflexcwrapper.cxx")], - include_dirs=[incpath, rootincpath], - library_dirs=[rootlibpath], - libraries=["Reflex"], - use_cpp_linker=True, -) - -c_callstatic_l = rffi.llexternal( - "callstatic_l", - [rffi.CCHARP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.LONG, - compilation_info=eci) -c_callstatic_d = rffi.llexternal( - "callstatic_d", - [rffi.CCHARP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.DOUBLE, - compilation_info=eci) -c_construct = rffi.llexternal( - "construct", - [rffi.CCHARP, rffi.INT, rffi.VOIDPP], rffi.VOIDP, - compilation_info=eci) -c_callmethod_l = rffi.llexternal( - "callmethod_l", - [rffi.CCHARP, rffi.INT, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.LONG, - compilation_info=eci) -c_destruct = rffi.llexternal( - "destruct", - [rffi.CCHARP, rffi.VOIDP], lltype.Void, - compilation_info=eci) - - -c_num_methods = rffi.llexternal( - "num_methods", - [rffi.CCHARP], rffi.INT, - compilation_info=eci) -c_method_name = rffi.llexternal( - "method_name", - [rffi.CCHARP, rffi.INT], rffi.CCHARP, - compilation_info=eci) -c_result_type_method = rffi.llexternal( - "result_type_method", - [rffi.CCHARP, rffi.INT], rffi.CCHARP, - compilation_info=eci) -c_num_args_method = rffi.llexternal( - "num_args_method", - [rffi.CCHARP, rffi.INT], rffi.INT, - compilation_info=eci) -c_arg_type_method = rffi.llexternal( - "arg_type_method", - [rffi.CCHARP, rffi.INT, rffi.INT], rffi.CCHARP, - compilation_info=eci) -c_is_constructor = rffi.llexternal( - "is_constructor", - [rffi.CCHARP, rffi.INT], rffi.INT, - compilation_info=eci) -c_is_static = rffi.llexternal( - "is_static", - [rffi.CCHARP, rffi.INT], rffi.INT, - compilation_info=eci) -c_myfree = rffi.llexternal( - "myfree", - [rffi.VOIDP], lltype.Void, - compilation_info=eci) +from pypy.module.cppyy import converter, executor NULL_VOIDP = lltype.nullptr(rffi.VOIDP.TO) @@ -113,13 +43,13 @@ self.cpptype = cpptype self.space = cpptype.space self.method_index = method_index - self.result_type = result_type self.arg_types = arg_types + self.executor = executor.get_executor( result_type ) self.arg_converters = None def call(self, cppthis, args_w): args = self.prepare_arguments(args_w) - result = c_callmethod_l(self.cpptype.name, self.method_index, + result = capi.c_callmethod_l(self.cpptype.name, self.method_index, cppthis, len(args_w), args) self.free_arguments(args) return self.space.wrap(result) @@ -159,27 +89,18 @@ lltype.free(args, flavor='raw') def __repr__(self): - return "CPPFunction(%s, %s, %s, %s)" % ( - self.cpptype, self.method_index, self.result_type, self.arg_types) + return "CPPFunction(%s, %s, %r, %s)" % ( + self.cpptype, self.method_index, self.executor, self.arg_types) class CPPFunction(CPPMethod): def call(self, cppthis, args_w): + if self.executor is None: + raise OperationError(self.space.w_TypeError, self.space.wrap("return type not handled")) + assert not cppthis args = self.prepare_arguments(args_w) try: - if self.result_type == "int": - result = c_callstatic_l(self.cpptype.name, self.method_index, len(args_w), args) - return self.space.wrap(result) - if self.result_type == "double": - result = c_callstatic_d(self.cpptype.name, self.method_index, len(args_w), args) - return self.space.wrap(result) - if self.result_type == "char*": - lresult = c_callstatic_l(self.cpptype.name, self.method_index, len(args_w), args) - ccpresult = rffi.cast(rffi.CCHARP, lresult) - result = charp2str_free(ccpresult) - return self.space.wrap(result) - else: - raise NotImplementedError + return self.executor.execute(self.space, self, len(args_w), args) finally: self.free_arguments(args) @@ -188,7 +109,7 @@ def call(self, cppthis, args_w): assert not cppthis args = self.prepare_arguments(args_w) - result = c_construct(self.cpptype.name, len(args_w), args) + result = capi.c_construct(self.cpptype.name, len(args_w), args) self.free_arguments(args) return W_CPPObject(self.cpptype, result) @@ -219,10 +140,6 @@ def __repr__(self): return "CPPOverload(%s, %s)" % (self.func_name, self.functions) -def charp2str_free(charp): - string = rffi.charp2str(charp) - c_myfree(charp) - return string class W_CPPType(Wrappable): _immutable_fields_ = ["cpplib", "name"] @@ -235,10 +152,10 @@ self._find_func_members() def _find_func_members(self): - num_func_members = c_num_methods(self.name) + num_func_members = capi.c_num_methods(self.name) args_temp = {} for i in range(num_func_members): - func_member_name = charp2str_free(c_method_name(self.name, i)) + func_member_name = capi.charp2str_free(capi.c_method_name(self.name, i)) cppfunction = self._make_cppfunction(i) overload = args_temp.setdefault(func_member_name, []) overload.append(cppfunction) @@ -247,15 +164,15 @@ self.function_members[name] = overload def _make_cppfunction(self, method_index): - result_type = charp2str_free(c_result_type_method(self.name, method_index)) - num_args = c_num_args_method(self.name, method_index) + result_type = capi.charp2str_free(capi.c_result_type_method(self.name, method_index)) + num_args = capi.c_num_args_method(self.name, method_index) argtypes = [] for i in range(num_args): - argtype = charp2str_free(c_arg_type_method(self.name, method_index, i)) + argtype = capi.charp2str_free(capi.c_arg_type_method(self.name, method_index, i)) argtypes.append(argtype) - if c_is_constructor(self.name, method_index): + if capi.c_is_constructor(self.name, method_index): cls = CPPConstructor - elif c_is_static(self.name, method_index): + elif capi.c_is_static(self.name, method_index): cls = CPPFunction else: cls = CPPMethod @@ -292,7 +209,7 @@ return overload.call(self.rawobject, args_w) def destruct(self): - c_destruct(self.cppclass.name, self.rawobject) + capi.c_destruct(self.cppclass.name, self.rawobject) W_CPPObject.typedef = TypeDef( 'CPPObject', Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Wed Jul 7 11:44:56 2010 @@ -1,7 +1,8 @@ import py import os from pypy.conftest import gettestobjspace -from pypy.module.cppyy import interp_cppyy +from pypy.module.cppyy import interp_cppyy, executor + currpath = py.path.local(__file__).dirpath() shared_lib = str(currpath.join("example01Dict.so")) @@ -17,7 +18,7 @@ w_cppyyclass = lib.type_byname("example01") adddouble = w_cppyyclass.function_members["adddouble"] func, = adddouble.functions - assert func.result_type == "double" + assert isinstance(func.executor, executor.DoubleExecutor) assert func.arg_types == ["double"] From arigo at codespeak.net Wed Jul 7 11:47:56 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 11:47:56 +0200 (CEST) Subject: [pypy-svn] r75961 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100707094756.52909282BE0@codespeak.net> Author: arigo Date: Wed Jul 7 11:47:53 2010 New Revision: 75961 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py (props changed) pypy/branch/reflex-support/pypy/module/cppyy/executor.py (props changed) Log: Remove svn:executable. From arigo at codespeak.net Wed Jul 7 12:16:50 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 12:16:50 +0200 (CEST) Subject: [pypy-svn] r75962 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100707101650.42070282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 12:16:48 2010 New Revision: 75962 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py Log: Tentative fix of translation. Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Wed Jul 7 12:16:48 2010 @@ -16,6 +16,7 @@ eci = ExternalCompilationInfo( separate_module_files=[srcpath.join("reflexcwrapper.cxx")], include_dirs=[incpath, rootincpath], + includes=["reflexcwrapper.h"], library_dirs=[rootlibpath], libraries=["Reflex"], use_cpp_linker=True, From cfbolz at codespeak.net Wed Jul 7 12:44:07 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Jul 2010 12:44:07 +0200 (CEST) Subject: [pypy-svn] r75963 - pypy/branch/reflex-support/pypy/module/cppyy/include Message-ID: <20100707104407.D4482282BF4@codespeak.net> Author: cfbolz Date: Wed Jul 7 12:44:06 2010 New Revision: 75963 Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Log: (cfbolz, wlav, arigo): make this work in C Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Wed Jul 7 12:44:06 2010 @@ -2,7 +2,9 @@ #ifndef CPPYY_REFLEXCWRAPPER #define CPPYY_REFLEXCWRAPPER +#ifdef __cplusplus extern "C" { +#endif // ifdef __cplusplus long callstatic_l(const char* class_name, int method_index, int numargs, void* args[]); double callstatic_d(const char* class_name, int method_index, int numargs, void* args[]); long callmethod_l(const char* class_name, int method_index, void* self, int numargs, void* args[]); @@ -18,6 +20,8 @@ int is_static(const char* class_name, int method_index); void myfree(void* ptr); +#ifdef __cplusplus } +#endif // ifdef __cplusplus #endif // ifndef CPPYY_REFLEXCWRAPPER From wlav at codespeak.net Wed Jul 7 12:46:52 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Wed, 7 Jul 2010 12:46:52 +0200 (CEST) Subject: [pypy-svn] r75964 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100707104652.AACC7282BD6@codespeak.net> Author: wlav Date: Wed Jul 7 12:46:51 2010 New Revision: 75964 Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Log: Made method call also use executors. Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Wed Jul 7 12:46:51 2010 @@ -11,18 +11,27 @@ class LongExecutor(FunctionExecutor): - def execute(self, space, func, num_args, args): - result = capi.c_callstatic_l(func.cpptype.name, func.method_index, num_args, args) + def execute(self, space, func, cppthis, num_args, args): + if cppthis is not None: + result = capi.c_callmethod_l(func.cpptype.name, func.method_index, cppthis, num_args, args) + else: + result = capi.c_callstatic_l(func.cpptype.name, func.method_index, num_args, args) return space.wrap(result) class DoubleExecutor(FunctionExecutor): - def execute(self, space, func, num_args, args): - result = capi.c_callstatic_d(func.cpptype.name, func.method_index, num_args, args) + def execute(self, space, func, cppthis, num_args, args): + if cppthis is not None: + raise NotImplementedError + else: + result = capi.c_callstatic_d(func.cpptype.name, func.method_index, num_args, args) return space.wrap(result) class CStringExecutor(FunctionExecutor): - def execute(self, space, func, num_args, args): - lresult = capi.c_callstatic_l(func.cpptype.name, func.method_index, num_args, args) + def execute(self, space, func, cppthis, num_args, args): + if cppthis is not None: + raise NotImplementedError + else: + lresult = capi.c_callstatic_l(func.cpptype.name, func.method_index, num_args, args) ccpresult = rffi.cast(rffi.CCHARP, lresult) result = capi.charp2str_free(ccpresult) return space.wrap(result) Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Wed Jul 7 12:46:51 2010 @@ -48,11 +48,14 @@ self.arg_converters = None def call(self, cppthis, args_w): + if self.executor is None: + raise OperationError(self.space.w_TypeError, self.space.wrap("return type not handled")) + args = self.prepare_arguments(args_w) - result = capi.c_callmethod_l(self.cpptype.name, self.method_index, - cppthis, len(args_w), args) - self.free_arguments(args) - return self.space.wrap(result) + try: + return self.executor.execute(self.space, self, cppthis, len(args_w), args) + finally: + self.free_arguments(args) def _build_converters(self): self.arg_converters = [converter.get_converter(arg_type) @@ -100,7 +103,7 @@ assert not cppthis args = self.prepare_arguments(args_w) try: - return self.executor.execute(self.space, self, len(args_w), args) + return self.executor.execute(self.space, self, None, len(args_w), args) finally: self.free_arguments(args) From fijal at codespeak.net Wed Jul 7 13:25:13 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Jul 2010 13:25:13 +0200 (CEST) Subject: [pypy-svn] r75965 - pypy/benchmarks/own/twisted Message-ID: <20100707112513.9FF12282B9E@codespeak.net> Author: fijal Date: Wed Jul 7 13:25:12 2010 New Revision: 75965 Modified: pypy/benchmarks/own/twisted/benchlib.py Log: an attempt to sleep in between benchmark runs as well Modified: pypy/benchmarks/own/twisted/benchlib.py ============================================================================== --- pypy/benchmarks/own/twisted/benchlib.py (original) +++ pypy/benchmarks/own/twisted/benchlib.py Wed Jul 7 13:25:12 2010 @@ -86,6 +86,7 @@ def multidriver(*f): jobs = iter(f) def work(): + sleep_to_purge_connexions() for job in jobs: d = setup_driver(job, sys.argv, reactor) d.addCallback(lambda ignored: work()) @@ -93,7 +94,6 @@ reactor.stop() reactor.callWhenRunning(work) reactor.run() - sleep_to_purge_connexions() _interface = 1 def rotate_local_intf(): From getxsick at codespeak.net Wed Jul 7 13:29:30 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Wed, 7 Jul 2010 13:29:30 +0200 (CEST) Subject: [pypy-svn] r75966 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100707112930.3995F282B9E@codespeak.net> Author: getxsick Date: Wed Jul 7 13:29:28 2010 New Revision: 75966 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: add cache for compiled loops to prevent recompiling the same type of function it needs some refactor though Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Wed Jul 7 13:29:28 2010 @@ -6,6 +6,8 @@ from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.typesystem import deref +cache = [] # XXX global! + class CDLL(object): def __init__(self, name, load=True): if load: @@ -37,56 +39,75 @@ self.cpu = cpu self.lib = lib.handler self.setup_stack() - - if self.res_type == 'i': - self.bres = BoxInt() - res = lltype.Signed - elif self.res_type == 'f': - self.bres = BoxFloat() - res = lltype.Float - elif self.res_type == 'p': - self.bres = BoxPtr() - res = lltype.Signed - elif self.res_type == 'v': - self.bres = NULLBOX - res = lltype.Void - else: - raise ValueError(self.res_type) + self.looptoken = None try: - addr = rffi.cast(lltype.Signed, rdynload.dlsym(self.lib, func)) + self.funcaddr = rffi.cast(lltype.Signed, rdynload.dlsym(self.lib, + func)) except KeyError: raise ValueError("Cannot find symbol %s", func) - self.bfuncaddr = BoxInt(addr) + self.bargs.append(BoxInt()) - args = [] - for arg in self.args_type: - if arg == 'i': - args.append(lltype.Signed) - elif arg == 'f': - args.append(lltype.Float) - elif arg == 'p': - args.append(lltype.Signed) + # check if it's not already compiled + for func in cache: + if self.args_type == func.args_type and \ + self.res_type == func.res_type: + self.looptoken = func.looptoken + self.oplist = func.oplist + break + + if self.looptoken is None: + args = [] + for arg in self.args_type: + if arg == 'i': + self.bargs.append(BoxInt()) + args.append(lltype.Signed) + elif arg == 'f': + self.bargs.append(BoxFloat()) + args.append(lltype.Float) + elif arg == 'p': + self.bargs.append(BoxPtr()) + args.append(lltype.Signed) + else: + raise ValueError(arg) + + if self.res_type == 'i': + self.bres = BoxInt() + res = lltype.Signed + elif self.res_type == 'f': + self.bres = BoxFloat() + res = lltype.Float + elif self.res_type == 'p': + self.bres = BoxPtr() + res = lltype.Signed + elif self.res_type == 'v': + self.bres = NULLBOX + res = lltype.Void else: - raise ValueError(arg) + raise ValueError(self.res_type) - FPTR = lltype.Ptr(lltype.FuncType(args, res)) - FUNC = deref(FPTR) - self.calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + FPTR = lltype.Ptr(lltype.FuncType(args, res)) + FUNC = deref(FPTR) + self.calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + + self.looptoken = LoopToken() + inputargs = self.bargs # [ self.bfuncaddr ] + self.bargs + + self.oplist = [ResOperation(rop.CALL, inputargs, self.bres, + descr=self.calldescr), + ResOperation(rop.FINISH, [self.bres], None, + descr=BasicFailDescr(0))] + self.cpu.compile_loop(inputargs, self.oplist, self.looptoken) + + # add to the cache + cache.append(_Func(self.args_type, self.res_type, self.looptoken, + self.oplist)) def call(self): - inputargs = [self.bfuncaddr] + self.bargs - - oplist = [ResOperation(rop.CALL, inputargs, self.bres, - descr=self.calldescr), - ResOperation(rop.FINISH, [self.bres], None, - descr=BasicFailDescr(0))] - looptoken = LoopToken() - self.cpu.compile_loop(inputargs, oplist, looptoken) - self.cpu.set_future_value_int(0, self.bfuncaddr.getint()) + self.push_funcaddr(self.funcaddr) - res = self.cpu.execute_token(looptoken) - if res is oplist[-1].descr: + res = self.cpu.execute_token(self.looptoken) + if res is self.oplist[-1].descr: self.guard_failed = False else: self.guard_failed = True @@ -107,19 +128,27 @@ def setup_stack(self): self.bargs = [] - self.esp = 1 # 0 is a func addr + self.esp = 1 # 0 is funcaddr + + def push_funcaddr(self, value): + self.cpu.set_future_value_int(0, value) + self.esp += 1 def push_int(self, value): self.cpu.set_future_value_int(self.esp, value) - self.bargs.append(BoxInt(value)) self.esp += 1 def push_float(self, value): self.cpu.set_future_value_float(self.esp, value) - self.bargs.append(BoxFloat(value)) self.esp += 1 def push_ref(self, value): self.cpu.set_future_value_ref(self.esp, value) - self.bargs.append(BoxPtr(value)) self.esp += 1 + +class _Func(object): + def __init__(self, args_type, res_type, looptoken, oplist): + self.args_type = args_type + self.res_type = res_type + self.looptoken = looptoken + self.oplist = oplist From getxsick at codespeak.net Wed Jul 7 13:57:20 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Wed, 7 Jul 2010 13:57:20 +0200 (CEST) Subject: [pypy-svn] r75967 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100707115720.2C434282B9E@codespeak.net> Author: getxsick Date: Wed Jul 7 13:57:18 2010 New Revision: 75967 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: clean up a bit Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Wed Jul 7 13:57:18 2010 @@ -37,52 +37,51 @@ self.args_type = args_type self.res_type = res_type self.cpu = cpu - self.lib = lib.handler - self.setup_stack() self.looptoken = None + lib = lib.handler + bargs = [] + self.setup_stack() try: - self.funcaddr = rffi.cast(lltype.Signed, rdynload.dlsym(self.lib, - func)) + self.funcaddr = rffi.cast(lltype.Signed, rdynload.dlsym(lib, func)) except KeyError: raise ValueError("Cannot find symbol %s", func) - self.bargs.append(BoxInt()) + bargs.append(BoxInt()) # check if it's not already compiled for func in cache: if self.args_type == func.args_type and \ self.res_type == func.res_type: self.looptoken = func.looptoken - self.oplist = func.oplist break if self.looptoken is None: args = [] for arg in self.args_type: if arg == 'i': - self.bargs.append(BoxInt()) + bargs.append(BoxInt()) args.append(lltype.Signed) elif arg == 'f': - self.bargs.append(BoxFloat()) + bargs.append(BoxFloat()) args.append(lltype.Float) elif arg == 'p': - self.bargs.append(BoxPtr()) + bargs.append(BoxPtr()) args.append(lltype.Signed) else: raise ValueError(arg) if self.res_type == 'i': - self.bres = BoxInt() res = lltype.Signed + bres = BoxInt() elif self.res_type == 'f': - self.bres = BoxFloat() res = lltype.Float + bres = BoxFloat() elif self.res_type == 'p': - self.bres = BoxPtr() res = lltype.Signed + bres = BoxPtr() elif self.res_type == 'v': - self.bres = NULLBOX res = lltype.Void + bres = NULLBOX else: raise ValueError(self.res_type) @@ -91,43 +90,34 @@ self.calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) self.looptoken = LoopToken() - inputargs = self.bargs # [ self.bfuncaddr ] + self.bargs - - self.oplist = [ResOperation(rop.CALL, inputargs, self.bres, + self.oplist = [ResOperation(rop.CALL, bargs, bres, descr=self.calldescr), - ResOperation(rop.FINISH, [self.bres], None, + ResOperation(rop.FINISH, [bres], None, descr=BasicFailDescr(0))] - self.cpu.compile_loop(inputargs, self.oplist, self.looptoken) + self.cpu.compile_loop(bargs, self.oplist, self.looptoken) # add to the cache - cache.append(_Func(self.args_type, self.res_type, self.looptoken, - self.oplist)) + cache.append(_Func(self.args_type, self.res_type, self.looptoken)) def call(self): self.push_funcaddr(self.funcaddr) - res = self.cpu.execute_token(self.looptoken) - if res is self.oplist[-1].descr: - self.guard_failed = False - else: - self.guard_failed = True self.setup_stack() # clean up the stack if self.res_type == 'i': - r = BoxInt(self.cpu.get_latest_value_int(0)).getint() + r = self.cpu.get_latest_value_int(0) elif self.res_type == 'f': - r = BoxFloat(self.cpu.get_latest_value_float(0)).getfloat() + r = self.cpu.get_latest_value_float(0) elif self.res_type == 'p': - r = BoxPtr(self.cpu.get_latest_value_ref(0)).getref() + r = self.cpu.get_latest_value_ref(0) elif self.res_type == 'v': r = None else: raise ValueError(self.res_type) - return r + return r # XXX can't return various types def setup_stack(self): - self.bargs = [] self.esp = 1 # 0 is funcaddr def push_funcaddr(self, value): @@ -147,8 +137,7 @@ self.esp += 1 class _Func(object): - def __init__(self, args_type, res_type, looptoken, oplist): + def __init__(self, args_type, res_type, looptoken): self.args_type = args_type self.res_type = res_type self.looptoken = looptoken - self.oplist = oplist From getxsick at codespeak.net Wed Jul 7 14:02:01 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Wed, 7 Jul 2010 14:02:01 +0200 (CEST) Subject: [pypy-svn] r75968 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100707120201.48463282B9E@codespeak.net> Author: getxsick Date: Wed Jul 7 14:01:59 2010 New Revision: 75968 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: don't encrease stack pointer when pushing func addr Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Wed Jul 7 14:01:59 2010 @@ -122,7 +122,7 @@ def push_funcaddr(self, value): self.cpu.set_future_value_int(0, value) - self.esp += 1 + #self.esp += 1 def push_int(self, value): self.cpu.set_future_value_int(self.esp, value) From getxsick at codespeak.net Wed Jul 7 14:15:05 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Wed, 7 Jul 2010 14:15:05 +0200 (CEST) Subject: [pypy-svn] r75969 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100707121505.4EC8D282B9E@codespeak.net> Author: getxsick Date: Wed Jul 7 14:15:03 2010 New Revision: 75969 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: keep as a local variable Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Wed Jul 7 14:15:03 2010 @@ -87,11 +87,10 @@ FPTR = lltype.Ptr(lltype.FuncType(args, res)) FUNC = deref(FPTR) - self.calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) self.looptoken = LoopToken() - self.oplist = [ResOperation(rop.CALL, bargs, bres, - descr=self.calldescr), + self.oplist = [ResOperation(rop.CALL, bargs, bres, descr=calldescr), ResOperation(rop.FINISH, [bres], None, descr=BasicFailDescr(0))] self.cpu.compile_loop(bargs, self.oplist, self.looptoken) From getxsick at codespeak.net Wed Jul 7 14:28:28 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Wed, 7 Jul 2010 14:28:28 +0200 (CEST) Subject: [pypy-svn] r75970 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100707122828.D7D12282B9E@codespeak.net> Author: getxsick Date: Wed Jul 7 14:28:25 2010 New Revision: 75970 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: better handling of the stack. Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Wed Jul 7 14:28:25 2010 @@ -40,7 +40,6 @@ self.looptoken = None lib = lib.handler bargs = [] - self.setup_stack() try: self.funcaddr = rffi.cast(lltype.Signed, rdynload.dlsym(lib, func)) @@ -97,13 +96,11 @@ # add to the cache cache.append(_Func(self.args_type, self.res_type, self.looptoken)) + self.setup_stack() def call(self): - self.push_funcaddr(self.funcaddr) res = self.cpu.execute_token(self.looptoken) - self.setup_stack() # clean up the stack - if self.res_type == 'i': r = self.cpu.get_latest_value_int(0) elif self.res_type == 'f': @@ -114,14 +111,17 @@ r = None else: raise ValueError(self.res_type) + + self.setup_stack() # clean up the stack return r # XXX can't return various types def setup_stack(self): - self.esp = 1 # 0 is funcaddr + self.esp = 0 + self.push_funcaddr(self.funcaddr) def push_funcaddr(self, value): - self.cpu.set_future_value_int(0, value) - #self.esp += 1 + self.cpu.set_future_value_int(self.esp, value) + self.esp += 1 def push_int(self, value): self.cpu.set_future_value_int(self.esp, value) From getxsick at codespeak.net Wed Jul 7 14:29:16 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Wed, 7 Jul 2010 14:29:16 +0200 (CEST) Subject: [pypy-svn] r75971 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100707122916.52D8A282B9E@codespeak.net> Author: getxsick Date: Wed Jul 7 14:29:15 2010 New Revision: 75971 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: oplist is a local variable now Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Wed Jul 7 14:29:15 2010 @@ -89,10 +89,10 @@ calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) self.looptoken = LoopToken() - self.oplist = [ResOperation(rop.CALL, bargs, bres, descr=calldescr), - ResOperation(rop.FINISH, [bres], None, - descr=BasicFailDescr(0))] - self.cpu.compile_loop(bargs, self.oplist, self.looptoken) + oplist = [ResOperation(rop.CALL, bargs, bres, descr=calldescr), + ResOperation(rop.FINISH, [bres], None, + descr=BasicFailDescr(0))] + self.cpu.compile_loop(bargs, oplist, self.looptoken) # add to the cache cache.append(_Func(self.args_type, self.res_type, self.looptoken)) From getxsick at codespeak.net Wed Jul 7 15:47:32 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Wed, 7 Jul 2010 15:47:32 +0200 (CEST) Subject: [pypy-svn] r75972 - in pypy/branch/fast-ctypes/pypy: module/jitffi rlib rlib/test Message-ID: <20100707134732.42772282B9E@codespeak.net> Author: getxsick Date: Wed Jul 7 15:47:30 2010 New Revision: 75972 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: return results as objects of Return subclasses to be more RPython Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Wed Jul 7 15:47:30 2010 @@ -68,9 +68,7 @@ self.rget = rjitffi._Get(cpu, lib, func, args_type, res_type) def call_w(self, space, w_args=None): - if space.is_w(w_args, space.w_None): - return space.wrap(self.rget.call()) - else: + if not space.is_w(w_args, space.w_None): i = 0 w_iterator = space.iter(w_args) while True: @@ -93,7 +91,8 @@ space.wrap('Unsupported type of argument: %s' % self.args_type[0])) i += 1 - return space.wrap(self.rget.call()) + res = self.rget.call() + return space.wrap(res.value) def W_Get___new__(space, w_type, cpu, lib, func, args_type, res_type): try: Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Wed Jul 7 15:47:30 2010 @@ -102,18 +102,18 @@ res = self.cpu.execute_token(self.looptoken) if self.res_type == 'i': - r = self.cpu.get_latest_value_int(0) + r = ReturnInt(self.cpu.get_latest_value_int(0)) elif self.res_type == 'f': - r = self.cpu.get_latest_value_float(0) + r = ReturnFloat(self.cpu.get_latest_value_float(0)) elif self.res_type == 'p': - r = self.cpu.get_latest_value_ref(0) + r = ReturnPtr(self.cpu.get_latest_value_ref(0)) elif self.res_type == 'v': - r = None + r = ReturnNone(None) else: raise ValueError(self.res_type) self.setup_stack() # clean up the stack - return r # XXX can't return various types + return r def setup_stack(self): self.esp = 0 @@ -140,3 +140,19 @@ self.args_type = args_type self.res_type = res_type self.looptoken = looptoken + +class Return(object): + def __init__(self, value): + self.value = value + +class ReturnInt(Return): + pass + +class ReturnFloat(Return): + pass + +class ReturnPtr(Return): + pass + +class ReturnNone(Return): + pass Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Wed Jul 7 15:47:30 2010 @@ -71,44 +71,44 @@ func = lib.get('add_integers', ['i', 'i'], 'i') func.push_int(1) func.push_int(2) - assert func.call() == 3 + assert func.call().value == 3 func = lib.get('add_integers', ['i', 'i'], 'i') func.push_int(-1) func.push_int(2) - assert func.call() == 1 + assert func.call().value == 1 func = lib.get('add_integers', ['i', 'i'], 'i') func.push_int(0) func.push_int(0) - assert func.call() == 0 + assert func.call().value == 0 func = lib.get('max3', ['i', 'i', 'i'], 'i') func.push_int(2) func.push_int(8) func.push_int(3) - assert func.call() == 8 + assert func.call().value == 8 func = lib.get('add_floats', ['f', 'f'], 'f') func.push_float(1.2) func.push_float(1.5) - assert func.call() == 2.7 + assert func.call().value == 2.7 def test_get_void(self): lib = rjitffi.CDLL(self.lib_name) func = lib.get('fvoid', [], 'i') - assert func.call() == 1 + assert func.call().value == 1 func = lib.get('return_void', ['i', 'i'], 'v') func.push_int(1) func.push_int(2) - assert func.call() is None + assert func.call().value is None func = lib.get('return_void', ['i', 'i']) func.push_int(1) func.push_int(2) - assert func.call() is None + assert func.call().value is None def test_various_type_args(self): lib = rjitffi.CDLL(self.lib_name) @@ -116,12 +116,12 @@ func = lib.get('add_intfloat', ['i', 'f'], 'i') func.push_int(1) func.push_float(2.9) - assert func.call() == 3 + assert func.call().value == 3 # stack is cleaned up after calling func.push_int(0) func.push_float(1.3) - assert func.call() == 1 + assert func.call().value == 1 def test_undefined_func(self): lib = rjitffi.CDLL(self.lib_name) From benjamin at codespeak.net Wed Jul 7 16:49:29 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 16:49:29 +0200 (CEST) Subject: [pypy-svn] r75973 - pypy/branch/fast-forward/pypy/module/math Message-ID: <20100707144929.9677F282B9E@codespeak.net> Author: benjamin Date: Wed Jul 7 16:49:28 2010 New Revision: 75973 Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py Log: fix silly error function bugs Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Wed Jul 7 16:49:28 2010 @@ -440,14 +440,14 @@ ERF_SERIES_TERMS = 25 ERFC_CONTFRAC_CUTOFF = 30. ERFC_CONTFRAC_TERMS = 50 -_sqrtpi = 1.772453850905516027298167483341145182798; +_sqrtpi = 1.772453850905516027298167483341145182798 def _erf_series(x): x2 = x * x acc = 0. - fk = ERF_SERIES_TERMS * + .5 + fk = ERF_SERIES_TERMS + .5 for i in range(ERF_SERIES_TERMS): - acc = 2.0 * x2 * acc / fk + acc = 2.0 + x2 * acc / fk fk -= 1. return acc * x * math.exp(-x2) / _sqrtpi @@ -456,7 +456,7 @@ return 0. x2 = x * x a = 0. - da = 0. + da = .5 p = 1. p_last = 0. q = da + x2 @@ -465,12 +465,8 @@ a += da da += 2. b = da + x2 - temp = p - p = b * p - a * p_last - p_last = temp - temp = q - q = b * q - a * q_last - q_last = temp + p_last, p = p, b * p - a * p_last + q_last, q = q, b * q - a * q_last return p / q * x * math.exp(-x2) / _sqrtpi def _erf(x): From arigo at codespeak.net Wed Jul 7 17:08:38 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 17:08:38 +0200 (CEST) Subject: [pypy-svn] r75975 - pypy/branch/rsre2/pypy/rlib/rsre/test Message-ID: <20100707150838.3277C282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 17:08:36 2010 New Revision: 75975 Added: pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py (contents, props changed) Log: Forgot to add this file. Added: pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py ============================================================================== --- (empty file) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py Wed Jul 7 17:08:36 2010 @@ -0,0 +1,25 @@ +from pypy.rlib.rsre import rsre +from pypy.rlib.rsre.test.test_match import get_code + + +class TestSearch: + + def test_code1(self): + r_code1 = get_code(r'[abc][def][ghi]') + res = rsre.search(r_code1, "fooahedixxx") + assert res is None + res = rsre.search(r_code1, "fooahcdixxx") + assert res is not None + assert res.span() == (5, 8) + + def test_code2(self): + r_code2 = get_code(r'\s*(.*?)') + res = rsre.search(r_code2, "foo bar abcdef") + assert res is not None + assert res.span() == (8, 34) + + def test_pure_literal(self): + r_code3 = get_code(r'foobar') + res = rsre.search(r_code3, "foo bar foobar baz") + assert res is not None + assert res.span() == (8, 14) From arigo at codespeak.net Wed Jul 7 17:10:20 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 17:10:20 +0200 (CEST) Subject: [pypy-svn] r75977 - pypy/branch/rsre2/pypy/rlib/rsre Message-ID: <20100707151020.C569F282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 17:10:19 2010 New Revision: 75977 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Log: Return the end and marks fields by sticking them directly on the ctx instead of returning them along with the MatchResult instance. This avoids some copying around and turns the multiple instances of base MatchResult into a single empty prebuilt instance. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Wed Jul 7 17:10:19 2010 @@ -131,25 +131,22 @@ class MatchResult(object): - - def __init__(self, end, marks): - self.end = end - self.marks = marks - - def move_to_next_result(self, ctx): - return False - -class AbstractMultipleMatchResult(MatchResult): + subresult = None def move_to_next_result(self, ctx): result = self.subresult + if result is None: + return False if result.move_to_next_result(ctx): - self.end = result.end - self.marks = result.marks return True return self.find_next_result(ctx) - -class BranchMatchResult(AbstractMultipleMatchResult): + + def find_next_result(self, ctx): + raise NotImplementedError + +MATCHED_OK = MatchResult() + +class BranchMatchResult(MatchResult): def __init__(self, ppos, ptr, marks): self.ppos = ppos @@ -163,14 +160,12 @@ ppos += ctx.pat(ppos) if result is not None: self.subresult = result - self.end = result.end - self.marks = result.marks self.ppos = ppos return True return False find_next_result = find_first_result -class RepeatOneMatchResult(AbstractMultipleMatchResult): +class RepeatOneMatchResult(MatchResult): def __init__(self, nextppos, minptr, ptr, marks): self.nextppos = nextppos @@ -185,15 +180,13 @@ ptr -= 1 if result is not None: self.subresult = result - self.end = result.end - self.marks = result.marks self.start_ptr = ptr return True return False find_next_result = find_first_result -class MinRepeatOneMatchResult(AbstractMultipleMatchResult): +class MinRepeatOneMatchResult(MatchResult): def __init__(self, nextppos, ppos3, maxptr, ptr, marks): self.nextppos = nextppos @@ -208,8 +201,6 @@ result = sre_match(ctx, self.nextppos, ptr, self.start_marks) if result is not None: self.subresult = result - self.end = result.end - self.marks = result.marks self.start_ptr = ptr return True ptr1 = find_repetition_end(ctx, self.ppos3, ptr, 1) @@ -226,7 +217,7 @@ self.start_ptr = ptr1 return self.find_first_result(ctx) -class AbstractUntilMatchResult(AbstractMultipleMatchResult): +class AbstractUntilMatchResult(MatchResult): def __init__(self, ppos, tailppos, ptr, marks): self.ppos = ppos @@ -255,8 +246,8 @@ if enum is not None: # matched one more 'item'. record it and continue self.pending.append((ptr, marks, enum)) - ptr = enum.end - marks = enum.marks + ptr = ctx.match_end + marks = ctx.match_marks break else: # 'item' no longer matches. @@ -265,8 +256,6 @@ result = sre_match(ctx, self.tailppos, ptr, marks) if result is not None: self.subresult = result - self.end = result.end - self.marks = result.marks self.cur_ptr = ptr self.cur_marks = marks return True @@ -303,8 +292,6 @@ result = sre_match(ctx, self.tailppos, ptr, marks) if result is not None: self.subresult = result - self.end = result.end - self.marks = result.marks self.cur_ptr = ptr self.cur_marks = marks return True @@ -320,8 +307,8 @@ if enum is not None: # matched one more 'item'. record it and continue self.pending.append((ptr, marks, enum)) - ptr = enum.end - marks = enum.marks + ptr = ctx.match_end + marks = ctx.match_marks break else: # 'item' no longer matches. @@ -350,7 +337,9 @@ if (op == OPCODE_SUCCESS or op == OPCODE_MAX_UNTIL or op == OPCODE_MIN_UNTIL): - return MatchResult(ptr, marks) + ctx.match_end = ptr + ctx.match_marks = marks + return MATCHED_OK elif op == OPCODE_ANY: # match anything (except a newline) @@ -659,8 +648,8 @@ AT_END_STRING = 7 AT_LOC_BOUNDARY = 8 AT_LOC_NON_BOUNDARY = 9 -AT_UNI_BOUNDARY =10 -AT_UNI_NON_BOUNDARY =11 +AT_UNI_BOUNDARY = 10 +AT_UNI_NON_BOUNDARY = 11 def sre_at(ctx, atcode, ptr): if (atcode == AT_BEGINNING or @@ -706,10 +695,7 @@ def match(pattern, string, start=0, flags=0): ctx = MatchContext(pattern, string, start, flags) - result = sre_match(ctx, 0, start, None) - if result is not None: - ctx.match_end = result.end - ctx.match_marks = result.marks + if sre_match(ctx, 0, start, None) is not None: return ctx return None @@ -723,11 +709,8 @@ def regular_search(ctx): start = ctx.match_start while start <= ctx.end: - result = sre_match(ctx, 0, start, None) - if result is not None: + if sre_match(ctx, 0, start, None) is not None: ctx.match_start = start - ctx.match_end = result.end - ctx.match_marks = result.marks return ctx start += 1 return None @@ -771,11 +754,8 @@ ctx.match_marks = None return ctx ppos = pattern_offset + 2 * prefix_skip - result = sre_match(ctx, ppos, ptr, None) - if result is not None: + if sre_match(ctx, ppos, ptr, None) is not None: ctx.match_start = start - ctx.match_end = result.end - ctx.match_marks = result.marks return ctx i = ctx.pat(overlap_offset + i) break From hakanardo at codespeak.net Wed Jul 7 17:12:04 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 7 Jul 2010 17:12:04 +0200 (CEST) Subject: [pypy-svn] r75978 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100707151204.2D8FD282B9E@codespeak.net> Author: hakanardo Date: Wed Jul 7 17:12:02 2010 New Revision: 75978 Added: pypy/branch/interplevel-array/pypy/module/array/app_array.py pypy/branch/interplevel-array/pypy/module/array/interp_simple.py Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: started to copy some function at application level from the previous implementation Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/__init__.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/__init__.py Wed Jul 7 17:12:02 2010 @@ -9,5 +9,5 @@ interpleveldefs = { 'array': 'interp_array.array', - #'sized_array': 'interp_array.sized_array', + 'simple_array': 'interp_simple.simple_array', } Added: pypy/branch/interplevel-array/pypy/module/array/app_array.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/app_array.py Wed Jul 7 17:12:02 2010 @@ -0,0 +1,90 @@ + +if True: + def initiate(self, initializer): + if initializer is not None: + if type(initializer) is str: + self.fromstring(initializer) + elif type(initializer) is unicode: + self.fromunicode(initializer) + elif type(initializer) is list: + self.fromlist(initializer) + else: + self.extend(initializer) + + def fromunicode(self, ustr): + """Extends this array with data from the unicode string ustr. The array + must be a type 'u' array; otherwise a ValueError is raised. Use + array.fromstring(ustr.encode(...)) to append Unicode data to an array of + some other type.""" + if not self.typecode == "u": + raise ValueError( + "fromunicode() may only be called on type 'u' arrays") + # XXX the following probable bug is not emulated: + # CPython accepts a non-unicode string or a buffer, and then + # behaves just like fromstring(), except that it strangely truncates + # string arguments at multiples of the unicode byte size. + # Let's only accept unicode arguments for now. + if not isinstance(ustr, unicode): + raise TypeError("fromunicode() argument should probably be " + "a unicode string") + self._fromsequence(ustr) + + + def fromfile(self, f, n): + """Read n objects from the file object f and append them to the end of + the array. Also called as read.""" + if not isinstance(f, file): + raise TypeError("arg1 must be open file") + self._fromfile(f, n) + + + def _fromfile(self, f, n): + size = self.itemsize * n + item = f.read(size) + if len(item) < size: + n = len(item) % self.itemsize + if n != 0: item = item[0:-(len(item) % self.itemsize)] + self.fromstring(item) + raise EOFError("not enough items in file") + self.fromstring(item) + + + def fromlist(self, l): + """Append items to array from list.""" + if not isinstance(l, list): + raise TypeError("arg must be list") + s=len(self) + try: + self._fromsequence(l) + except(OverflowError, TypeError, ValueError): + self._setlen(s) + raise + + + def tounicode(self): + """Convert the array to a unicode string. The array must be a type 'u' + array; otherwise a ValueError is raised. Use array.tostring().decode() + to obtain a unicode string from an array of some other type.""" + if self.typecode != "u": + raise ValueError("tounicode() may only be called on type 'u' arrays") + # XXX performance is not too good + return u"".join(self.tolist()) + + def tofile(self, f): + """Write all items (as machine values) to the file object f. Also + called as write.""" + if not isinstance(f, file): + raise TypeError("arg must be open file") + f.write(self.tostring()) + + def tostring(self): + import struct + s='' + for i in range(len(self)): + s+=struct.pack(self.typecode, self[i]) + return s + + + + + Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Wed Jul 7 17:12:02 2010 @@ -2,19 +2,33 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.rpython.lltypesystem import lltype, rffi -from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root +from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root, ApplevelClass from pypy.rlib.jit import dont_look_inside from pypy.rlib import rgc from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rstruct.runpack import runpack +import os, types +path, _ = os.path.split(__file__) +app_array = os.path.join(path, 'app_array.py') +app = ApplevelClass(file(app_array).read()) + +def appmethod(n): + import app_array + f=getattr(app_array,n) + args = f.func_code.co_varnames[0:f.func_code.co_argcount] + args = ', '.join(['space'] + ['w_'+s for s in args]) + exec """def descr(%s): + return app.interphook('%s')(%s)"""%(args, n, args) + return interp2app(descr) + class W_ArrayBase(Wrappable): pass class TypeCode(object): def __init__(self, itemtype, unwrap, canoverflow=False, signed=False): self.itemtype = itemtype - if itemtype is lltype.SingleFloat: + if itemtype is lltype.SingleFloat: # FUIXME self.bytes = 4 else: self.bytes = rffi.sizeof(itemtype) @@ -33,16 +47,16 @@ types = { 'c': TypeCode(lltype.Char, 'str_w'), 'u': TypeCode(lltype.UniChar, 'unicode_w'), - #'b': TypeCode(rffi.SIGNEDCHAR, 'int_w', True, True), - #'B': TypeCode(rffi.UCHAR, 'int_w', True), - #'h': TypeCode(rffi.SHORT, 'int_w', True, True), - #'H': TypeCode(rffi.USHORT, 'int_w', True), - #'i': TypeCode(rffi.INT, 'int_w', True, True), - #'I': TypeCode(rffi.UINT, 'int_w', True), - #'l': TypeCode(rffi.LONG, 'int_w', True, True), - #'L': TypeCode(rffi.ULONG, 'bigint_w', True), # FIXME: Won't compile - #'f': TypeCode(lltype.SingleFloat, 'float_w'), - #'d': TypeCode(lltype.Float, 'float_w'), + 'b': TypeCode(rffi.SIGNEDCHAR, 'int_w', True, True), + 'B': TypeCode(rffi.UCHAR, 'int_w', True), + 'h': TypeCode(rffi.SHORT, 'int_w', True, True), + 'H': TypeCode(rffi.USHORT, 'int_w', True), + 'i': TypeCode(rffi.INT, 'int_w', True, True), + 'I': TypeCode(rffi.UINT, 'int_w', True), + 'l': TypeCode(rffi.LONG, 'int_w', True, True), + 'L': TypeCode(rffi.ULONG, 'bigint_w', True), # FIXME: Won't compile + 'f': TypeCode(lltype.SingleFloat, 'float_w'), + 'd': TypeCode(lltype.Float, 'float_w'), } for k, v in types.items(): v.typecode=k unroll_typecodes = unrolling_iterable(types.keys()) @@ -93,11 +107,11 @@ def setlen(self, size): new_buffer = lltype.malloc(self.mytype.arraytype, size) - for i in range(self.len): + for i in range(min(size,self.len)): new_buffer[i] = self.buffer[i] self.buffer = new_buffer self.len = size - + setlen.unwrap_spec = ['self', int] def descr_len(self): return self.space.wrap(self.len) @@ -135,9 +149,32 @@ descr_append.unwrap_spec = ['self', W_Root] - def descr_extend(self, w_initializer): + def descr_fromsequence(self, w_seq): space = self.space - w_iterator = space.iter(w_initializer) + w_new = space.call_function(space.getattr(w_seq, space.wrap('__len__'))) + new = space.int_w(w_new) + oldlen = self.len + self.setlen(self.len + new) + for i in range(new): + w_item = space.call_function( + space.getattr(w_seq, space.wrap('__getitem__')), + space.wrap(i)) + try: + item=self.item_w(w_item) + except OperationError: + self.setlen(oldlen + i) + raise + self.buffer[oldlen + i ] = item + descr_fromsequence.unwrap_spec = ['self', W_Root] + + + def descr_extend(self, w_iterable): + space=self.space + if isinstance(w_iterable, W_ArrayBase): + if self.mytype.typecode != w_iterable.mytype.typecode: + msg = "can only extend with array of same kind" + raise OperationError(space.w_TypeError, space.wrap(msg)) + w_iterator = space.iter(w_iterable) while True: try: w_item = space.next(w_iterator) @@ -148,14 +185,14 @@ self.descr_append(w_item) descr_extend.unwrap_spec = ['self', W_Root] - + def descr_setitem(self, w_idx, w_item): start, stop, step = self.space.decode_index(w_idx, self.len) if step==0: item = self.item_w(w_item) self.buffer[start] = item else: - if isinstance(w_item, W_Array): + if isinstance(w_item, W_ArrayBase): if self.mytype.typecode == w_item.mytype.typecode: size = (stop - start) / step if (stop - start) % step > 0: size += 1 @@ -171,18 +208,58 @@ return msg='can only assign array to array slice' raise OperationError(self.space.w_TypeError, self.space.wrap(msg)) - descr_setitem.unwrap_spec = ['self', W_Root, W_Root] - + + def descr_fromstring(self, s): + if len(s)%self.mytype.bytes !=0: + msg = 'string length not a multiple of item size' + raise OperationError(self.space.w_ValueError, self.space.wrap(msg)) + oldlen = self.len + new = len(s) / self.mytype.bytes + self.setlen(oldlen + new) + for i in range(new): + p = i * self.mytype.bytes + item=runpack(self.mytype.typecode, s[p:p + self.mytype.bytes]) + #self.buffer[oldlen + i]=self.item_w(self.space.wrap(item)) + self.buffer[oldlen + i]=rffi.cast(self.mytype.itemtype, item) + descr_fromstring.unwrap_spec = ['self', str] + + def descr_tolist(self): + return self.space.newlist([self.space.wrap(i) for i in self.buffer]) + descr_tolist.unwrap_spec = ['self'] + + + + def descr_itemsize(space, self): + return space.wrap(self.mytype.bytes) + def descr_typecode(space, self): + return space.wrap(self.mytype.typecode) W_Array.__name__ = 'W_ArrayType_'+thetypecode W_Array.typedef = TypeDef( 'ArrayType_'+thetypecode, - append = interp2app(W_Array.descr_append), - extend = interp2app(W_Array.descr_extend), - __len__ = interp2app(W_Array.descr_len), - __getitem__ = interp2app(W_Array.descr_getitem), - __setitem__ = interp2app(W_Array.descr_setitem), + append = interp2app(W_Array.descr_append), + __len__ = interp2app(W_Array.descr_len), + __getitem__ = interp2app(W_Array.descr_getitem), + __setitem__ = interp2app(W_Array.descr_setitem), + + itemsize = GetSetProperty(descr_itemsize, cls=W_Array), + typecode = GetSetProperty(descr_typecode, cls=W_Array), + extend = interp2app(W_Array.descr_extend), + + _fromsequence = interp2app(W_Array.descr_fromsequence), + fromstring = interp2app(W_Array.descr_fromstring), + fromunicode = appmethod('fromunicode'), + fromfile = appmethod('fromfile'), + _fromfile = appmethod('_fromfile'), + fromlist = appmethod('fromlist'), + + tolist = interp2app(W_Array.descr_tolist), + tounicode = appmethod('tounicode'), + tofile = appmethod('tofile'), + tostring = appmethod('tostring'), + + _setlen = interp2app(W_Array.setlen), ) thetype.w_class = W_Array @@ -193,25 +270,19 @@ msg = 'array() argument 1 must be char, not str' raise OperationError(space.w_TypeError, space.wrap(msg)) typecode=typecode[0] - + for tc in unroll_typecodes: if typecode == tc: a = types[tc].w_class(space) - if w_initializer is not None: - if not space.is_w(w_initializer, space.w_None): - a.descr_extend(w_initializer) - ## if space.is_w(space.type(w_initializer), space.w_str): - ## a.descr_fromstring(space.str_w(w_initializer)) - ## elif space.is_w(space.type(w_initializer), space.w_unicode): - ## a.descr_fromunicode(space.unicode_w(w_initializer)) - ## elif space.is_w(space.type(w_initializer), space.w_list): - ## a.descr_fromlist(w_initializer) - ## elif not space.is_w(w_initializer, space.w_None): - ## a.descr_extend(w_initializer) + app.interphook('initiate')(space, a, w_initializer) + ## if w_initializer is not None: + ## if not space.is_w(w_initializer, space.w_None): + ## a.descr_fromsequence(w_initializer) break else: msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' raise OperationError(space.w_ValueError, space.wrap(msg)) + return a array.unwrap_spec = (ObjSpace, str, W_Root) Added: pypy/branch/interplevel-array/pypy/module/array/interp_simple.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/interp_simple.py Wed Jul 7 17:12:02 2010 @@ -0,0 +1,47 @@ +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.error import OperationError +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root +from pypy.rlib.jit import dont_look_inside +from pypy.rlib import rgc + + +FloatArray = lltype.GcArray(lltype.Float) + + +class W_SizedFloatArray(Wrappable): + @dont_look_inside + def __init__(self, space, size): + self.space = space + self.size = size + print "malloc" + #self.buffer = lltype.malloc(rffi.DOUBLEP.TO, size, flavor='raw', + # zero=True) + #self.buffer = rgc.malloc_nonmovable(lltype.GcArray(lltype.Float), + # size) + self.buffer = lltype.malloc(FloatArray, size, zero=True) + print "buf: ", self.buffer + + def __del__(self): + print "free" + #lltype.free(self.buffer, flavor='raw') + + def descr_getitem(self, idx): + return self.space.wrap(self.buffer[idx]) + descr_getitem.unwrap_spec = ['self', int] + + def descr_setitem(self, idx, val): + self.buffer[idx] = val + descr_setitem.unwrap_spec = ['self', int, float] + +W_SizedFloatArray.typedef = TypeDef( + 'SizedFloatArray', + __getitem__=interp2app(W_SizedFloatArray.descr_getitem), + __setitem__=interp2app(W_SizedFloatArray.descr_setitem), +) + + +def simple_array(space, size): + return W_SizedFloatArray(space, size) +simple_array.unwrap_spec = (ObjSpace, int) Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Wed Jul 7 17:12:02 2010 @@ -1,16 +1,17 @@ from pypy.conftest import gettestobjspace +import py -class AppTestSizedArray: +class AppTestSimpleArray: def setup_class(cls): cls.space = gettestobjspace(usemodules=('array',)) - cls.w_sized_array = cls.space.appexec([], """(): + cls.w_simple_array = cls.space.appexec([], """(): import array - return array.sized_array + return array.simple_array """) def test_simple(self): - a = self.sized_array(10) + a = self.simple_array(10) a[5] = 7.42 assert a[5] == 7.42 @@ -155,23 +156,23 @@ def read(self,n): return self.c*min(n,self.s) - f=myfile('\x00', 20) + f=open('/dev/zero','r') for t in 'bBhHiIlLfd': a = self.array(t) a.fromfile(f,2) assert len(a)==2 and a[0]==0 and a[1]==0 a = self.array('b') - a.fromfile(myfile('\x01', 20),2) + a._fromfile(myfile('\x01', 20),2) assert len(a)==2 and a[0]==1 and a[1]==1 a = self.array('h') - a.fromfile(myfile('\x01', 20),2) + a._fromfile(myfile('\x01', 20),2) assert len(a)==2 and a[0]==257 and a[1]==257 for i in (0,1): a = self.array('h') - raises(EOFError, a.fromfile, myfile('\x01', 2+i),2) + raises(EOFError, a._fromfile, myfile('\x01', 2+i),2) assert len(a)==1 and a[0]==257 @@ -189,12 +190,20 @@ assert len(a) == 2 and a[0] == 1 and a[1] == 2 a = self.array('b') - raises(OverflowError, a.fromlist, (1, 2, 400)) - assert len(a) == 0 + raises(TypeError, a.fromlist, (1, 2, 400)) raises(OverflowError, a.extend, (1, 2, 400)) assert len(a) == 2 and a[0] == 1 and a[1] == 2 + raises(TypeError, a.extend, self.array('i',(7,8))) + assert len(a) == 2 and a[0] == 1 and a[1] == 2 + + def gen(): + for i in range(4): + yield i + 10 + a = self.array('i', gen()) + assert len(a) == 4 and a[2] == 12 + raises(OverflowError, self.array, 'b', (1, 2, 400)) a = self.array('b', (1, 2)) @@ -278,22 +287,25 @@ a = self.array('i',[0,0,0]) assert a.tostring() == '\x00'*3*a.itemsize - from cStringIO import StringIO - f=StringIO() - self.array('c', ('h', 'i')).tofile(f) - assert f.getvalue() == 'hi' - + #FXIME: How to test? + #from cStringIO import StringIO + #f=StringIO() + #self.array('c', ('h', 'i')).tofile(f) + #assert f.getvalue() == 'hi' + raises(ValueError, self.array('i').tounicode) assert self.array('u', unicode('hello')).tounicode() == unicode('hello') - - def test_type(self): - for t in 'bBhHiIlLfdcu': - assert type(self.array(t)) is self.array -class AppTestAppArray(AppTestArray): - def setup_class(cls): - cls.space = gettestobjspace(usemodules=('array',)) - cls.w_array = cls.space.appexec([], """(): - import apparray - return apparray.array - """) + #FIXME + #def test_type(self): + # for t in 'bBhHiIlLfdcu': + # assert type(self.array(t)) is self.array + + +## class AppTestAppArray(AppTestArray): +## def setup_class(cls): +## cls.space = gettestobjspace(usemodules=('array',)) +## cls.w_array = cls.space.appexec([], """(): +## import apparray +## return apparray.array +## """) From arigo at codespeak.net Wed Jul 7 17:12:35 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 17:12:35 +0200 (CEST) Subject: [pypy-svn] r75979 - in pypy/branch/rsre2/pypy/rlib/rsre-jit: . test Message-ID: <20100707151235.28C73282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 17:12:33 2010 New Revision: 75979 Added: pypy/branch/rsre2/pypy/rlib/rsre-jit/ - copied from r75882, pypy/branch/rsre2/pypy/rlib/rsre/ pypy/branch/rsre2/pypy/rlib/rsre-jit/rsre.py - copied unchanged from r75977, pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre-jit/test/re_tests.py - copied unchanged from r75890, pypy/branch/rsre2/pypy/rlib/rsre/test/re_tests.py pypy/branch/rsre2/pypy/rlib/rsre-jit/test/targetrsre.py - copied unchanged from r75959, pypy/branch/rsre2/pypy/rlib/rsre/test/targetrsre.py pypy/branch/rsre2/pypy/rlib/rsre-jit/test/test_match.py - copied unchanged from r75897, pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py pypy/branch/rsre2/pypy/rlib/rsre-jit/test/test_more.py - copied unchanged from r75911, pypy/branch/rsre2/pypy/rlib/rsre/test/test_more.py pypy/branch/rsre2/pypy/rlib/rsre-jit/test/test_search.py - copied unchanged from r75975, pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py pypy/branch/rsre2/pypy/rlib/rsre-jit/test/test_zexternal.py - copied unchanged from r75957, pypy/branch/rsre2/pypy/rlib/rsre/test/test_zexternal.py Log: A "local branch" in which to play with the jit. From benjamin at codespeak.net Wed Jul 7 17:13:18 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 17:13:18 +0200 (CEST) Subject: [pypy-svn] r75980 - pypy/branch/fast-forward/pypy/module/math Message-ID: <20100707151318.A05CD282B9E@codespeak.net> Author: benjamin Date: Wed Jul 7 17:13:17 2010 New Revision: 75980 Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py pypy/branch/fast-forward/pypy/module/math/interp_math.py Log: add gamma and lgamma Modified: pypy/branch/fast-forward/pypy/module/math/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/math/__init__.py Wed Jul 7 17:13:17 2010 @@ -47,5 +47,7 @@ 'expm1' : 'interp_math.expm1', 'erf' : 'interp_math.erf', 'erfc' : 'interp_math.erfc', + 'gamma' : 'interp_math.gamma', + 'lgamma' : 'interp_math.lgamma', } Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Wed Jul 7 17:13:17 2010 @@ -432,6 +432,16 @@ return math1(space, _erfc, x) erfc.unwrap_spec = [ObjSpace, float] +def gamma(space, x): + """Compute the gamma function for x.""" + return math1(space, _gamma, x) +gamma.unwrap_spec = [ObjSpace, float] + +def lgamma(space, x): + """Compute the natural logarithm of the gamma function for x.""" + return math1(space, _lgamma, x) +lgamma.unwrap_spec = [ObjSpace, float] + # Implementation of the error function, the complimentary error function, the # gamma function, and the natural log of the gamma function. This exist in # libm, but I hear those implementations are horrible. @@ -488,3 +498,137 @@ else: cf = _erfc_contfrac(absx) return cf if x > 0. else 2. - cf + +def _sinpi(x): + y = math.fmod(abs(x), 2.) + n = int(round(2. * y)) + if n == 0: + r = math.sin(math.pi * y) + elif n == 1: + r = math.cos(math.pi * (y - .5)) + elif n == 2: + r = math.sin(math.pi * (1. - y)) + elif n == 3: + r = -math.cos(math.pi * (y - 1.5)) + elif n == 4: + r = math.sin(math.pi * (y - 2.)) + else: + raise AssertionError("should not reach") + return rarithmetic.copysign(1., x) * r + +_lanczos_g = 6.024680040776729583740234375 +_lanczos_g_minus_half = 5.524680040776729583740234375 +_lanczos_num_coeffs = [ + 23531376880.410759688572007674451636754734846804940, + 42919803642.649098768957899047001988850926355848959, + 35711959237.355668049440185451547166705960488635843, + 17921034426.037209699919755754458931112671403265390, + 6039542586.3520280050642916443072979210699388420708, + 1439720407.3117216736632230727949123939715485786772, + 248874557.86205415651146038641322942321632125127801, + 31426415.585400194380614231628318205362874684987640, + 2876370.6289353724412254090516208496135991145378768, + 186056.26539522349504029498971604569928220784236328, + 8071.6720023658162106380029022722506138218516325024, + 210.82427775157934587250973392071336271166969580291, + 2.5066282746310002701649081771338373386264310793408 +] +_lanczos_den_coeffs = [ + 0.0, 39916800.0, 120543840.0, 150917976.0, 105258076.0, 45995730.0, + 13339535.0, 2637558.0, 357423.0, 32670.0, 1925.0, 66.0, 1.0] +_gamma_integrals = [ + 1.0, 1.0, 2.0, 6.0, 24.0, 120.0, 720.0, 5040.0, 40320.0, 362880.0, + 3628800.0, 39916800.0, 479001600.0, 6227020800.0, 87178291200.0, + 1307674368000.0, 20922789888000.0, 355687428096000.0, + 6402373705728000.0, 121645100408832000.0, 2432902008176640000.0, + 51090942171709440000.0, 1124000727777607680000.0] + +def _lanczos_sum(x): + num = 0. + den = 0. + assert x > 0. + if x < 5.: + for i in range(len(_lanczos_den_coeffs) - 1, -1, -1): + num = num * x + _lanczos_num_coeffs[i] + den = den * x + _lanczos_den_coeffs[i] + else: + for i in range(len(_lanczos_den_coeffs)): + num = num / x + _lanczos_num_coeffs[i] + den = den / x + _lanczos_den_coeffs[i] + return num / den + +def _gamma(x): + if rarithmetic.isnan(x) or (rarithmetic.isinf(x) and x > 0.): + return x + if rarithmetic.isinf(x): + raise ValueError("math domain error") + if x == 0.: + raise ValueError("math domain error") + if x == math.floor(x): + if x < 0.: + raise ValueError("math domain error") + if x < len(_gamma_integrals): + return _gamma_integrals[int(x) - 1] + absx = abs(x) + if absx < 1e-20: + r = 1. / x + if rarithmetic.isinf(r): + raise OverflowError("math range error") + return r + if absx > 200.: + if x < 0.: + return 0. / -_sinpi(x) + else: + raise OverflowError("math range error") + y = absx + _lanczos_g_minus_half + if absx > _lanczos_g_minus_half: + q = y - absx + z = q - _lanczos_g_minus_half + else: + q = y - _lanczos_g_minus_half + z = q - absx + z = z * _lanczos_g / y + if x < 0.: + r = -math.pi / _sinpi(absx) / absx * math.exp(y) / _lanczos_sum(absx) + r -= z * r + if absx < 140.: + r /= math.pow(y, absx - .5) + else: + sqrtpow = math.pow(y, absx / 2. - .25) + r /= sqrtpow + r /= sqrtpow + else: + r = _lanczos_sum(absx) / math.exp(y) + r += z * r + if absx < 140.: + r *= math.pow(y, absx - .5) + else: + sqrtpow = math.pow(y, absx / 2. - .25) + r *= sqrtpow + r *= sqrtpow + if rarithmetic.isinf(r): + raise OverflowError("math range error") + return r + +def _lgamma(x): + if rarithmetic.isnan(x): + return x + if rarithmetic.isinf(x): + return rarithmetic.INFINITY + if x == math.floor(x) and x <= 2.: + if x <= 0.: + raise ValueError("math range error") + return 0. + absx = abs(x) + if absx < 1e-20: + return -math.log(absx) + if x > 0.: + r = (math.log(_lanczos_sum(x)) - _lanczos_g + (x - .5) * + (math.log(x + _lanczos_g - .5) - 1)) + else: + r = (math.log(math.pi) - math.log(abs(_sinpi(absx))) - math.log(absx) - + (math.log(_lanczos_sum(absx)) - _lanczos_g + + (absx - .5) * (math.log(absx + _lanczos_g - .5) - 1))) + if rarithmetic.isinf(r): + raise OverflowError("math domain error") + return r From arigo at codespeak.net Wed Jul 7 17:13:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 17:13:37 +0200 (CEST) Subject: [pypy-svn] r75981 - in pypy/branch/rsre2/pypy/rlib: rsre-jit rsre_jit Message-ID: <20100707151337.0AD2F282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 17:13:36 2010 New Revision: 75981 Added: pypy/branch/rsre2/pypy/rlib/rsre_jit/ - copied from r75979, pypy/branch/rsre2/pypy/rlib/rsre-jit/ Removed: pypy/branch/rsre2/pypy/rlib/rsre-jit/ Log: Give it a importable name... From arigo at codespeak.net Wed Jul 7 17:15:42 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 17:15:42 +0200 (CEST) Subject: [pypy-svn] r75982 - in pypy/branch/rsre2/pypy/rlib/rsre_jit: . test Message-ID: <20100707151542.249D6282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 17:15:40 2010 New Revision: 75982 Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py pypy/branch/rsre2/pypy/rlib/rsre_jit/test/targetrsre.py pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_match.py pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_more.py pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_search.py pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_zexternal.py Log: Fix imports. Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py Wed Jul 7 17:15:40 2010 @@ -1,5 +1,5 @@ from pypy.rlib.debug import check_nonneg -from pypy.rlib.rsre import rsre_char +from pypy.rlib.rsre_jit import rsre_char OPCODE_FAILURE = 0 Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/test/targetrsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre_jit/test/targetrsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre_jit/test/targetrsre.py Wed Jul 7 17:15:40 2010 @@ -1,6 +1,6 @@ #!/usr/bin/env python from pypy.rlib.rarithmetic import intmask -from pypy.rlib.rsre import rsre +from pypy.rlib.rsre_jit import rsre import os, time Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_match.py Wed Jul 7 17:15:40 2010 @@ -1,5 +1,5 @@ import _sre, re, sre_compile -from pypy.rlib.rsre import rsre +from pypy.rlib.rsre_jit import rsre def get_code(regexp): Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_more.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_more.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_more.py Wed Jul 7 17:15:40 2010 @@ -1,5 +1,5 @@ -from pypy.rlib.rsre import rsre -from pypy.rlib.rsre.test.test_match import get_code, get_code_and_re +from pypy.rlib.rsre_jit import rsre +from pypy.rlib.rsre_jit.test.test_match import get_code, get_code_and_re class TestSearch: Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_search.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_search.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_search.py Wed Jul 7 17:15:40 2010 @@ -1,5 +1,5 @@ -from pypy.rlib.rsre import rsre -from pypy.rlib.rsre.test.test_match import get_code +from pypy.rlib.rsre_jit import rsre +from pypy.rlib.rsre_jit.test.test_match import get_code class TestSearch: Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_zexternal.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_zexternal.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_zexternal.py Wed Jul 7 17:15:40 2010 @@ -1,20 +1,20 @@ import re -from pypy.rlib.rsre.test.test_match import get_code -from pypy.rlib.rsre import rsre +from pypy.rlib.rsre_jit.test.test_match import get_code +from pypy.rlib.rsre_jit import rsre def test_external_match(): - from pypy.rlib.rsre.test.re_tests import tests + from pypy.rlib.rsre_jit.test.re_tests import tests for t in tests: yield run_external, t, False def test_external_search(): - from pypy.rlib.rsre.test.re_tests import tests + from pypy.rlib.rsre_jit.test.re_tests import tests for t in tests: yield run_external, t, True def run_external(t, use_search): - from pypy.rlib.rsre.test.re_tests import SUCCEED, FAIL, SYNTAX_ERROR + from pypy.rlib.rsre_jit.test.re_tests import SUCCEED, FAIL, SYNTAX_ERROR pattern, s, outcome = t[:3] if len(t) == 5: repl, expected = t[3:5] From benjamin at codespeak.net Wed Jul 7 17:23:37 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 17:23:37 +0200 (CEST) Subject: [pypy-svn] r75983 - in pypy/branch/fast-forward/pypy: objspace/std rlib/rstruct Message-ID: <20100707152337.D4D89282B9E@codespeak.net> Author: benjamin Date: Wed Jul 7 17:23:36 2010 New Revision: 75983 Modified: pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Log: fix struct float/double ops Modified: pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py Wed Jul 7 17:23:36 2010 @@ -169,11 +169,11 @@ def pack_float(f): result = [] - ieee.pack_float8(result, f) + ieee.pack_float(result, f, 8, False) return ''.join(result) def unpack_float(s): - return ieee.unpack_float8(s) + return ieee.unpack_float(s, False) def marshal_w__Float(space, w_float, m): if m.version > 1: Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Wed Jul 7 17:23:36 2010 @@ -123,14 +123,17 @@ return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant -def pack_float8(result, x): - unsigned = float_pack(x, 8) - for i in range(8): +def pack_float(result, x, size, be): + unsigned = float_pack(x, size) + for i in range(size): result.append(chr((unsigned >> (i * 8)) & 0xFF)) + if be: + result.reverse() -def unpack_float8(s): +def unpack_float(s, be): unsigned = 0 - for i in range(8): - unsigned |= ord(s[i]) << (i * 8) - return float_unpack(unsigned, 8) + for i in range(len(s)): + c = ord(s[len(s) - 1 - i if be else i]) + unsigned |= c << (i * 8) + return float_unpack(unsigned, len(s)) From wlav at codespeak.net Wed Jul 7 17:48:03 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Wed, 7 Jul 2010 17:48:03 +0200 (CEST) Subject: [pypy-svn] r75985 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src Message-ID: <20100707154803.231F4282BF2@codespeak.net> Author: wlav Date: Wed Jul 7 17:48:01 2010 New Revision: 75985 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Log: Use an opaque handle instead of class name to locate Reflex::Type. Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Wed Jul 7 17:48:01 2010 @@ -22,55 +22,60 @@ use_cpp_linker=True, ) +c_cppyy_get_typehandle = rffi.llexternal( + "cppyy_get_typehandle", + [rffi.CCHARP], rffi.VOIDP, + compilation_info=eci) + c_callstatic_l = rffi.llexternal( "callstatic_l", - [rffi.CCHARP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.LONG, + [rffi.VOIDP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) c_callstatic_d = rffi.llexternal( "callstatic_d", - [rffi.CCHARP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.DOUBLE, + [rffi.VOIDP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.DOUBLE, compilation_info=eci) c_construct = rffi.llexternal( "construct", - [rffi.CCHARP, rffi.INT, rffi.VOIDPP], rffi.VOIDP, + [rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.VOIDP, compilation_info=eci) c_callmethod_l = rffi.llexternal( "callmethod_l", - [rffi.CCHARP, rffi.INT, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.LONG, + [rffi.VOIDP, rffi.INT, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) c_destruct = rffi.llexternal( "destruct", - [rffi.CCHARP, rffi.VOIDP], lltype.Void, + [rffi.VOIDP, rffi.VOIDP], lltype.Void, compilation_info=eci) c_num_methods = rffi.llexternal( "num_methods", - [rffi.CCHARP], rffi.INT, + [rffi.VOIDP], rffi.INT, compilation_info=eci) c_method_name = rffi.llexternal( "method_name", - [rffi.CCHARP, rffi.INT], rffi.CCHARP, + [rffi.VOIDP, rffi.INT], rffi.CCHARP, compilation_info=eci) c_result_type_method = rffi.llexternal( "result_type_method", - [rffi.CCHARP, rffi.INT], rffi.CCHARP, + [rffi.VOIDP, rffi.INT], rffi.CCHARP, compilation_info=eci) c_num_args_method = rffi.llexternal( "num_args_method", - [rffi.CCHARP, rffi.INT], rffi.INT, + [rffi.VOIDP, rffi.INT], rffi.INT, compilation_info=eci) c_arg_type_method = rffi.llexternal( "arg_type_method", - [rffi.CCHARP, rffi.INT, rffi.INT], rffi.CCHARP, + [rffi.VOIDP, rffi.INT, rffi.INT], rffi.CCHARP, compilation_info=eci) c_is_constructor = rffi.llexternal( "is_constructor", - [rffi.CCHARP, rffi.INT], rffi.INT, + [rffi.VOIDP, rffi.INT], rffi.INT, compilation_info=eci) c_is_static = rffi.llexternal( "is_static", - [rffi.CCHARP, rffi.INT], rffi.INT, + [rffi.VOIDP, rffi.INT], rffi.INT, compilation_info=eci) c_myfree = rffi.llexternal( "myfree", Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Wed Jul 7 17:48:01 2010 @@ -13,9 +13,9 @@ class LongExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): if cppthis is not None: - result = capi.c_callmethod_l(func.cpptype.name, func.method_index, cppthis, num_args, args) + result = capi.c_callmethod_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) else: - result = capi.c_callstatic_l(func.cpptype.name, func.method_index, num_args, args) + result = capi.c_callstatic_l(func.cpptype.handle, func.method_index, num_args, args) return space.wrap(result) class DoubleExecutor(FunctionExecutor): @@ -23,7 +23,7 @@ if cppthis is not None: raise NotImplementedError else: - result = capi.c_callstatic_d(func.cpptype.name, func.method_index, num_args, args) + result = capi.c_callstatic_d(func.cpptype.handle, func.method_index, num_args, args) return space.wrap(result) class CStringExecutor(FunctionExecutor): @@ -31,7 +31,7 @@ if cppthis is not None: raise NotImplementedError else: - lresult = capi.c_callstatic_l(func.cpptype.name, func.method_index, num_args, args) + lresult = capi.c_callstatic_l(func.cpptype.handle, func.method_index, num_args, args) ccpresult = rffi.cast(rffi.CCHARP, lresult) result = capi.charp2str_free(ccpresult) return space.wrap(result) Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Wed Jul 7 17:48:01 2010 @@ -5,19 +5,21 @@ #ifdef __cplusplus extern "C" { #endif // ifdef __cplusplus - long callstatic_l(const char* class_name, int method_index, int numargs, void* args[]); - double callstatic_d(const char* class_name, int method_index, int numargs, void* args[]); - long callmethod_l(const char* class_name, int method_index, void* self, int numargs, void* args[]); - void* construct(const char* class_name, int numargs, void* args[]); - void destruct(const char* class_name, void* self); + void* cppyy_get_typehandle(const char* class_name); - int num_methods(const char* class_name); - char* method_name(const char* class_name, int method_index); - char* result_type_method(const char* class_name, int method_index); - int num_args_method(const char* class_name, int method_index); - char* arg_type_method(const char* class_name, int method_index, int index); - int is_constructor(const char* class_name, int method_index); - int is_static(const char* class_name, int method_index); + long callstatic_l(void* handle, int method_index, int numargs, void* args[]); + double callstatic_d(void* handle, int method_index, int numargs, void* args[]); + long callmethod_l(void* handle, int method_index, void* self, int numargs, void* args[]); + void* construct(void* handle, int numargs, void* args[]); + void destruct(void* handle, void* self); + + int num_methods(void* handle); + char* method_name(void* handle, int method_index); + char* result_type_method(void* handle, int method_index); + int num_args_method(void* handle, int method_index); + char* arg_type_method(void* handle, int method_index, int index); + int is_constructor(void* handle, int method_index); + int is_static(void* handle, int method_index); void myfree(void* ptr); #ifdef __cplusplus Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Wed Jul 7 17:48:01 2010 @@ -27,7 +27,8 @@ self.space = space def type_byname(self, name): - return W_CPPType(self, name) + handle = capi.c_cppyy_get_typehandle(name) + return W_CPPType(self, name, handle) W_CPPLibrary.typedef = TypeDef( 'CPPLibrary', @@ -112,7 +113,7 @@ def call(self, cppthis, args_w): assert not cppthis args = self.prepare_arguments(args_w) - result = capi.c_construct(self.cpptype.name, len(args_w), args) + result = capi.c_construct(self.cpptype.handle, len(args_w), args) self.free_arguments(args) return W_CPPObject(self.cpptype, result) @@ -145,20 +146,21 @@ class W_CPPType(Wrappable): - _immutable_fields_ = ["cpplib", "name"] + _immutable_fields_ = ["cpplib", "name","handle"] - def __init__(self, cpplib, name): + def __init__(self, cpplib, name, handle): self.space = cpplib.space self.cpplib = cpplib self.name = name + self.handle = handle self.function_members = {} self._find_func_members() def _find_func_members(self): - num_func_members = capi.c_num_methods(self.name) + num_func_members = capi.c_num_methods(self.handle) args_temp = {} for i in range(num_func_members): - func_member_name = capi.charp2str_free(capi.c_method_name(self.name, i)) + func_member_name = capi.charp2str_free(capi.c_method_name(self.handle, i)) cppfunction = self._make_cppfunction(i) overload = args_temp.setdefault(func_member_name, []) overload.append(cppfunction) @@ -167,15 +169,15 @@ self.function_members[name] = overload def _make_cppfunction(self, method_index): - result_type = capi.charp2str_free(capi.c_result_type_method(self.name, method_index)) - num_args = capi.c_num_args_method(self.name, method_index) + result_type = capi.charp2str_free(capi.c_result_type_method(self.handle, method_index)) + num_args = capi.c_num_args_method(self.handle, method_index) argtypes = [] for i in range(num_args): - argtype = capi.charp2str_free(capi.c_arg_type_method(self.name, method_index, i)) + argtype = capi.charp2str_free(capi.c_arg_type_method(self.handle, method_index, i)) argtypes.append(argtype) - if capi.c_is_constructor(self.name, method_index): + if capi.c_is_constructor(self.handle, method_index): cls = CPPConstructor - elif capi.c_is_static(self.name, method_index): + elif capi.c_is_static(self.handle, method_index): cls = CPPFunction else: cls = CPPMethod @@ -212,7 +214,7 @@ return overload.call(self.rawobject, args_w) def destruct(self): - capi.c_destruct(self.cppclass.name, self.rawobject) + capi.c_destruct(self.cppclass.handle, self.rawobject) W_CPPObject.typedef = TypeDef( 'CPPObject', Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Wed Jul 7 17:48:01 2010 @@ -3,37 +3,42 @@ #include #include -long callstatic_l(const char* class_name, int method_index, int numargs, void* args[]) { + +void* cppyy_get_typehandle(const char* class_name) { + return Reflex::Type::ByName(class_name).Id(); +} + +long callstatic_l(void* handle, int method_index, int numargs, void* args[]) { long result; std::vector arguments(args, args+numargs); - Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); m.Invoke(result, arguments); return result; } -double callstatic_d(const char* class_name, int method_index, int numargs, void* args[]) { +double callstatic_d(void* handle, int method_index, int numargs, void* args[]) { double result; std::vector arguments(args, args+numargs); - Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); m.Invoke(result, arguments); return result; } -long callmethod_l(const char* class_name, int method_index, +long callmethod_l(void* handle, int method_index, void* self, int numargs, void* args[]) { long result; std::vector arguments(args, args+numargs); - Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Type t((Reflex::TypeName*)handle); Reflex::Object o(t, self); Reflex::Member m = t.FunctionMemberAt(method_index); m.Invoke(o, result, arguments); return result; } -void* construct(const char* class_name, int numargs, void* args[]) { +void* construct(void* handle, int numargs, void* args[]) { std::vector arguments(args, args+numargs); - Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Type t((Reflex::TypeName*)handle); std::vector argtypes; argtypes.reserve(numargs); for (int i = 0; i < numargs; i++) { @@ -44,14 +49,14 @@ return t.Construct(constructor_type, arguments).Address(); } -void destruct(const char* class_name, void* self) { - Reflex::Type t = Reflex::Type::ByName(class_name); +void destruct(void* handle, void* self) { + Reflex::Type t((Reflex::TypeName*)handle); t.Destruct(self, true); } -int num_methods(const char* class_name) { - Reflex::Type t = Reflex::Type::ByName(class_name); +int num_methods(void* handle) { + Reflex::Type t((Reflex::TypeName*)handle); for (int i = 0; i < (int)t.FunctionMemberSize(); i++) { Reflex::Member m = t.FunctionMemberAt(i); std::cout << i << " " << m.Name() << std::endl; @@ -63,8 +68,8 @@ return t.FunctionMemberSize(); } -char* method_name(const char* class_name, int method_index) { - Reflex::Type t = Reflex::Type::ByName(class_name); +char* method_name(void* handle, int method_index) { + Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); std::string name = m.Name(); char* name_char = (char*)malloc(name.size() + 1); @@ -72,8 +77,8 @@ return name_char; } -char* result_type_method(const char* class_name, int method_index) { - Reflex::Type t = Reflex::Type::ByName(class_name); +char* result_type_method(void* handle, int method_index) { + Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); Reflex::Type rt = m.TypeOf().ReturnType(); std::string name = rt.Name(Reflex::FINAL|Reflex::SCOPED|Reflex::QUALIFIED); @@ -82,15 +87,15 @@ return name_char; } -int num_args_method(const char* class_name, int method_index) { - Reflex::Type t = Reflex::Type::ByName(class_name); +int num_args_method(void* handle, int method_index) { + Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); return m.FunctionParameterSize(); } -char* arg_type_method(const char* class_name, int method_index, int arg_index) { +char* arg_type_method(void* handle, int method_index, int arg_index) { - Reflex::Type t = Reflex::Type::ByName(class_name); + Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); Reflex::Type at = m.TypeOf().FunctionParameterAt(arg_index); std::string name = at.Name(Reflex::FINAL|Reflex::SCOPED|Reflex::QUALIFIED); @@ -99,14 +104,14 @@ return name_char; } -int is_constructor(const char* class_name, int method_index) { - Reflex::Type t = Reflex::Type::ByName(class_name); +int is_constructor(void* handle, int method_index) { + Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); return m.IsConstructor(); } -int is_static(const char* class_name, int method_index) { - Reflex::Type t = Reflex::Type::ByName(class_name); +int is_static(void* handle, int method_index) { + Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); return m.IsStatic(); } From arigo at codespeak.net Wed Jul 7 17:55:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 17:55:52 +0200 (CEST) Subject: [pypy-svn] r75986 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100707155552.77614282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 17:55:51 2010 New Revision: 75986 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Log: Fix the behavior of (?=(group)) to really return the group. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Wed Jul 7 17:55:51 2010 @@ -361,6 +361,7 @@ ptr1 = ptr - ctx.pat(ppos+1) if ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None: return + marks = ctx.match_marks ppos += ctx.pat(ppos) elif op == OPCODE_ASSERT_NOT: Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Wed Jul 7 17:55:51 2010 @@ -195,3 +195,19 @@ assert rsre.match(r, "xyxyxyB") is not None r = get_code(r"(?:xy(?:xy)+?)+?B") assert rsre.match(r, "xyxyxyB") is not None + + def test_assert_group(self): + r = get_code(r"abc(?=(..)f)(.)") + res = rsre.match(r, "abcdefghi") + assert res is not None + assert res.span(2) == (3, 4) + assert res.span(1) == (3, 5) + + def test_assert_not_group(self): + r = get_code(r"abc(?!(de)f)(.)") + res = rsre.match(r, "abcdeFghi") + assert res is not None + assert res.span(2) == (3, 4) + # this I definitely classify as Horrendously Implementation Dependent. + # CPython answers (3, 5). + assert res.span(1) == (-1, -1) From wlav at codespeak.net Wed Jul 7 18:13:33 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Wed, 7 Jul 2010 18:13:33 +0200 (CEST) Subject: [pypy-svn] r75987 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src Message-ID: <20100707161333.4D054282B9E@codespeak.net> Author: wlav Date: Wed Jul 7 18:13:31 2010 New Revision: 75987 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Log: Merge static and method call for resolution on the C++ side. Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Wed Jul 7 18:13:31 2010 @@ -39,8 +39,8 @@ "construct", [rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.VOIDP, compilation_info=eci) -c_callmethod_l = rffi.llexternal( - "callmethod_l", +c_cppyy_call_l = rffi.llexternal( + "cppyy_call_l", [rffi.VOIDP, rffi.INT, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) c_destruct = rffi.llexternal( Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Wed Jul 7 18:13:31 2010 @@ -12,10 +12,7 @@ class LongExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): - if cppthis is not None: - result = capi.c_callmethod_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) - else: - result = capi.c_callstatic_l(func.cpptype.handle, func.method_index, num_args, args) + result = capi.c_cppyy_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) return space.wrap(result) class DoubleExecutor(FunctionExecutor): @@ -31,7 +28,7 @@ if cppthis is not None: raise NotImplementedError else: - lresult = capi.c_callstatic_l(func.cpptype.handle, func.method_index, num_args, args) + lresult = capi.c_cppyy_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) ccpresult = rffi.cast(rffi.CCHARP, lresult) result = capi.charp2str_free(ccpresult) return space.wrap(result) Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Wed Jul 7 18:13:31 2010 @@ -7,9 +7,8 @@ #endif // ifdef __cplusplus void* cppyy_get_typehandle(const char* class_name); - long callstatic_l(void* handle, int method_index, int numargs, void* args[]); double callstatic_d(void* handle, int method_index, int numargs, void* args[]); - long callmethod_l(void* handle, int method_index, void* self, int numargs, void* args[]); + long cppyy_call_l(void* handle, int method_index, void* self, int numargs, void* args[]); void* construct(void* handle, int numargs, void* args[]); void destruct(void* handle, void* self); Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Wed Jul 7 18:13:31 2010 @@ -8,14 +8,6 @@ return Reflex::Type::ByName(class_name).Id(); } -long callstatic_l(void* handle, int method_index, int numargs, void* args[]) { - long result; - std::vector arguments(args, args+numargs); - Reflex::Type t((Reflex::TypeName*)handle); - Reflex::Member m = t.FunctionMemberAt(method_index); - m.Invoke(result, arguments); - return result; -} double callstatic_d(void* handle, int method_index, int numargs, void* args[]) { double result; std::vector arguments(args, args+numargs); @@ -25,14 +17,18 @@ return result; } -long callmethod_l(void* handle, int method_index, +long cppyy_call_l(void* handle, int method_index, void* self, int numargs, void* args[]) { long result; std::vector arguments(args, args+numargs); Reflex::Type t((Reflex::TypeName*)handle); - Reflex::Object o(t, self); Reflex::Member m = t.FunctionMemberAt(method_index); - m.Invoke(o, result, arguments); + if (self) { + Reflex::Object o(t, self); + m.Invoke(o, result, arguments); + } else { + m.Invoke(result, arguments); + } return result; } From benjamin at codespeak.net Wed Jul 7 18:14:25 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 18:14:25 +0200 (CEST) Subject: [pypy-svn] r75988 - pypy/branch/fast-forward/pypy/module/math Message-ID: <20100707161425.400F4282B9E@codespeak.net> Author: benjamin Date: Wed Jul 7 18:14:23 2010 New Revision: 75988 Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py Log: math functions will accept arguments with just a __float__ method Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Wed Jul 7 18:14:23 2010 @@ -10,9 +10,16 @@ self.w_e = space.wrap(math.e) self.w_pi = space.wrap(math.pi) def get(space): - return space.fromcache(State) + return space.fromcache(State) -def math1(space, f, x): +def _get_double(space, w_x): + if space.is_w(space.type(w_x), space.w_float): + return space.float_w(w_x) + else: + return space.float_w(space.float(w_x)) + +def math1(space, f, w_x): + x = _get_double(space, w_x) try: y = f(x) except OverflowError: @@ -24,7 +31,8 @@ return space.wrap(y) math1._annspecialcase_ = 'specialize:arg(1)' -def math1_w(space, f, x): +def math1_w(space, f, w_x): + x = _get_double(space, w_x) try: r = f(x) except OverflowError: @@ -36,7 +44,9 @@ return r math1_w._annspecialcase_ = 'specialize:arg(1)' -def math2(space, f, x, snd): +def math2(space, f, w_x, w_snd): + x = _get_double(space, w_x) + snd = _get_double(space, w_snd) try: r = f(x, snd) except OverflowError: @@ -53,41 +63,44 @@ return space.trunc(w_x) trunc.unwrap_spec = [ObjSpace, W_Root] -def copysign(space, x, y): +def copysign(space, w_x, w_y): """Return x with the sign of y.""" # No exceptions possible. + x = _get_double(space, w_x) + y = _get_double(space, w_y) return space.wrap(rarithmetic.copysign(x, y)) -copysign.unwrap_spec = [ObjSpace, float, float] +copysign.unwrap_spec = [ObjSpace, W_Root, W_Root] -def isinf(space, x): +def isinf(space, w_x): """Return True if x is infinity.""" - return space.wrap(rarithmetic.isinf(x)) -isinf.unwrap_spec = [ObjSpace, float] + return space.wrap(rarithmetic.isinf(_get_double(space, w_x))) +isinf.unwrap_spec = [ObjSpace, W_Root] -def isnan(space, x): +def isnan(space, w_x): """Return True if x is not a number.""" - return space.wrap(rarithmetic.isnan(x)) -isnan.unwrap_spec = [ObjSpace, float] + return space.wrap(rarithmetic.isnan(_get_double(space, w_x))) +isnan.unwrap_spec = [ObjSpace, W_Root] -def pow(space, x, y): +def pow(space, w_x, w_y): """pow(x,y) - + Return x**y (x to the power of y). """ - return math2(space, math.pow, x, y) -pow.unwrap_spec = [ObjSpace, float, float] + return math2(space, math.pow, w_x, w_y) +pow.unwrap_spec = [ObjSpace, W_Root, W_Root] -def cosh(space, x): +def cosh(space, w_x): """cosh(x) - + Return the hyperbolic cosine of x. """ - return math1(space, math.cosh, x) -cosh.unwrap_spec = [ObjSpace, float] + return math1(space, math.cosh, w_x) +cosh.unwrap_spec = [ObjSpace, W_Root] -def ldexp(space, x, w_i): +def ldexp(space, w_x, w_i): """ldexp(x, i) -> x * (2**i) """ + x = _get_double(space, w_x) if (space.isinstance_w(w_i, space.w_int) or space.isinstance_w(w_i, space.w_long)): try: @@ -102,76 +115,84 @@ else: raise OperationError(space.w_TypeError, space.wrap("integer required for second argument")) - return math2(space, math.ldexp, x, exp) -ldexp.unwrap_spec = [ObjSpace, float, W_Root] + try: + r = math.ldexp(x, exp) + except OverflowError: + raise OperationError(space.w_OverflowError, + space.wrap("math range error")) + except ValueError: + raise OperationError(space.w_ValueError, + space.wrap("math domain error")) + return space.wrap(r) +ldexp.unwrap_spec = [ObjSpace, W_Root, W_Root] -def hypot(space, x, y): +def hypot(space, w_x, w_y): """hypot(x,y) - + Return the Euclidean distance, sqrt(x*x + y*y). """ - return math2(space, math.hypot, x, y) -hypot.unwrap_spec = [ObjSpace, float, float] + return math2(space, math.hypot, w_x, w_y) +hypot.unwrap_spec = [ObjSpace, W_Root, W_Root] -def tan(space, x): +def tan(space, w_x): """tan(x) - + Return the tangent of x (measured in radians). """ - return math1(space, math.tan, x) -tan.unwrap_spec = [ObjSpace, float] + return math1(space, math.tan, w_x) +tan.unwrap_spec = [ObjSpace, W_Root] -def asin(space, x): +def asin(space, w_x): """asin(x) - + Return the arc sine (measured in radians) of x. """ - return math1(space, math.asin, x) -asin.unwrap_spec = [ObjSpace, float] + return math1(space, math.asin, w_x) +asin.unwrap_spec = [ObjSpace, W_Root] -def fabs(space, x): +def fabs(space, w_x): """fabs(x) - + Return the absolute value of the float x. """ - return math1(space, math.fabs, x) -fabs.unwrap_spec = [ObjSpace, float] + return math1(space, math.fabs, w_x) +fabs.unwrap_spec = [ObjSpace, W_Root] -def floor(space, x): +def floor(space, w_x): """floor(x) - + Return the floor of x as a float. This is the largest integral value <= x. """ - return math1(space, math.floor, x) -floor.unwrap_spec = [ObjSpace, float] + return math1(space, math.floor, w_x) +floor.unwrap_spec = [ObjSpace, W_Root] -def sqrt(space, x): +def sqrt(space, w_x): """sqrt(x) - + Return the square root of x. """ - return math1(space, math.sqrt, x) -sqrt.unwrap_spec = [ObjSpace, float] + return math1(space, math.sqrt, w_x) +sqrt.unwrap_spec = [ObjSpace, W_Root] -def frexp(space, x): +def frexp(space, w_x): """frexp(x) - + Return the mantissa and exponent of x, as pair (m, e). m is a float and e is an int, such that x = m * 2.**e. If x is 0, m and e are both 0. Else 0.5 <= abs(m) < 1.0. """ - mant, expo = math1_w(space, math.frexp, x) + mant, expo = math1_w(space, math.frexp, w_x) return space.newtuple([space.wrap(mant), space.wrap(expo)]) -frexp.unwrap_spec = [ObjSpace, float] +frexp.unwrap_spec = [ObjSpace, W_Root] degToRad = math.pi / 180.0 -def degrees(space, x): +def degrees(space, w_x): """degrees(x) -> converts angle x from radians to degrees """ - return space.wrap(x / degToRad) -degrees.unwrap_spec = [ObjSpace, float] + return space.wrap(_get_double(space, w_x) / degToRad) +degrees.unwrap_spec = [ObjSpace, W_Root] def _log_any(space, w_x, base): # base is supposed to be positive or 0.0, which means we use e @@ -181,11 +202,11 @@ num = space.bigint_w(w_x) result = num.log(base) else: - x = space.float_w(w_x) + x = _get_double(space, w_x) if base == 10.0: result = math.log10(x) else: - result = math.log(x) + result = math.log(x) if base != 0.0: den = math.log(base) result /= den @@ -204,116 +225,116 @@ if w_base is None: base = 0.0 else: - base = space.float_w(w_base) + base = _get_double(space, w_base) if base <= 0.0: # just for raising the proper errors return math1(space, math.log, base) return _log_any(space, w_x, base) log.unwrap_spec = [ObjSpace, W_Root, W_Root] -def log10(space, w_x): +def log10(space, w_x): """log10(x) -> the base 10 logarithm of x. - """ + """ return _log_any(space, w_x, 10.0) log10.unwrap_spec = [ObjSpace, W_Root] -def fmod(space, x, y): +def fmod(space, w_x, w_y): """fmod(x,y) - + Return fmod(x, y), according to platform C. x % y may differ. """ - return math2(space, math.fmod, x, y) -fmod.unwrap_spec = [ObjSpace, float, float] + return math2(space, math.fmod, w_x, w_y) +fmod.unwrap_spec = [ObjSpace, W_Root, W_Root] -def atan(space, x): +def atan(space, w_x): """atan(x) - + Return the arc tangent (measured in radians) of x. """ - return math1(space, math.atan, x) -atan.unwrap_spec = [ObjSpace, float] + return math1(space, math.atan, w_x) +atan.unwrap_spec = [ObjSpace, W_Root] -def ceil(space, x): +def ceil(space, w_x): """ceil(x) - + Return the ceiling of x as a float. This is the smallest integral value >= x. """ - return math1(space, math.ceil, x) -ceil.unwrap_spec = [ObjSpace, float] + return math1(space, math.ceil, w_x) +ceil.unwrap_spec = [ObjSpace, W_Root] -def sinh(space, x): +def sinh(space, w_x): """sinh(x) - + Return the hyperbolic sine of x. """ - return math1(space, math.sinh, x) -sinh.unwrap_spec = [ObjSpace, float] + return math1(space, math.sinh, w_x) +sinh.unwrap_spec = [ObjSpace, W_Root] -def cos(space, x): +def cos(space, w_x): """cos(x) - + Return the cosine of x (measured in radians). """ - return math1(space, math.cos, x) -cos.unwrap_spec = [ObjSpace, float] + return math1(space, math.cos, w_x) +cos.unwrap_spec = [ObjSpace, W_Root] -def tanh(space, x): +def tanh(space, w_x): """tanh(x) - + Return the hyperbolic tangent of x. """ - return math1(space, math.tanh, x) -tanh.unwrap_spec = [ObjSpace, float] + return math1(space, math.tanh, w_x) +tanh.unwrap_spec = [ObjSpace, W_Root] -def radians(space, x): +def radians(space, w_x): """radians(x) -> converts angle x from degrees to radians """ - return space.wrap(x * degToRad) -radians.unwrap_spec = [ObjSpace, float] + return space.wrap(_get_double(space, w_x) * degToRad) +radians.unwrap_spec = [ObjSpace, W_Root] -def sin(space, x): +def sin(space, w_x): """sin(x) - + Return the sine of x (measured in radians). """ - return math1(space, math.sin, x) -sin.unwrap_spec = [ObjSpace, float] + return math1(space, math.sin, w_x) +sin.unwrap_spec = [ObjSpace, W_Root] -def atan2(space, y, x): +def atan2(space, w_y, w_x): """atan2(y, x) - + Return the arc tangent (measured in radians) of y/x. Unlike atan(y/x), the signs of both x and y are considered. """ - return math2(space, math.atan2, y, x) -atan2.unwrap_spec = [ObjSpace, float, float] + return math2(space, math.atan2, w_y, w_x) +atan2.unwrap_spec = [ObjSpace, W_Root, W_Root] -def modf(space, x): +def modf(space, w_x): """modf(x) - + Return the fractional and integer parts of x. Both results carry the sign of x. The integer part is returned as a real. """ - frac, intpart = math1_w(space, math.modf, x) + frac, intpart = math1_w(space, math.modf, w_x) return space.newtuple([space.wrap(frac), space.wrap(intpart)]) -modf.unwrap_spec = [ObjSpace, float] +modf.unwrap_spec = [ObjSpace, W_Root] -def exp(space, x): +def exp(space, w_x): """exp(x) - + Return e raised to the power of x. """ - return math1(space, math.exp, x) -exp.unwrap_spec = [ObjSpace, float] + return math1(space, math.exp, w_x) +exp.unwrap_spec = [ObjSpace, W_Root] -def acos(space, x): +def acos(space, w_x): """acos(x) - + Return the arc cosine (measured in radians) of x. """ - return math1(space, math.acos, x) -acos.unwrap_spec = [ObjSpace, float] + return math1(space, math.acos, w_x) +acos.unwrap_spec = [ObjSpace, W_Root] def fsum(space, w_iterable): """Sum an iterable of floats, trying to keep precision.""" @@ -327,7 +348,7 @@ if not e.match(space, space.w_StopIteration): raise break - v = space.float_w(w_value) + v = _get_double(space, w_value) original = v added = 0 for y in partials: @@ -397,50 +418,50 @@ w_res = space.mul(w_res, space.wrap(i)) return w_res -def log1p(space, x): +def log1p(space, w_x): """Find log(x + 1).""" - return math1(space, rarithmetic.log1p, x) -log1p.unwrap_spec = [ObjSpace, float] + return math1(space, rarithmetic.log1p, w_x) +log1p.unwrap_spec = [ObjSpace, W_Root] -def acosh(space, x): +def acosh(space, w_x): """Inverse hyperbolic cosine""" - return math1(space, rarithmetic.acosh, x) -acosh.unwrap_spec = [ObjSpace, float] + return math1(space, rarithmetic.acosh, w_x) +acosh.unwrap_spec = [ObjSpace, W_Root] -def asinh(space, x): +def asinh(space, w_x): """Inverse hyperbolic sine""" - return math1(space, rarithmetic.asinh, x) -asinh.unwrap_spec = [ObjSpace, float] + return math1(space, rarithmetic.asinh, w_x) +asinh.unwrap_spec = [ObjSpace, W_Root] -def atanh(space, x): +def atanh(space, w_x): """Inverse hyperbolic tangent""" - return math1(space, rarithmetic.atanh, x) -atanh.unwrap_spec = [ObjSpace, float] + return math1(space, rarithmetic.atanh, w_x) +atanh.unwrap_spec = [ObjSpace, W_Root] -def expm1(space, x): +def expm1(space, w_x): """exp(x) - 1""" - return math1(space, rarithmetic.expm1, x) -expm1.unwrap_spec = [ObjSpace, float] + return math1(space, rarithmetic.expm1, w_x) +expm1.unwrap_spec = [ObjSpace, W_Root] -def erf(space, x): +def erf(space, w_x): """The error function""" - return math1(space, _erf, x) -erf.unwrap_spec = [ObjSpace, float] + return math1(space, _erf, w_x) +erf.unwrap_spec = [ObjSpace, W_Root] -def erfc(space, x): +def erfc(space, w_x): """The complementary error function""" - return math1(space, _erfc, x) -erfc.unwrap_spec = [ObjSpace, float] + return math1(space, _erfc, w_x) +erfc.unwrap_spec = [ObjSpace, W_Root] -def gamma(space, x): +def gamma(space, w_x): """Compute the gamma function for x.""" - return math1(space, _gamma, x) -gamma.unwrap_spec = [ObjSpace, float] + return math1(space, _gamma, w_x) +gamma.unwrap_spec = [ObjSpace, W_Root] -def lgamma(space, x): +def lgamma(space, w_x): """Compute the natural logarithm of the gamma function for x.""" - return math1(space, _lgamma, x) -lgamma.unwrap_spec = [ObjSpace, float] + return math1(space, _lgamma, w_x) +lgamma.unwrap_spec = [ObjSpace, W_Root] # Implementation of the error function, the complimentary error function, the # gamma function, and the natural log of the gamma function. This exist in From arigo at codespeak.net Wed Jul 7 18:14:55 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 18:14:55 +0200 (CEST) Subject: [pypy-svn] r75989 - in pypy/branch/rsre2/pypy/rlib/rsre_jit: . test Message-ID: <20100707161455.75E55282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 18:14:53 2010 New Revision: 75989 Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_match.py Log: Merge from rsre. Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py Wed Jul 7 18:14:53 2010 @@ -369,6 +369,7 @@ ptr1 = ptr - ctx.pat(ppos+1) if ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks) is not None: return + marks = ctx.match_marks ppos += ctx.pat(ppos) elif op == OPCODE_AT: Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre_jit/test/test_match.py Wed Jul 7 18:14:53 2010 @@ -195,3 +195,19 @@ assert rsre.match(r, "xyxyxyB") is not None r = get_code(r"(?:xy(?:xy)+?)+?B") assert rsre.match(r, "xyxyxyB") is not None + + def test_assert_group(self): + r = get_code(r"abc(?=(..)f)(.)") + res = rsre.match(r, "abcdefghi") + assert res is not None + assert res.span(2) == (3, 4) + assert res.span(1) == (3, 5) + + def test_assert_not_group(self): + r = get_code(r"abc(?!(de)f)(.)") + res = rsre.match(r, "abcdeFghi") + assert res is not None + assert res.span(2) == (3, 4) + # this I definitely classify as Horrendously Implementation Dependent. + # CPython answers (3, 5). + assert res.span(1) == (-1, -1) From arigo at codespeak.net Wed Jul 7 18:16:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 18:16:10 +0200 (CEST) Subject: [pypy-svn] r75990 - pypy/branch/rsre2/pypy/rlib/rsre_jit Message-ID: <20100707161610.53A1F282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 18:16:08 2010 New Revision: 75990 Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py Log: Argh. Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py Wed Jul 7 18:16:08 2010 @@ -361,6 +361,7 @@ ptr1 = ptr - ctx.pat(ppos+1) if ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None: return + marks = ctx.match_marks ppos += ctx.pat(ppos) elif op == OPCODE_ASSERT_NOT: @@ -369,7 +370,6 @@ ptr1 = ptr - ctx.pat(ppos+1) if ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks) is not None: return - marks = ctx.match_marks ppos += ctx.pat(ppos) elif op == OPCODE_AT: From antocuni at codespeak.net Wed Jul 7 18:28:03 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 7 Jul 2010 18:28:03 +0200 (CEST) Subject: [pypy-svn] r75991 - in pypy/branch/reflex-support/pypy/module/cppyy: include src Message-ID: <20100707162803.F1E13282B9E@codespeak.net> Author: antocuni Date: Wed Jul 7 18:28:02 2010 New Revision: 75991 Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Log: lookup and print the property "MethPtrGetter" for methods; so far it's unused and it's always empty, because genreflex.py does not generate this information yet. Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h Wed Jul 7 18:28:02 2010 @@ -5,5 +5,6 @@ #include "Reflex/Member.h" #include "Reflex/Object.h" #include "Reflex/Builder/TypeBuilder.h" +#include "Reflex/PropertyList.h" #endif // CPPYY_CPPYY Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Wed Jul 7 18:28:02 2010 @@ -50,12 +50,26 @@ t.Destruct(self, true); } +typedef void* (*MethPtrGetter)(void*); +static MethPtrGetter get_methptr_getter(Reflex::Member m) +{ + Reflex::PropertyList plist = m.Properties(); + if (plist.HasProperty("MethPtrGetter")) { + Reflex::Any& value = plist.PropertyValue("MethPtrGetter"); + return (MethPtrGetter)Reflex::any_cast(value); + } + else + return 0; +} + int num_methods(void* handle) { Reflex::Type t((Reflex::TypeName*)handle); for (int i = 0; i < (int)t.FunctionMemberSize(); i++) { Reflex::Member m = t.FunctionMemberAt(i); std::cout << i << " " << m.Name() << std::endl; + std::cout << " " << "Stubfunction: " << (void*)m.Stubfunction() << std::endl; + std::cout << " " << "MethPtrGetter: " << (void*)get_methptr_getter(m) << std::endl; for (int j = 0; j < (int)m.FunctionParameterSize(); j++) { Reflex::Type at = m.TypeOf().FunctionParameterAt(j); std::cout << " " << j << " " << at.Name() << std::endl; From arigo at codespeak.net Wed Jul 7 18:30:35 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 18:30:35 +0200 (CEST) Subject: [pypy-svn] r75992 - pypy/branch/rsre2/pypy/rlib/rsre Message-ID: <20100707163035.2874E282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 18:30:33 2010 New Revision: 75992 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Log: Change the return value of find_first_result() & co, allowing the calls from sre_match() to be tail calls. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Wed Jul 7 18:30:33 2010 @@ -136,9 +136,9 @@ def move_to_next_result(self, ctx): result = self.subresult if result is None: - return False + return if result.move_to_next_result(ctx): - return True + return result return self.find_next_result(ctx) def find_next_result(self, ctx): @@ -161,8 +161,7 @@ if result is not None: self.subresult = result self.ppos = ppos - return True - return False + return self find_next_result = find_first_result class RepeatOneMatchResult(MatchResult): @@ -181,8 +180,7 @@ if result is not None: self.subresult = result self.start_ptr = ptr - return True - return False + return self find_next_result = find_first_result @@ -202,18 +200,17 @@ if result is not None: self.subresult = result self.start_ptr = ptr - return True + return self ptr1 = find_repetition_end(ctx, self.ppos3, ptr, 1) if ptr1 == ptr: break ptr = ptr1 - return False def find_next_result(self, ctx): ptr = self.start_ptr ptr1 = find_repetition_end(ctx, self.ppos3, ptr, 1) if ptr1 == ptr: - return False + return self.start_ptr = ptr1 return self.find_first_result(ctx) @@ -258,13 +255,12 @@ self.subresult = result self.cur_ptr = ptr self.cur_marks = marks - return True + return self resume = False if len(self.pending) == 0: - return False + return ptr, marks, enum = self.pending.pop() - if not enum.move_to_next_result(ctx): - enum = None + enum = enum.move_to_next_result(ctx) # if max == 65535 or len(self.pending) < max: # try to match one more 'item' @@ -294,7 +290,7 @@ self.subresult = result self.cur_ptr = ptr self.cur_marks = marks - return True + return self resume = False if max == 65535 or len(self.pending) < max: @@ -313,10 +309,9 @@ else: # 'item' no longer matches. if len(self.pending) == 0: - return False + return ptr, marks, enum = self.pending.pop() - if not enum.move_to_next_result(ctx): - enum = None + enum = enum.move_to_next_result(ctx) continue break @@ -383,9 +378,7 @@ # alternation # <0=skip> code ... result = BranchMatchResult(ppos, ptr, marks) - if not result.find_first_result(ctx): - result = None - return result + return result.find_first_result(ctx) #elif op == OPCODE_CATEGORY: # seems to be never produced @@ -508,17 +501,13 @@ # remembering each state for each possible number of # 'item' matching. result = MaxUntilMatchResult(ppos, tailppos, ptr, marks) - if not result.find_first_result(ctx): - result = None - return result + return result.find_first_result(ctx) elif op == OPCODE_MIN_UNTIL: # first try to match the 'tail', and if it fails, try # to match one more 'item' and try again result = MinUntilMatchResult(ppos, tailppos, ptr, marks) - if not result.find_first_result(ctx): - result = None - return result + return result.find_first_result(ctx) else: raise AssertionError("missing UNTIL after REPEAT") @@ -540,9 +529,7 @@ # and backtrack if not. nextppos = ppos + ctx.pat(ppos) result = RepeatOneMatchResult(nextppos, minptr, ptr, marks) - if not result.find_first_result(ctx): - result = None - return result + return result.find_first_result(ctx) elif op == OPCODE_MIN_REPEAT_ONE: # match repeated sequence (minimizing regexp). @@ -571,9 +558,7 @@ nextppos = ppos + ctx.pat(ppos) result = MinRepeatOneMatchResult(nextppos, ppos+3, maxptr, ptr, marks) - if not result.find_first_result(ctx): - result = None - return result + return result.find_first_result(ctx) else: assert 0, "bad pattern code %d" % op From arigo at codespeak.net Wed Jul 7 18:30:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 18:30:45 +0200 (CEST) Subject: [pypy-svn] r75993 - pypy/branch/rsre2/pypy/rlib/rsre_jit Message-ID: <20100707163045.A66F5282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 18:30:44 2010 New Revision: 75993 Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py Log: Merge from ../rsre. Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py Wed Jul 7 18:30:44 2010 @@ -136,9 +136,9 @@ def move_to_next_result(self, ctx): result = self.subresult if result is None: - return False + return if result.move_to_next_result(ctx): - return True + return result return self.find_next_result(ctx) def find_next_result(self, ctx): @@ -161,8 +161,7 @@ if result is not None: self.subresult = result self.ppos = ppos - return True - return False + return self find_next_result = find_first_result class RepeatOneMatchResult(MatchResult): @@ -181,8 +180,7 @@ if result is not None: self.subresult = result self.start_ptr = ptr - return True - return False + return self find_next_result = find_first_result @@ -202,18 +200,17 @@ if result is not None: self.subresult = result self.start_ptr = ptr - return True + return self ptr1 = find_repetition_end(ctx, self.ppos3, ptr, 1) if ptr1 == ptr: break ptr = ptr1 - return False def find_next_result(self, ctx): ptr = self.start_ptr ptr1 = find_repetition_end(ctx, self.ppos3, ptr, 1) if ptr1 == ptr: - return False + return self.start_ptr = ptr1 return self.find_first_result(ctx) @@ -258,13 +255,12 @@ self.subresult = result self.cur_ptr = ptr self.cur_marks = marks - return True + return self resume = False if len(self.pending) == 0: - return False + return ptr, marks, enum = self.pending.pop() - if not enum.move_to_next_result(ctx): - enum = None + enum = enum.move_to_next_result(ctx) # if max == 65535 or len(self.pending) < max: # try to match one more 'item' @@ -294,7 +290,7 @@ self.subresult = result self.cur_ptr = ptr self.cur_marks = marks - return True + return self resume = False if max == 65535 or len(self.pending) < max: @@ -313,10 +309,9 @@ else: # 'item' no longer matches. if len(self.pending) == 0: - return False + return ptr, marks, enum = self.pending.pop() - if not enum.move_to_next_result(ctx): - enum = None + enum = enum.move_to_next_result(ctx) continue break @@ -383,9 +378,7 @@ # alternation # <0=skip> code ... result = BranchMatchResult(ppos, ptr, marks) - if not result.find_first_result(ctx): - result = None - return result + return result.find_first_result(ctx) #elif op == OPCODE_CATEGORY: # seems to be never produced @@ -508,17 +501,13 @@ # remembering each state for each possible number of # 'item' matching. result = MaxUntilMatchResult(ppos, tailppos, ptr, marks) - if not result.find_first_result(ctx): - result = None - return result + return result.find_first_result(ctx) elif op == OPCODE_MIN_UNTIL: # first try to match the 'tail', and if it fails, try # to match one more 'item' and try again result = MinUntilMatchResult(ppos, tailppos, ptr, marks) - if not result.find_first_result(ctx): - result = None - return result + return result.find_first_result(ctx) else: raise AssertionError("missing UNTIL after REPEAT") @@ -540,9 +529,7 @@ # and backtrack if not. nextppos = ppos + ctx.pat(ppos) result = RepeatOneMatchResult(nextppos, minptr, ptr, marks) - if not result.find_first_result(ctx): - result = None - return result + return result.find_first_result(ctx) elif op == OPCODE_MIN_REPEAT_ONE: # match repeated sequence (minimizing regexp). @@ -571,9 +558,7 @@ nextppos = ppos + ctx.pat(ppos) result = MinRepeatOneMatchResult(nextppos, ppos+3, maxptr, ptr, marks) - if not result.find_first_result(ctx): - result = None - return result + return result.find_first_result(ctx) else: assert 0, "bad pattern code %d" % op From arigo at codespeak.net Wed Jul 7 18:42:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 18:42:37 +0200 (CEST) Subject: [pypy-svn] r75994 - pypy/branch/rsre2/pypy/rlib/rsre Message-ID: <20100707164237.683DF282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 18:42:35 2010 New Revision: 75994 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Log: Minor simplification of logic. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre.py Wed Jul 7 18:42:35 2010 @@ -299,21 +299,18 @@ else: enum = None # 'max' reached, no more matches - while True: - if enum is not None: - # matched one more 'item'. record it and continue - self.pending.append((ptr, marks, enum)) - ptr = ctx.match_end - marks = ctx.match_marks - break - else: - # 'item' no longer matches. - if len(self.pending) == 0: - return - ptr, marks, enum = self.pending.pop() - enum = enum.move_to_next_result(ctx) - continue - break + while enum is None: + # 'item' does not match; try to get further results from + # the 'pending' list. + if len(self.pending) == 0: + return + ptr, marks, enum = self.pending.pop() + enum = enum.move_to_next_result(ctx) + + # matched one more 'item'. record it and continue + self.pending.append((ptr, marks, enum)) + ptr = ctx.match_end + marks = ctx.match_marks # ____________________________________________________________ From arigo at codespeak.net Wed Jul 7 18:42:48 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 18:42:48 +0200 (CEST) Subject: [pypy-svn] r75995 - pypy/branch/rsre2/pypy/rlib/rsre_jit Message-ID: <20100707164248.D44F0282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 18:42:47 2010 New Revision: 75995 Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py Log: Merge from ../rsre. Modified: pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre_jit/rsre.py Wed Jul 7 18:42:47 2010 @@ -299,21 +299,18 @@ else: enum = None # 'max' reached, no more matches - while True: - if enum is not None: - # matched one more 'item'. record it and continue - self.pending.append((ptr, marks, enum)) - ptr = ctx.match_end - marks = ctx.match_marks - break - else: - # 'item' no longer matches. - if len(self.pending) == 0: - return - ptr, marks, enum = self.pending.pop() - enum = enum.move_to_next_result(ctx) - continue - break + while enum is None: + # 'item' does not match; try to get further results from + # the 'pending' list. + if len(self.pending) == 0: + return + ptr, marks, enum = self.pending.pop() + enum = enum.move_to_next_result(ctx) + + # matched one more 'item'. record it and continue + self.pending.append((ptr, marks, enum)) + ptr = ctx.match_end + marks = ctx.match_marks # ____________________________________________________________ From arigo at codespeak.net Wed Jul 7 20:02:14 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Jul 2010 20:02:14 +0200 (CEST) Subject: [pypy-svn] r75997 - pypy/branch/kill-caninline Message-ID: <20100707180214.4D988282B9E@codespeak.net> Author: arigo Date: Wed Jul 7 20:02:13 2010 New Revision: 75997 Added: pypy/branch/kill-caninline/ - copied from r75996, pypy/trunk/ Log: A branch in which to try to remove 'can_inline' from the JitDriver, and detect automatically when we are tracing an inlined function and hitting a loop. From wlav at codespeak.net Wed Jul 7 20:04:05 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Wed, 7 Jul 2010 20:04:05 +0200 (CEST) Subject: [pypy-svn] r75998 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src test Message-ID: <20100707180405.459B2282B9E@codespeak.net> Author: wlav Date: Wed Jul 7 20:04:02 2010 New Revision: 75998 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Log: Support calls for double and char* on instances. Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Wed Jul 7 20:04:02 2010 @@ -31,20 +31,20 @@ "callstatic_l", [rffi.VOIDP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) -c_callstatic_d = rffi.llexternal( - "callstatic_d", - [rffi.VOIDP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.DOUBLE, - compilation_info=eci) -c_construct = rffi.llexternal( - "construct", +c_cppyy_construct = rffi.llexternal( + "cppyy_construct", [rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.VOIDP, compilation_info=eci) c_cppyy_call_l = rffi.llexternal( "cppyy_call_l", [rffi.VOIDP, rffi.INT, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) -c_destruct = rffi.llexternal( - "destruct", +c_cppyy_call_d = rffi.llexternal( + "cppyy_call_d", + [rffi.VOIDP, rffi.INT, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.DOUBLE, + compilation_info=eci) +c_cppyy_destruct = rffi.llexternal( + "cppyy_destruct", [rffi.VOIDP, rffi.VOIDP], lltype.Void, compilation_info=eci) Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Wed Jul 7 20:04:02 2010 @@ -12,7 +12,7 @@ class IntConverter(TypeConverter): def convert_argument(self, space, w_obj): - arg = space.int_w(w_obj) + arg = space.c_int_w(w_obj) x = lltype.malloc(rffi.LONGP.TO, 1, flavor='raw') x[0] = arg return rffi.cast(rffi.VOIDP, x) Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Wed Jul 7 20:04:02 2010 @@ -17,18 +17,12 @@ class DoubleExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): - if cppthis is not None: - raise NotImplementedError - else: - result = capi.c_callstatic_d(func.cpptype.handle, func.method_index, num_args, args) + result = capi.c_cppyy_call_d(func.cpptype.handle, func.method_index, cppthis, num_args, args) return space.wrap(result) class CStringExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): - if cppthis is not None: - raise NotImplementedError - else: - lresult = capi.c_cppyy_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) + lresult = capi.c_cppyy_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) ccpresult = rffi.cast(rffi.CCHARP, lresult) result = capi.charp2str_free(ccpresult) return space.wrap(result) Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Wed Jul 7 20:04:02 2010 @@ -7,10 +7,10 @@ #endif // ifdef __cplusplus void* cppyy_get_typehandle(const char* class_name); - double callstatic_d(void* handle, int method_index, int numargs, void* args[]); + void* cppyy_construct(void* handle, int numargs, void* args[]); long cppyy_call_l(void* handle, int method_index, void* self, int numargs, void* args[]); - void* construct(void* handle, int numargs, void* args[]); - void destruct(void* handle, void* self); + double cppyy_call_d(void* handle, int method_index, void* self, int numargs, void* args[]); + void cppyy_destruct(void* handle, void* self); int num_methods(void* handle); char* method_name(void* handle, int method_index); Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Wed Jul 7 20:04:02 2010 @@ -113,7 +113,7 @@ def call(self, cppthis, args_w): assert not cppthis args = self.prepare_arguments(args_w) - result = capi.c_construct(self.cpptype.handle, len(args_w), args) + result = capi.c_cppyy_construct(self.cpptype.handle, len(args_w), args) self.free_arguments(args) return W_CPPObject(self.cpptype, result) @@ -208,13 +208,19 @@ self.cppclass = cppclass self.rawobject = rawobject + def _nullcheck(self): + if not self.rawobject: + raise OperationError(self.space.w_ReferenceError, self.space.wrap("trying to access a NULL pointer")) + def invoke(self, method_name, args_w): + self._nullcheck() cppclass = jit.hint(self.cppclass, promote=True) overload = cppclass.get_overload(method_name) return overload.call(self.rawobject, args_w) def destruct(self): - capi.c_destruct(self.cppclass.handle, self.rawobject) + capi.c_cppyy_destruct(self.cppclass.handle, self.rawobject) + self.rawobject = NULL_VOIDP W_CPPObject.typedef = TypeDef( 'CPPObject', Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Wed Jul 7 20:04:02 2010 @@ -8,13 +8,18 @@ return Reflex::Type::ByName(class_name).Id(); } -double callstatic_d(void* handle, int method_index, int numargs, void* args[]) { - double result; + +void* cppyy_construct(void* handle, int numargs, void* args[]) { std::vector arguments(args, args+numargs); Reflex::Type t((Reflex::TypeName*)handle); - Reflex::Member m = t.FunctionMemberAt(method_index); - m.Invoke(result, arguments); - return result; + std::vector argtypes; + argtypes.reserve(numargs); + for (int i = 0; i < numargs; i++) { + argtypes.push_back(Reflex::Type::ByName("int")); + } + Reflex::Type constructor_type = Reflex::FunctionTypeBuilder( + Reflex::Type::ByName("void"), argtypes); + return t.Construct(constructor_type, arguments).Address(); } long cppyy_call_l(void* handle, int method_index, @@ -32,20 +37,22 @@ return result; } -void* construct(void* handle, int numargs, void* args[]) { +double cppyy_call_d(void* handle, int method_index, + void* self, int numargs, void* args[]) { + double result; std::vector arguments(args, args+numargs); Reflex::Type t((Reflex::TypeName*)handle); - std::vector argtypes; - argtypes.reserve(numargs); - for (int i = 0; i < numargs; i++) { - argtypes.push_back(Reflex::Type::ByName("int")); + Reflex::Member m = t.FunctionMemberAt(method_index); + if (self) { + Reflex::Object o(t, self); + m.Invoke(o, result, arguments); + } else { + m.Invoke(result, arguments); } - Reflex::Type constructor_type = Reflex::FunctionTypeBuilder( - Reflex::Type::ByName("void"), argtypes); - return t.Construct(constructor_type, arguments).Address(); -} + return result; +} -void destruct(void* handle, void* self) { +void cppyy_destruct(void* handle, void* self) { Reflex::Type t((Reflex::TypeName*)handle); t.Destruct(self, true); } Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx Wed Jul 7 20:04:02 2010 @@ -1,4 +1,6 @@ #include +#include +#include #include #include @@ -28,31 +30,53 @@ count--; } - static int add1(int a) { +// class methods + static int staticAddOneToInt(int a) { return a + 1; } - static int add1(int a, int b) { + static int staticAddOneToInt(int a, int b) { return a + b + 1; } - static double adddouble(double a) { + static double staticAddToDouble(double a) { return a + 0.01; } - static int atoi(const char* str) { + static int staticAtoi(const char* str) { return ::atoi(str); } - static char* strcpy(const char* strin) { + static char* staticStrcpy(const char* strin) { char* strout = (char*)malloc(::strlen(strin + 1)); ::strcpy(strout, strin); return strout; } - static int getcount() { + static int getCount() { std::cout << "getcount called" << std::endl; return count; } - int add(int a) { + +// instance methods + int addDataToInt(int a) { + return somedata + a; + } + + double addDataToDouble(double a) { return somedata + a; } + + int addDataToAtoi(const char* str) { + return ::atoi(str) + somedata; + } + + char* addToStringValue(const char* str) { + int out = ::atoi(str) + somedata; + std::ostringstream ss; + ss << out << std::ends; + std::string result = ss.str(); + char* cresult = (char*)malloc(result.size()+1); + ::strcpy(cresult, result.c_str()); + return cresult; + } + }; int example01::count = 0; Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Wed Jul 7 20:04:02 2010 @@ -1,5 +1,4 @@ -import py -import os +import py, os from pypy.conftest import gettestobjspace from pypy.module.cppyy import interp_cppyy, executor @@ -16,7 +15,7 @@ def test_class_query(self): lib = interp_cppyy.load_lib(space, shared_lib) w_cppyyclass = lib.type_byname("example01") - adddouble = w_cppyyclass.function_members["adddouble"] + adddouble = w_cppyyclass.function_members["staticAddToDouble"] func, = adddouble.functions assert isinstance(func.executor, executor.DoubleExecutor) assert func.arg_types == ["double"] @@ -30,40 +29,95 @@ import cppyy return cppyy.load_lib(%r)""" % (shared_lib, )) - def test_example01static(self): + def test_example01static_int(self): + """Test passing of an int, returning of an int, and overloading on a + differening number of arguments.""" + import sys t = self.example01.type_byname("example01") - # also tests overloading by number of args - res = t.invoke("add1", 1) + res = t.invoke("staticAddOneToInt", 1) assert res == 2 - res = t.invoke("add1", 1, 2) + res = t.invoke("staticAddOneToInt", 1L) + assert res == 2 + res = t.invoke("staticAddOneToInt", 1, 2) assert res == 4 - raises(TypeError, 't.invoke("add1", 1, [])') + res = t.invoke("staticAddOneToInt", -1) + assert res == 0 + res = t.invoke("staticAddOneToInt", sys.maxint-1) + assert res == sys.maxint + res = t.invoke("staticAddOneToInt", sys.maxint) + assert res == -sys.maxint-1 + + raises(TypeError, 't.invoke("staticAddOneToInt", 1, [])') + raises(TypeError, 't.invoke("staticAddOneToInt", 1.)') + raises(OverflowError, 't.invoke("staticAddOneToInt", sys.maxint+1)') def test_example01static_double(self): + """Test passing of a double and returning of a double on a static function.""" t = self.example01.type_byname("example01") - res = t.invoke("adddouble", 0.09) + res = t.invoke("staticAddToDouble", 0.09) assert res == 0.09 + 0.01 def test_example01static_constcharp(self): + """Test passing of a C string and returning of a C string on a static + function.""" t = self.example01.type_byname("example01") - res = t.invoke("atoi", "1") + res = t.invoke("staticAtoi", "1") assert res == 1 - res = t.invoke("strcpy", "aap") + res = t.invoke("staticStrcpy", "aap") + assert res == "aap" + + res = t.invoke("staticStrcpy", u"aap") assert res == "aap" - def test_example01method(self): + raises(TypeError, 't.invoke("staticStrcpy", 1.)') + + def test_example01method_int(self): + """Test passing of a int, returning of a int, and memory cleanup, on + a method.""" t = self.example01.type_byname("example01") - count = t.invoke("getcount") - assert count == 0 + assert t.invoke("getCount") == 0 instance = t.construct(7) - count = t.invoke("getcount") - assert count == 1 - res = instance.invoke("add", 4) + assert t.invoke("getCount") == 1 + res = instance.invoke("addDataToInt", 4) assert res == 11 + res = instance.invoke("addDataToInt", -4) + assert res == 3 + instance.destruct() + assert t.invoke("getCount") == 0 + raises(ReferenceError, 'instance.invoke("addDataToInt", 4)') + + instance = t.construct(7) + instance2 = t.construct(8) + assert t.invoke("getCount") == 2 instance.destruct() - count = t.invoke("getcount") - assert count == 0 + assert t.invoke("getCount") == 1 + instance2.destruct() + assert t.invoke("getCount") == 0 + + def test_example01method_double(self): + """Test passing of a double and returning of double on a method""" + t = self.example01.type_byname("example01") + instance = t.construct(13) + res = instance.invoke("addDataToDouble", 16) + assert round(res-29, 8) == 0. + instance = t.construct(-13) + res = instance.invoke("addDataToDouble", 16) + assert round(res-3, 8) == 0. + + def test_example01method_constcharp(self): + """Test passing of a C string and returning of a C string on a + method.""" + + t = self.example01.type_byname("example01") + instance = t.construct(42) + + res = instance.invoke("addDataToAtoi", "13") + assert res == 55 + res = instance.invoke("addToStringValue", "12") + assert res == "54" + res = instance.invoke("addToStringValue", "-12") + assert res == "30" From benjamin at codespeak.net Wed Jul 7 20:34:00 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 20:34:00 +0200 (CEST) Subject: [pypy-svn] r75999 - in pypy/branch/fast-forward/pypy: module/struct/test rlib/rstruct Message-ID: <20100707183400.29D6F282B9E@codespeak.net> Author: benjamin Date: Wed Jul 7 20:33:57 2010 New Revision: 75999 Modified: pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Log: complain when float is too large for 'f' Modified: pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py (original) +++ pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py Wed Jul 7 20:33:57 2010 @@ -260,7 +260,7 @@ assert pack("= MAX_EXP - MIN_EXP + 2: - raise OverflowError("float too large to pack in this format") + raise ValueError("float too large to pack in this format") # check constraints assert 0 <= mant < 1 << MANT_DIG - 1 Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Wed Jul 7 20:33:57 2010 @@ -57,10 +57,14 @@ fmtiter.result.append('\x00') def make_float_packer(size): - return lambda fmtiter: ieee.pack_float(fmtiter.result, - fmtiter.accept_float_arg(), - size, - fmtiter.bigendian) + def packer(fmtiter): + fl = fmtiter.accept_float_arg() + try: + return ieee.pack_float(fmtiter.result, fl, size, fmtiter.bigendian) + except ValueError: + assert size == 4 + raise StructError("float too large to pack with format f") + return packer # ____________________________________________________________ @@ -155,10 +159,11 @@ fmtiter.appendobj(data[1:end]) def make_float_unpacker(size): - return specialize.argtype(0)( - lambda fmtiter: fmtiter.appendobj(ieee.unpack_float( - fmtiter.read(size), - fmtiter.bigendian))) + @specialize.argtype(0) + def unpacker(fmtiter): + data = fmtiter.read(size) + fmtiter.appendobj(ieee.unpack_float(data, fmtiter.bigendian)) + return unpacker # ____________________________________________________________ From benjamin at codespeak.net Wed Jul 7 21:13:58 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 21:13:58 +0200 (CEST) Subject: [pypy-svn] r76000 - in pypy/branch/fast-forward/pypy: module/struct module/struct/test rlib/rstruct Message-ID: <20100707191358.B7FBB282BF4@codespeak.net> Author: benjamin Date: Wed Jul 7 21:13:33 2010 New Revision: 76000 Modified: pypy/branch/fast-forward/pypy/module/struct/formatiterator.py pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Log: support the bool format Modified: pypy/branch/fast-forward/pypy/module/struct/formatiterator.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/struct/formatiterator.py (original) +++ pypy/branch/fast-forward/pypy/module/struct/formatiterator.py Wed Jul 7 21:13:33 2010 @@ -98,6 +98,10 @@ w_obj = self.accept_obj_arg() return self.space.r_ulonglong_w(w_obj) + def accept_bool_arg(self): + w_obj = self.accept_obj_arg() + return self.space.is_true(w_obj) + def accept_str_arg(self): w_obj = self.accept_obj_arg() return self.space.str_w(w_obj) Modified: pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py (original) +++ pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py Wed Jul 7 21:13:33 2010 @@ -262,6 +262,14 @@ assert unpack("?", True) == '\x01' + assert pack("!?", False) == '\x00' + assert pack(">?", False) == '\x00' + def test_struct_error(self): """ Check the various ways to get a struct.error. Note that CPython Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Wed Jul 7 21:13:33 2010 @@ -31,6 +31,10 @@ c = string[0] # string->char conversion for the annotator fmtiter.result.append(c) +def pack_bool(fmtiter): + c = '\x01' if fmtiter.accept_bool_arg() else '\x00' + fmtiter.result.append(c) + def pack_string(fmtiter, count): string = fmtiter.accept_str_arg() if len(string) < count: @@ -145,6 +149,10 @@ fmtiter.appendobj(fmtiter.read(1)) @specialize.argtype(0) +def unpack_bool(fmtiter): + fmtiter.appendobj(bool(int(fmtiter.read(1)))) + + at specialize.argtype(0) def unpack_string(fmtiter, count): fmtiter.appendobj(fmtiter.read(count)) @@ -225,7 +233,8 @@ 'unpack' : make_float_unpacker(4)}, 'd':{ 'size' : 8, 'pack' : make_float_packer(8), 'unpack' : make_float_unpacker(8)}, - } + '?':{ 'size' : 1, 'pack' : pack_bool, 'unpack' : unpack_bool}, + } for c, size in [('b', 1), ('h', 2), ('i', 4), ('l', 4), ('q', 8)]: standard_fmttable[c] = {'size': size, From benjamin at codespeak.net Wed Jul 7 21:42:23 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 21:42:23 +0200 (CEST) Subject: [pypy-svn] r76001 - pypy/branch/fast-forward/pypy/rlib/rstruct Message-ID: <20100707194223.95EC1282BF2@codespeak.net> Author: benjamin Date: Wed Jul 7 21:42:20 2010 New Revision: 76001 Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py Log: find bool alignment Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py Wed Jul 7 21:42:20 2010 @@ -77,6 +77,7 @@ 'P': 'char *', 'f': 'float', 'd': 'double', + '?': '_Bool', } pre_include_bits = [] From benjamin at codespeak.net Wed Jul 7 22:03:17 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Jul 2010 22:03:17 +0200 (CEST) Subject: [pypy-svn] r76002 - in pypy/branch/fast-forward/pypy: module/struct/test rlib/rstruct Message-ID: <20100707200317.EDC19282BF2@codespeak.net> Author: benjamin Date: Wed Jul 7 22:03:16 2010 New Revision: 76002 Modified: pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Log: support native bool Modified: pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py (original) +++ pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py Wed Jul 7 22:03:16 2010 @@ -269,6 +269,8 @@ assert pack(">?", True) == '\x01' assert pack("!?", False) == '\x00' assert pack(">?", False) == '\x00' + assert pack("@?", True) == '\x01' + assert pack("@?", False) == '\x00' def test_struct_error(self): """ Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py Wed Jul 7 22:03:16 2010 @@ -59,6 +59,13 @@ doubleval = float(floatval) fmtiter.appendobj(doubleval) +def pack_bool(fmtiter): + truth = fmtiter.accept_bool_arg() + fmtiter.result.append('\x01' if truth else '\x00') + +def unpack_bool(fmtiter): + fmtiter.appendobj(bool(ord(fmtiter.read(1)))) + # ____________________________________________________________ # # Use rffi_platform to get the native sizes and alignments from the C compiler @@ -81,13 +88,16 @@ } pre_include_bits = [] - for fmtchar, ctype in INSPECT.items(): + field_names = dict.fromkeys(INSPECT) + for fmtchar, ctype in INSPECT.iteritems(): + field_name = ctype.replace(" ", "_").replace("*", "star") + field_names[fmtchar] = field_name pre_include_bits.append(""" struct about_%s { char pad; %s field; }; - """ % (fmtchar, ctype)) + """ % (field_name, ctype)) class CConfig: _compilation_info_ = ExternalCompilationInfo( @@ -95,14 +105,14 @@ ) for fmtchar, ctype in INSPECT.items(): - setattr(CConfig, fmtchar, rffi_platform.Struct( - "struct about_%s" % (fmtchar,), + setattr(CConfig, field_names[fmtchar], rffi_platform.Struct( + "struct about_%s" % (field_names[fmtchar],), [('field', lltype.FixedSizeArray(rffi.CHAR, 1))])) cConfig = rffi_platform.configure(CConfig) for fmtchar, ctype in INSPECT.items(): - S = cConfig[fmtchar] + S = cConfig[field_names[fmtchar]] alignment = rffi.offsetof(S, 'c_field') size = rffi.sizeof(S.c_field) signed = 'a' <= fmtchar <= 'z' @@ -113,6 +123,9 @@ elif fmtchar == 'd': pack = pack_double unpack = unpack_double + elif fmtchar == '?': + pack = pack_bool + unpack = unpack_bool else: cpython_checks_range = fmtchar in 'bBhH' pack = std.make_int_packer(size, signed, cpython_checks_range) Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Wed Jul 7 22:03:16 2010 @@ -150,7 +150,7 @@ @specialize.argtype(0) def unpack_bool(fmtiter): - fmtiter.appendobj(bool(int(fmtiter.read(1)))) + fmtiter.appendobj(bool(ord(fmtiter.read(1)))) @specialize.argtype(0) def unpack_string(fmtiter, count): From benjamin at codespeak.net Thu Jul 8 00:26:25 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 8 Jul 2010 00:26:25 +0200 (CEST) Subject: [pypy-svn] r76003 - in pypy/branch/fast-forward/pypy: module/struct/test rlib/rstruct Message-ID: <20100707222625.9EBA1282BF2@codespeak.net> Author: benjamin Date: Thu Jul 8 00:26:23 2010 New Revision: 76003 Modified: pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Log: fix bigendian floats Modified: pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py (original) +++ pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py Thu Jul 8 00:26:23 2010 @@ -272,6 +272,30 @@ assert pack("@?", True) == '\x01' assert pack("@?", False) == '\x00' + def test_transitiveness(self): + c = 'a' + b = 1 + h = 255 + i = 65535 + l = 65536 + f = 3.1415 + d = 3.1415 + t = True + + for prefix in ('', '@', '<', '>', '=', '!'): + for format in ('xcbhilfd?', 'xcBHILfd?'): + format = prefix + format + s = self.struct.pack(format, c, b, h, i, l, f, d, t) + cp, bp, hp, ip, lp, fp, dp, tp = self.struct.unpack(format, s) + assert cp == c + assert bp == b + assert hp == h + assert ip == i + assert lp == l + assert int(100 * fp) == int(100 * f) + assert int(100 * dp) == int(100 * d) + assert tp == t + def test_struct_error(self): """ Check the various ways to get a struct.error. Note that CPython Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Thu Jul 8 00:26:23 2010 @@ -124,11 +124,13 @@ def pack_float(result, x, size, be): + l = [] if be else result unsigned = float_pack(x, size) for i in range(size): - result.append(chr((unsigned >> (i * 8)) & 0xFF)) + l.append(chr((unsigned >> (i * 8)) & 0xFF)) if be: - result.reverse() + l.reverse() + result.extend(l) def unpack_float(s, be): From benjamin at codespeak.net Thu Jul 8 00:52:34 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 8 Jul 2010 00:52:34 +0200 (CEST) Subject: [pypy-svn] r76004 - in pypy/branch/fast-forward/pypy: module/struct/test rlib/rstruct Message-ID: <20100707225234.92195282BF2@codespeak.net> Author: benjamin Date: Thu Jul 8 00:52:32 2010 New Revision: 76004 Modified: pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py pypy/branch/fast-forward/pypy/rlib/rstruct/error.py pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Log: floats which are too large should actually be OverflowErrors Modified: pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py (original) +++ pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py Thu Jul 8 00:52:32 2010 @@ -260,7 +260,7 @@ assert pack("= MAX_EXP - MIN_EXP + 2: - raise ValueError("float too large to pack in this format") + raise OverflowError("float too large to pack in this format") # check constraints assert 0 <= mant < 1 << MANT_DIG - 1 Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Thu Jul 8 00:52:32 2010 @@ -6,7 +6,7 @@ # values when packing. import struct -from pypy.rlib.rstruct.error import StructError +from pypy.rlib.rstruct.error import StructError, StructOverflowError from pypy.rlib.rstruct import ieee from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong @@ -65,9 +65,9 @@ fl = fmtiter.accept_float_arg() try: return ieee.pack_float(fmtiter.result, fl, size, fmtiter.bigendian) - except ValueError: + except OverflowError: assert size == 4 - raise StructError("float too large to pack with format f") + raise StructOverflowError("float too large for format 'f'") return packer # ____________________________________________________________ From hakanardo at codespeak.net Thu Jul 8 08:17:14 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 8 Jul 2010 08:17:14 +0200 (CEST) Subject: [pypy-svn] r76005 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100708061714.A48A9282BF2@codespeak.net> Author: hakanardo Date: Thu Jul 8 08:17:11 2010 New Revision: 76005 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: somewhat rpythonified Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Thu Jul 8 08:17:11 2010 @@ -8,19 +8,26 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rstruct.runpack import runpack -import os, types +import os, types, re path, _ = os.path.split(__file__) app_array = os.path.join(path, 'app_array.py') app = ApplevelClass(file(app_array).read()) -def appmethod(n): - import app_array - f=getattr(app_array,n) - args = f.func_code.co_varnames[0:f.func_code.co_argcount] - args = ', '.join(['space'] + ['w_'+s for s in args]) - exec """def descr(%s): - return app.interphook('%s')(%s)"""%(args, n, args) - return interp2app(descr) +def appmethod(n,allappfn={}): + if not allappfn.has_key(n): + #exec 'import %s as mod'%f.__module__ + #src=file(re.sub('.pyc$', '.py', mod.__file__)).read() + #app = ApplevelClass(src) + import app_array + f=getattr(app_array,n) + args = f.func_code.co_varnames[0:f.func_code.co_argcount] + args = ', '.join(['space'] + ['w_'+s for s in args]) + appfn = app.interphook(n) + exec """def descr(%s): + return appfn(%s)"""%(args, args) in locals() + descr.__name__='descr_appmethod_%s'%n + allappfn[n]=interp2app(descr) + return allappfn[n] class W_ArrayBase(Wrappable): pass @@ -46,17 +53,17 @@ types = { 'c': TypeCode(lltype.Char, 'str_w'), - 'u': TypeCode(lltype.UniChar, 'unicode_w'), - 'b': TypeCode(rffi.SIGNEDCHAR, 'int_w', True, True), - 'B': TypeCode(rffi.UCHAR, 'int_w', True), - 'h': TypeCode(rffi.SHORT, 'int_w', True, True), - 'H': TypeCode(rffi.USHORT, 'int_w', True), - 'i': TypeCode(rffi.INT, 'int_w', True, True), - 'I': TypeCode(rffi.UINT, 'int_w', True), - 'l': TypeCode(rffi.LONG, 'int_w', True, True), - 'L': TypeCode(rffi.ULONG, 'bigint_w', True), # FIXME: Won't compile - 'f': TypeCode(lltype.SingleFloat, 'float_w'), - 'd': TypeCode(lltype.Float, 'float_w'), +# 'u': TypeCode(lltype.UniChar, 'unicode_w'), +# 'b': TypeCode(rffi.SIGNEDCHAR, 'int_w', True, True), +# 'B': TypeCode(rffi.UCHAR, 'int_w', True), +# 'h': TypeCode(rffi.SHORT, 'int_w', True, True), +# 'H': TypeCode(rffi.USHORT, 'int_w', True), +# 'i': TypeCode(rffi.INT, 'int_w', True, True), +# 'I': TypeCode(rffi.UINT, 'int_w', True), +# 'l': TypeCode(rffi.LONG, 'int_w', True, True), +# 'L': TypeCode(rffi.ULONG, 'bigint_w', True), # FIXME: Won't compile +# 'f': TypeCode(lltype.SingleFloat, 'float_w'), +# 'd': TypeCode(lltype.Float, 'float_w'), } for k, v in types.items(): v.typecode=k unroll_typecodes = unrolling_iterable(types.keys()) @@ -225,7 +232,11 @@ descr_fromstring.unwrap_spec = ['self', str] def descr_tolist(self): - return self.space.newlist([self.space.wrap(i) for i in self.buffer]) + w_l=self.space.newlist([]) + for i in range(self.len): + w_l.append(self.descr_getitem(self.space.wrap(i))) + return w_l + #return self.space.newlist([self.space.wrap(i) for i in self.buffer]) descr_tolist.unwrap_spec = ['self'] @@ -265,6 +276,7 @@ thetype.w_class = W_Array +initiate=app.interphook('initiate') def array(space, typecode, w_initializer=None): if len(typecode) != 1: msg = 'array() argument 1 must be char, not str' @@ -274,7 +286,7 @@ for tc in unroll_typecodes: if typecode == tc: a = types[tc].w_class(space) - app.interphook('initiate')(space, a, w_initializer) + initiate(space, a, w_initializer) ## if w_initializer is not None: ## if not space.is_w(w_initializer, space.w_None): ## a.descr_fromsequence(w_initializer) From fijall at gmail.com Thu Jul 8 08:45:57 2010 From: fijall at gmail.com (Maciej Fijalkowski) Date: Thu, 8 Jul 2010 08:45:57 +0200 Subject: [pypy-svn] r75972 - in pypy/branch/fast-ctypes/pypy: module/jitffi rlib rlib/test In-Reply-To: <20100707134732.42772282B9E@codespeak.net> References: <20100707134732.42772282B9E@codespeak.net> Message-ID: That's wrong. It's useless boxing. Instead you should have a different way of passing (like push_result_int) on a caller that would wrap it. On Wed, Jul 7, 2010 at 3:47 PM, wrote: > Author: getxsick > Date: Wed Jul ?7 15:47:30 2010 > New Revision: 75972 > > Modified: > ? pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py > ? pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py > ? pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py > Log: > return results as objects of Return subclasses to be more RPython > > > Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py > ============================================================================== > --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) > +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Wed Jul ?7 15:47:30 2010 > @@ -68,9 +68,7 @@ > ? ? ? ? self.rget = rjitffi._Get(cpu, lib, func, args_type, res_type) > > ? ? def call_w(self, space, w_args=None): > - ? ? ? ?if space.is_w(w_args, space.w_None): > - ? ? ? ? ? ?return space.wrap(self.rget.call()) > - ? ? ? ?else: > + ? ? ? ?if not space.is_w(w_args, space.w_None): > ? ? ? ? ? ? i = 0 > ? ? ? ? ? ? w_iterator = space.iter(w_args) > ? ? ? ? ? ? while True: > @@ -93,7 +91,8 @@ > ? ? ? ? ? ? ? ? ? ? ? ? ? ? space.wrap('Unsupported type of argument: %s' > ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? % self.args_type[0])) > ? ? ? ? ? ? ? ? i += 1 > - ? ? ? ?return space.wrap(self.rget.call()) > + ? ? ? ?res = self.rget.call() > + ? ? ? ?return space.wrap(res.value) > > ?def W_Get___new__(space, w_type, cpu, lib, func, args_type, res_type): > ? ? try: > > Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py > ============================================================================== > --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ? ? ? ?(original) > +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ? ? ? ?Wed Jul ?7 15:47:30 2010 > @@ -102,18 +102,18 @@ > ? ? ? ? res = self.cpu.execute_token(self.looptoken) > > ? ? ? ? if self.res_type == 'i': > - ? ? ? ? ? ?r = self.cpu.get_latest_value_int(0) > + ? ? ? ? ? ?r = ReturnInt(self.cpu.get_latest_value_int(0)) > ? ? ? ? elif self.res_type == 'f': > - ? ? ? ? ? ?r = self.cpu.get_latest_value_float(0) > + ? ? ? ? ? ?r = ReturnFloat(self.cpu.get_latest_value_float(0)) > ? ? ? ? elif self.res_type == 'p': > - ? ? ? ? ? ?r = self.cpu.get_latest_value_ref(0) > + ? ? ? ? ? ?r = ReturnPtr(self.cpu.get_latest_value_ref(0)) > ? ? ? ? elif self.res_type == 'v': > - ? ? ? ? ? ?r = None > + ? ? ? ? ? ?r = ReturnNone(None) > ? ? ? ? else: > ? ? ? ? ? ? raise ValueError(self.res_type) > > ? ? ? ? self.setup_stack() # clean up the stack > - ? ? ? ?return r # XXX can't return various types > + ? ? ? ?return r > > ? ? def setup_stack(self): > ? ? ? ? self.esp = 0 > @@ -140,3 +140,19 @@ > ? ? ? ? self.args_type = args_type > ? ? ? ? self.res_type = res_type > ? ? ? ? self.looptoken = looptoken > + > +class Return(object): > + ? ?def __init__(self, value): > + ? ? ? ?self.value = value > + > +class ReturnInt(Return): > + ? ?pass > + > +class ReturnFloat(Return): > + ? ?pass > + > +class ReturnPtr(Return): > + ? ?pass > + > +class ReturnNone(Return): > + ? ?pass > > Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py > ============================================================================== > --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ? ? ?(original) > +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ? ? ?Wed Jul ?7 15:47:30 2010 > @@ -71,44 +71,44 @@ > ? ? ? ? func = lib.get('add_integers', ['i', 'i'], 'i') > ? ? ? ? func.push_int(1) > ? ? ? ? func.push_int(2) > - ? ? ? ?assert func.call() == 3 > + ? ? ? ?assert func.call().value == 3 > > ? ? ? ? func = lib.get('add_integers', ['i', 'i'], 'i') > ? ? ? ? func.push_int(-1) > ? ? ? ? func.push_int(2) > - ? ? ? ?assert func.call() == 1 > + ? ? ? ?assert func.call().value == 1 > > ? ? ? ? func = lib.get('add_integers', ['i', 'i'], 'i') > ? ? ? ? func.push_int(0) > ? ? ? ? func.push_int(0) > - ? ? ? ?assert func.call() == 0 > + ? ? ? ?assert func.call().value == 0 > > ? ? ? ? func = lib.get('max3', ['i', 'i', 'i'], 'i') > ? ? ? ? func.push_int(2) > ? ? ? ? func.push_int(8) > ? ? ? ? func.push_int(3) > - ? ? ? ?assert func.call() == 8 > + ? ? ? ?assert func.call().value == 8 > > ? ? ? ? func = lib.get('add_floats', ['f', 'f'], 'f') > ? ? ? ? func.push_float(1.2) > ? ? ? ? func.push_float(1.5) > - ? ? ? ?assert func.call() == 2.7 > + ? ? ? ?assert func.call().value == 2.7 > > ? ? def test_get_void(self): > ? ? ? ? lib = rjitffi.CDLL(self.lib_name) > > ? ? ? ? func = lib.get('fvoid', [], 'i') > - ? ? ? ?assert func.call() == 1 > + ? ? ? ?assert func.call().value == 1 > > ? ? ? ? func = lib.get('return_void', ['i', 'i'], 'v') > ? ? ? ? func.push_int(1) > ? ? ? ? func.push_int(2) > - ? ? ? ?assert func.call() is None > + ? ? ? ?assert func.call().value is None > > ? ? ? ? func = lib.get('return_void', ['i', 'i']) > ? ? ? ? func.push_int(1) > ? ? ? ? func.push_int(2) > - ? ? ? ?assert func.call() is None > + ? ? ? ?assert func.call().value is None > > ? ? def test_various_type_args(self): > ? ? ? ? lib = rjitffi.CDLL(self.lib_name) > @@ -116,12 +116,12 @@ > ? ? ? ? func = lib.get('add_intfloat', ['i', 'f'], 'i') > ? ? ? ? func.push_int(1) > ? ? ? ? func.push_float(2.9) > - ? ? ? ?assert func.call() == 3 > + ? ? ? ?assert func.call().value == 3 > > ? ? ? ? # stack is cleaned up after calling > ? ? ? ? func.push_int(0) > ? ? ? ? func.push_float(1.3) > - ? ? ? ?assert func.call() == 1 > + ? ? ? ?assert func.call().value == 1 > > ? ? def test_undefined_func(self): > ? ? ? ? lib = rjitffi.CDLL(self.lib_name) > _______________________________________________ > pypy-svn mailing list > pypy-svn at codespeak.net > http://codespeak.net/mailman/listinfo/pypy-svn > From fijal at codespeak.net Thu Jul 8 09:37:57 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Jul 2010 09:37:57 +0200 (CEST) Subject: [pypy-svn] r76006 - pypy/trunk/pypy/jit/codewriter Message-ID: <20100708073757.80B2B282BE0@codespeak.net> Author: fijal Date: Thu Jul 8 09:37:55 2010 New Revision: 76006 Modified: pypy/trunk/pypy/jit/codewriter/codewriter.py Log: Only print code if debug is on Modified: pypy/trunk/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/codewriter.py (original) +++ pypy/trunk/pypy/jit/codewriter/codewriter.py Thu Jul 8 09:37:55 2010 @@ -14,11 +14,12 @@ class CodeWriter(object): callcontrol = None # for tests - def __init__(self, cpu=None, jitdrivers_sd=[]): + def __init__(self, cpu=None, jitdrivers_sd=[], debug=False): self.cpu = cpu self.assembler = Assembler() self.callcontrol = CallControl(cpu, jitdrivers_sd) self._seen_files = set() + self.debug = debug def transform_func_to_jitcode(self, func, values, type_system='lltype'): """For testing.""" @@ -60,7 +61,8 @@ self.assembler.assemble(ssarepr, jitcode) # # print the resulting assembler - self.print_ssa_repr(ssarepr, portal_jd, verbose) + if self.debug: + self.print_ssa_repr(ssarepr, portal_jd, verbose) def make_jitcodes(self, verbose=False): log.info("making JitCodes...") From cfbolz at codespeak.net Thu Jul 8 10:21:50 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 8 Jul 2010 10:21:50 +0200 (CEST) Subject: [pypy-svn] r76007 - pypy/extradoc/sprintinfo/cern2010 Message-ID: <20100708082150.0F27F282BE0@codespeak.net> Author: cfbolz Date: Thu Jul 8 10:21:46 2010 New Revision: 76007 Added: pypy/extradoc/sprintinfo/cern2010/ pypy/extradoc/sprintinfo/cern2010/planning.txt (contents, props changed) Log: (all): the first thing that resembles a planning session! How very disciplined of us. Added: pypy/extradoc/sprintinfo/cern2010/planning.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/cern2010/planning.txt Thu Jul 8 10:21:46 2010 @@ -0,0 +1,9 @@ + + - nicer python interface (Carl Friedrich, Wim) + - fix Makefile generation (Armin, Anto) + - set up tests correctly, use ROOTSYS if it is there (Anto, Armin) + - genreflex whacking (Anto, Armin) + - data members + - passing and returning objects to C++ (Carl Friedrich, Wim) + - garbage collection? + - tons of details From hakanardo at codespeak.net Thu Jul 8 10:39:21 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 8 Jul 2010 10:39:21 +0200 (CEST) Subject: [pypy-svn] r76008 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100708083921.CF905282BE0@codespeak.net> Author: hakanardo Date: Thu Jul 8 10:39:20 2010 New Revision: 76008 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: Passing rtyper Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Thu Jul 8 10:39:20 2010 @@ -35,10 +35,7 @@ class TypeCode(object): def __init__(self, itemtype, unwrap, canoverflow=False, signed=False): self.itemtype = itemtype - if itemtype is lltype.SingleFloat: # FUIXME - self.bytes = 4 - else: - self.bytes = rffi.sizeof(itemtype) + self.bytes = rffi.sizeof(itemtype) self.arraytype = lltype.GcArray(itemtype) self.unwrap = unwrap self.signed = signed @@ -53,67 +50,66 @@ types = { 'c': TypeCode(lltype.Char, 'str_w'), -# 'u': TypeCode(lltype.UniChar, 'unicode_w'), -# 'b': TypeCode(rffi.SIGNEDCHAR, 'int_w', True, True), -# 'B': TypeCode(rffi.UCHAR, 'int_w', True), -# 'h': TypeCode(rffi.SHORT, 'int_w', True, True), -# 'H': TypeCode(rffi.USHORT, 'int_w', True), -# 'i': TypeCode(rffi.INT, 'int_w', True, True), -# 'I': TypeCode(rffi.UINT, 'int_w', True), -# 'l': TypeCode(rffi.LONG, 'int_w', True, True), -# 'L': TypeCode(rffi.ULONG, 'bigint_w', True), # FIXME: Won't compile -# 'f': TypeCode(lltype.SingleFloat, 'float_w'), -# 'd': TypeCode(lltype.Float, 'float_w'), + 'u': TypeCode(lltype.UniChar, 'unicode_w'), + 'b': TypeCode(rffi.SIGNEDCHAR, 'int_w', True, True), + 'B': TypeCode(rffi.UCHAR, 'int_w', True), + 'h': TypeCode(rffi.SHORT, 'int_w', True, True), + 'H': TypeCode(rffi.USHORT, 'int_w', True), + 'i': TypeCode(rffi.INT, 'int_w', True, True), + 'I': TypeCode(rffi.UINT, 'int_w', True), + 'l': TypeCode(rffi.LONG, 'int_w', True, True), + 'L': TypeCode(rffi.ULONG, 'bigint_w'), # Overflow handled by rbigint.touint() which + # corresponds to the C-type unsigned long + 'f': TypeCode(lltype.SingleFloat, 'float_w'), + 'd': TypeCode(lltype.Float, 'float_w'), } for k, v in types.items(): v.typecode=k unroll_typecodes = unrolling_iterable(types.keys()) -for thetypecode, thetype in types.items(): +def make_array(mytype): class W_Array(W_ArrayBase): - mytype = thetype + itemsize=mytype.bytes + typecode=mytype.typecode def __init__(self, space): self.space = space self.len = 0 - self.buffer = lltype.nullptr(self.mytype.arraytype) + self.buffer = lltype.nullptr(mytype.arraytype) def item_w(self, w_item): space = self.space - unwrap = getattr(space, self.mytype.unwrap) + unwrap = getattr(space, mytype.unwrap) item = unwrap(w_item) - if self.mytype.unwrap == 'bigint_w': + if mytype.unwrap == 'bigint_w': try: - if self.mytype.signed: - item = item.tolonglong() - else: - item = item.toulonglong() + item = item.touint() except (ValueError, OverflowError): - msg = 'unsigned %d-byte integer out of range' % self.mytype.bytes + msg = 'unsigned %d-byte integer out of range' % mytype.bytes raise OperationError(space.w_OverflowError, space.wrap(msg)) - elif self.mytype.unwrap == 'str_w' or self.mytype.unwrap == 'unicode_w': + elif mytype.unwrap == 'str_w' or mytype.unwrap == 'unicode_w': if len(item) != 1: msg = 'array item must be char' raise OperationError(space.w_TypeError, space.wrap(msg)) item=item[0] - if self.mytype.canoverflow: + if mytype.canoverflow: msg = None - if self.mytype.signed: - if item < -1 << (self.mytype.bytes * 8 - 1): - msg = 'signed %d-byte integer is less than minimum' % self.mytype.bytes - elif item > (1 << (self.mytype.bytes * 8 - 1)) - 1: - msg = 'signed %d-byte integer is greater than maximum' % self.mytype.bytes + if mytype.signed: + if item < -1 << (mytype.bytes * 8 - 1): + msg = 'signed %d-byte integer is less than minimum' % mytype.bytes + elif item > (1 << (mytype.bytes * 8 - 1)) - 1: + msg = 'signed %d-byte integer is greater than maximum' % mytype.bytes else: if item < 0: - msg = 'unsigned %d-byte integer is less than minimum' % self.mytype.bytes - elif item > (1 << (self.mytype.bytes * 8)) - 1: - msg = 'unsigned %d-byte integer is greater than maximum' % self.mytype.bytes + msg = 'unsigned %d-byte integer is less than minimum' % mytype.bytes + elif item > (1 << (mytype.bytes * 8)) - 1: + msg = 'unsigned %d-byte integer is greater than maximum' % mytype.bytes if msg is not None: raise OperationError(space.w_OverflowError, space.wrap(msg)) - return rffi.cast(self.mytype.itemtype, item) + return rffi.cast(mytype.itemtype, item) def setlen(self, size): - new_buffer = lltype.malloc(self.mytype.arraytype, size) + new_buffer = lltype.malloc(mytype.arraytype, size) for i in range(min(size,self.len)): new_buffer[i] = self.buffer[i] self.buffer = new_buffer @@ -130,16 +126,16 @@ start, stop, step = space.decode_index(w_idx, self.len) if step==0: item = self.buffer[start] - tc=self.mytype.typecode + tc=mytype.typecode if tc == 'b' or tc == 'B' or tc == 'h' or tc == 'H' or tc == 'i' or tc == 'l': item = rffi.cast(lltype.Signed, item) - elif self.mytype.typecode == 'f': + elif mytype.typecode == 'f': item = float(item) return self.space.wrap(item) else: size = (stop - start) / step if (stop - start) % step > 0: size += 1 - w_a=self.mytype.w_class(self.space) + w_a=mytype.w_class(self.space) w_a.setlen(size) j=0 for i in range(start, stop, step): @@ -178,7 +174,7 @@ def descr_extend(self, w_iterable): space=self.space if isinstance(w_iterable, W_ArrayBase): - if self.mytype.typecode != w_iterable.mytype.typecode: + if mytype.typecode != w_iterable.typecode: msg = "can only extend with array of same kind" raise OperationError(space.w_TypeError, space.wrap(msg)) w_iterator = space.iter(w_iterable) @@ -199,36 +195,35 @@ item = self.item_w(w_item) self.buffer[start] = item else: - if isinstance(w_item, W_ArrayBase): - if self.mytype.typecode == w_item.mytype.typecode: - size = (stop - start) / step - if (stop - start) % step > 0: size += 1 - if w_item.len != size: # FIXME: Support for step=1 - msg = ('attempt to assign array of size %d to ' + - 'slice of size %d') % (w_item.len, size) - raise OperationError(self.space.w_ValueError, - self.space.wrap(msg)) - j=0 - for i in range(start, stop, step): - self.buffer[i]=w_item.buffer[j] - j+=1 - return + if isinstance(w_item, W_Array): # Implies mytype.typecode == w_item.typecode + size = (stop - start) / step + if (stop - start) % step > 0: size += 1 + if w_item.len != size: # FIXME: Support for step=1 + msg = ('attempt to assign array of size %d to ' + + 'slice of size %d') % (w_item.len, size) + raise OperationError(self.space.w_ValueError, + self.space.wrap(msg)) + j=0 + for i in range(start, stop, step): + self.buffer[i]=w_item.buffer[j] + j+=1 + return msg='can only assign array to array slice' raise OperationError(self.space.w_TypeError, self.space.wrap(msg)) descr_setitem.unwrap_spec = ['self', W_Root, W_Root] def descr_fromstring(self, s): - if len(s)%self.mytype.bytes !=0: + if len(s)%mytype.bytes !=0: msg = 'string length not a multiple of item size' raise OperationError(self.space.w_ValueError, self.space.wrap(msg)) oldlen = self.len - new = len(s) / self.mytype.bytes + new = len(s) / mytype.bytes self.setlen(oldlen + new) for i in range(new): - p = i * self.mytype.bytes - item=runpack(self.mytype.typecode, s[p:p + self.mytype.bytes]) + p = i * mytype.bytes + item=runpack(mytype.typecode, s[p:p + mytype.bytes]) #self.buffer[oldlen + i]=self.item_w(self.space.wrap(item)) - self.buffer[oldlen + i]=rffi.cast(self.mytype.itemtype, item) + self.buffer[oldlen + i]=rffi.cast(mytype.itemtype, item) descr_fromstring.unwrap_spec = ['self', str] def descr_tolist(self): @@ -242,13 +237,13 @@ def descr_itemsize(space, self): - return space.wrap(self.mytype.bytes) + return space.wrap(self.itemsize) def descr_typecode(space, self): - return space.wrap(self.mytype.typecode) + return space.wrap(self.typecode) - W_Array.__name__ = 'W_ArrayType_'+thetypecode + W_Array.__name__ = 'W_ArrayType_'+mytype.typecode W_Array.typedef = TypeDef( - 'ArrayType_'+thetypecode, + 'ArrayType_'+mytype.typecode, append = interp2app(W_Array.descr_append), __len__ = interp2app(W_Array.descr_len), __getitem__ = interp2app(W_Array.descr_getitem), @@ -258,7 +253,7 @@ typecode = GetSetProperty(descr_typecode, cls=W_Array), extend = interp2app(W_Array.descr_extend), - _fromsequence = interp2app(W_Array.descr_fromsequence), + _fromsequence= interp2app(W_Array.descr_fromsequence), fromstring = interp2app(W_Array.descr_fromstring), fromunicode = appmethod('fromunicode'), fromfile = appmethod('fromfile'), @@ -273,8 +268,10 @@ _setlen = interp2app(W_Array.setlen), ) - thetype.w_class = W_Array + mytype.w_class = W_Array +for mytype in types.values(): + make_array(mytype) initiate=app.interphook('initiate') def array(space, typecode, w_initializer=None): From wlav at codespeak.net Thu Jul 8 10:42:50 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Thu, 8 Jul 2010 10:42:50 +0200 (CEST) Subject: [pypy-svn] r76009 - pypy/extradoc/sprintinfo/cern2010 Message-ID: <20100708084250.5E373282BE0@codespeak.net> Author: wlav Date: Thu Jul 8 10:42:48 2010 New Revision: 76009 Modified: pypy/extradoc/sprintinfo/cern2010/planning.txt Log: Added entry to detailed TODO list. Modified: pypy/extradoc/sprintinfo/cern2010/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/cern2010/planning.txt (original) +++ pypy/extradoc/sprintinfo/cern2010/planning.txt Thu Jul 8 10:42:48 2010 @@ -7,3 +7,7 @@ - passing and returning objects to C++ (Carl Friedrich, Wim) - garbage collection? - tons of details + +Detailed TODO list: + + - Deal with opened cpplibs: ownership and possible offloading From arigo at codespeak.net Thu Jul 8 10:51:05 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Jul 2010 10:51:05 +0200 (CEST) Subject: [pypy-svn] r76010 - in pypy/trunk/pypy/rlib: . test Message-ID: <20100708085105.253B1282BE0@codespeak.net> Author: arigo Date: Thu Jul 8 10:51:03 2010 New Revision: 76010 Modified: pypy/trunk/pypy/rlib/rarithmetic.py pypy/trunk/pypy/rlib/test/test_rarithmetic.py Log: Fix the pure Python implementation of ovfcheck_float_to_int. Modified: pypy/trunk/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/trunk/pypy/rlib/rarithmetic.py (original) +++ pypy/trunk/pypy/rlib/rarithmetic.py Thu Jul 8 10:51:03 2010 @@ -113,14 +113,14 @@ "NOT_RPYTHON" return _local_ovfcheck(int(long(a) << b)) -FL_MAXINT = float(LONG_TEST-1) -FL_MININT = float(-LONG_TEST) - def ovfcheck_float_to_int(x): - _, intp = math.modf(x) - if FL_MININT < intp < FL_MAXINT: - return int(intp) - raise OverflowError + try: + result = int(int(x)) # -2147483648.0 => -2147483648L => -2147483648 + except (OverflowError, ValueError): # ValueError for int(nan) on Py>=2.6 + raise OverflowError + if type(result) is not int: + raise OverflowError + return result def compute_restype(self_type, other_type): if self_type is other_type: Modified: pypy/trunk/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/trunk/pypy/rlib/test/test_rarithmetic.py Thu Jul 8 10:51:03 2010 @@ -271,26 +271,29 @@ assert ovfcheck_float_to_int(13.0) == 13 assert ovfcheck_float_to_int(-1.0) == -1 assert ovfcheck_float_to_int(-13.0) == -13 - # strange things happening for float to int on 64 bit - maxint32 = 2 ** 31 - 1 - assert ovfcheck_float_to_int(float(maxint32-1)) == maxint32-1 - #assert ovfcheck_float_to_int(float(maxint32)) == maxint32 - assert ovfcheck_float_to_int(float(-maxint32)) == -maxint32 - #assert ovfcheck_float_to_int(float(-maxint32-1)) == -maxint32-1 - - try: - ovfcheck_float_to_int(float(-sys.maxint-1)-1) - except OverflowError: - pass - else: - assert False - - try: - ovfcheck_float_to_int(float(sys.maxint)+1) - except OverflowError: - pass - else: - assert False + + # strange things happening for float to int on 64 bit: + # int(float(i)) != i because of rounding issues + x = sys.maxint + while int(float(x)) > sys.maxint: + x -= 1 + assert ovfcheck_float_to_int(float(x)) == int(float(x)) + + x = sys.maxint + 1 + while int(float(x)) <= sys.maxint: + x += 1 + py.test.raises(OverflowError, ovfcheck_float_to_int, x) + + x = -sys.maxint-1 + while int(float(x)) < -sys.maxint-1: + x += 1 + assert ovfcheck_float_to_int(float(x)) == int(float(x)) + + x = -sys.maxint-1 + while int(float(x)) >= -sys.maxint-1: + x -= 1 + py.test.raises(OverflowError, ovfcheck_float_to_int, x) + def test_abs(): assert type(abs(r_longlong(1))) is r_longlong From wlav at codespeak.net Thu Jul 8 11:00:44 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Thu, 8 Jul 2010 11:00:44 +0200 (CEST) Subject: [pypy-svn] r76011 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100708090044.3E101282BE0@codespeak.net> Author: wlav Date: Thu Jul 8 11:00:42 2010 New Revision: 76011 Added: pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py (contents, props changed) pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py (contents, props changed) Modified: pypy/branch/reflex-support/pypy/module/cppyy/__init__.py Log: (cfbolz,wlav) New module pythonify for user interface, and its test. Make load_lib cache resulting libraries. Modified: pypy/branch/reflex-support/pypy/module/cppyy/__init__.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/__init__.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/__init__.py Thu Jul 8 11:00:42 2010 @@ -4,8 +4,10 @@ """ """ interpleveldefs = { - 'load_lib': 'interp_cppyy.load_lib', + '_load_lib': 'interp_cppyy.load_lib', } appleveldefs = { + 'gbl' : 'pythonify.gbl', + 'load_lib' : 'pythonify.load_lib', } Added: pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py Thu Jul 8 11:00:42 2010 @@ -0,0 +1,36 @@ +# NOT_RPYTHON +import cppyy + +class _gbl(object): + """Global C++ namespace, i.e. ::.""" + + def __getattr__(self, attr): + raise AttributeError + +class CppyyClass(object): + def __init__(self, cppclass): + # fill dictionary + pass + + +def get_cppclass(name): + # lookup class + + # if failed, create + + # create base classes + pass + + +_loaded_shared_libs = {} +def load_lib(name): + try: + return _loaded_shared_libs[name] + except KeyError: + lib = cppyy._load_lib(name) + _loaded_shared_libs[name] = lib + return lib + + +# user interface objects +gbl = _gbl() Added: pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Thu Jul 8 11:00:42 2010 @@ -0,0 +1,27 @@ +import py, os +from pypy.conftest import gettestobjspace +from pypy.module.cppyy import interp_cppyy, executor + + +currpath = py.path.local(__file__).dirpath() +shared_lib = str(currpath.join("example01Dict.so")) + +space = gettestobjspace(usemodules=['cppyy']) + +def setup_module(mod): + os.system("make") + +class AppTestPYTHONIFY: + def setup_class(cls): + cls.space = space + env = os.environ + cls.w_shared_lib = space.wrap(shared_lib) + cls.w_example01 = cls.space.appexec([], """(): + import cppyy + return cppyy.load_lib(%r)""" % (shared_lib, )) + + def testLoadLibCache(self): + """Test whether loading a library twice results in the same object.""" + import cppyy + lib2 = cppyy.load_lib(self.shared_lib) + assert self.example01 is lib2 From arigo at codespeak.net Thu Jul 8 11:32:49 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Jul 2010 11:32:49 +0200 (CEST) Subject: [pypy-svn] r76012 - in pypy/trunk/pypy/jit/codewriter: . test Message-ID: <20100708093249.4F39B282BE0@codespeak.net> Author: arigo Date: Thu Jul 8 11:32:47 2010 New Revision: 76012 Modified: pypy/trunk/pypy/jit/codewriter/jtransform.py pypy/trunk/pypy/jit/codewriter/test/test_flatten.py Log: Full support for force_cast. Messy test. Modified: pypy/trunk/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/jtransform.py (original) +++ pypy/trunk/pypy/jit/codewriter/jtransform.py Thu Jul 8 11:32:47 2010 @@ -702,15 +702,40 @@ raise NotImplementedError("cast_ptr_to_int") def rewrite_op_force_cast(self, op): - from pypy.rpython.lltypesystem.rffi import size_and_sign + from pypy.rpython.lltypesystem.rffi import size_and_sign, sizeof from pypy.rlib.rarithmetic import intmask assert not self._is_gc(op.args[0]) size1, unsigned1 = size_and_sign(op.args[0].concretetype) size2, unsigned2 = size_and_sign(op.result.concretetype) - if size1 == size2 and unsigned1 == unsigned2: - return - raise NotImplementedError("cast not supported yet: %s" % (op, )) - + if size2 >= sizeof(lltype.Signed): + return # the target type is LONG or ULONG + # + def bounds(size, unsigned): + if unsigned: + return 0, 1<<(8*size) + else: + return -(1<<(8*size-1)), 1<<(8*size-1) + min1, max1 = bounds(size1, unsigned1) + min2, max2 = bounds(size2, unsigned2) + if min2 <= min1 <= max1 <= max2: + return # the target type includes the source range + # + result = [] + v1 = op.args[0] + if min2: + c_min2 = Constant(min2, lltype.Signed) + v2 = Variable(); v2.concretetype = lltype.Signed + result.append(SpaceOperation('int_sub', [v1, c_min2], v2)) + else: + v2 = v1 + c_mask = Constant(int((1<<(8*size2))-1), lltype.Signed) + v3 = Variable(); v3.concretetype = lltype.Signed + result.append(SpaceOperation('int_and', [v2, c_mask], v3)) + if min2: + result.append(SpaceOperation('int_add', [v3, c_min2], op.result)) + else: + result[-1].result = op.result + return result # ---------- # Renames, from the _old opname to the _new one. Modified: pypy/trunk/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/trunk/pypy/jit/codewriter/test/test_flatten.py Thu Jul 8 11:32:47 2010 @@ -731,25 +731,77 @@ """, transform=True) def test_force_cast(self): - py.test.skip("later") from pypy.rpython.lltypesystem import rffi - def f(n): - c = chr(n) - return rffi.cast(rffi.INT, c) - self.encoding_test(f, [42], """ - int_return %i0 - """, transform=True) - def g(n): - return rffi.cast(rffi.UCHAR, n) - self.encoding_test(g, [42], """ - int_and %i0, $255 -> %i1 - int_return %i1 - """, transform=True) - def h(n): - return rffi.cast(rffi.SCHAR, n) - self.encoding_test(h, [42], """ - ... - """, transform=True) + + for FROM, TO, expected in [ + (rffi.SIGNEDCHAR, rffi.SIGNEDCHAR, ""), + (rffi.SIGNEDCHAR, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.SIGNEDCHAR, rffi.SHORT, ""), + (rffi.SIGNEDCHAR, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.SIGNEDCHAR, rffi.LONG, ""), + (rffi.SIGNEDCHAR, rffi.ULONG, ""), + + (rffi.UCHAR, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.UCHAR, rffi.UCHAR, ""), + (rffi.UCHAR, rffi.SHORT, ""), + (rffi.UCHAR, rffi.USHORT, ""), + (rffi.UCHAR, rffi.LONG, ""), + (rffi.UCHAR, rffi.ULONG, ""), + + (rffi.SHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.SHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.SHORT, rffi.SHORT, ""), + (rffi.SHORT, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.SHORT, rffi.LONG, ""), + (rffi.SHORT, rffi.ULONG, ""), + + (rffi.USHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.USHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.USHORT, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.USHORT, rffi.USHORT, ""), + (rffi.USHORT, rffi.LONG, ""), + (rffi.USHORT, rffi.ULONG, ""), + + (rffi.LONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.LONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.LONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.LONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.LONG, rffi.LONG, ""), + (rffi.LONG, rffi.ULONG, ""), + + (rffi.ULONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.ULONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.ULONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.ULONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.ULONG, rffi.LONG, ""), + (rffi.ULONG, rffi.ULONG, ""), + ]: + expected = [s.strip() for s in expected.splitlines()] + check_force_cast(FROM, TO, expected, 42) + check_force_cast(FROM, TO, expected, -42) + expected.append('int_return %i' + str(len(expected))) + expected = '\n'.join(expected) + # + def f(n): + return rffi.cast(TO, n) + self.encoding_test(f, [rffi.cast(FROM, 42)], expected, + transform=True) def test_force_cast_pointer(self): from pypy.rpython.lltypesystem import rffi @@ -758,3 +810,27 @@ self.encoding_test(h, [lltype.nullptr(rffi.CCHARP.TO)], """ int_return %i0 """, transform=True) + + +def check_force_cast(FROM, TO, operations, value): + """Check that the test is correctly written...""" + from pypy.rpython.lltypesystem import rffi + import re + r = re.compile('(\w+) \%i\d, \$(-?\d+)') + # + value = rffi.cast(FROM, value) + value = rffi.cast(lltype.Signed, value) + # + expected_value = rffi.cast(TO, value) + expected_value = rffi.cast(lltype.Signed, expected_value) + # + for op in operations: + match = r.match(op) + assert match, "line %r does not match regexp" % (op,) + opname = match.group(1) + if opname == 'int_add': value += int(match.group(2)) + elif opname == 'int_sub': value -= int(match.group(2)) + elif opname == 'int_and': value &= int(match.group(2)) + else: assert 0, opname + # + assert rffi.cast(lltype.Signed, value) == expected_value From hakanardo at codespeak.net Thu Jul 8 11:40:26 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 8 Jul 2010 11:40:26 +0200 (CEST) Subject: [pypy-svn] r76013 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100708094026.3B115282BE0@codespeak.net> Author: hakanardo Date: Thu Jul 8 11:40:24 2010 New Revision: 76013 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: moved to raw mallocs Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Thu Jul 8 11:40:24 2010 @@ -36,7 +36,8 @@ def __init__(self, itemtype, unwrap, canoverflow=False, signed=False): self.itemtype = itemtype self.bytes = rffi.sizeof(itemtype) - self.arraytype = lltype.GcArray(itemtype) + #self.arraytype = lltype.GcArray(itemtype) + self.arraytype = lltype.Array(itemtype, hints={'nolength': True}) self.unwrap = unwrap self.signed = signed self.canoverflow = canoverflow @@ -108,10 +109,20 @@ return rffi.cast(mytype.itemtype, item) + def __del__(self): + self.setlen(0) + + def setlen(self, size): - new_buffer = lltype.malloc(mytype.arraytype, size) - for i in range(min(size,self.len)): - new_buffer[i] = self.buffer[i] + if size > 0: + #new_buffer = lltype.malloc(mytype.arraytype, size) + new_buffer = lltype.malloc(mytype.arraytype, size, flavor='raw') + for i in range(min(size,self.len)): + new_buffer[i] = self.buffer[i] + else: + new_buffer = lltype.nullptr(mytype.arraytype) + if self.buffer != lltype.nullptr(mytype.arraytype): + lltype.free(self.buffer, flavor='raw') self.buffer = new_buffer self.len = size setlen.unwrap_spec = ['self', int] @@ -219,11 +230,17 @@ oldlen = self.len new = len(s) / mytype.bytes self.setlen(oldlen + new) - for i in range(new): - p = i * mytype.bytes - item=runpack(mytype.typecode, s[p:p + mytype.bytes]) - #self.buffer[oldlen + i]=self.item_w(self.space.wrap(item)) - self.buffer[oldlen + i]=rffi.cast(mytype.itemtype, item) + if False: + for i in range(new): + p = i * mytype.bytes + item=runpack(mytype.typecode, s[p:p + mytype.bytes]) + #self.buffer[oldlen + i]=self.item_w(self.space.wrap(item)) + self.buffer[oldlen + i]=rffi.cast(mytype.itemtype, item) + else: + pbuf = rffi.cast(rffi.CCHARP, self.buffer) + for i in range(len(s)): + pbuf[oldlen * mytype.bytes + i] = s[i] + descr_fromstring.unwrap_spec = ['self', str] def descr_tolist(self): @@ -234,6 +251,12 @@ #return self.space.newlist([self.space.wrap(i) for i in self.buffer]) descr_tolist.unwrap_spec = ['self'] + def descr_tostring(self): + pbuf = rffi.cast(rffi.CCHARP, self.buffer) + s = '' + for i in range(self.len * self.itemsize): + s += pbuf[i] + return self.space.wrap(s) def descr_itemsize(space, self): @@ -263,7 +286,8 @@ tolist = interp2app(W_Array.descr_tolist), tounicode = appmethod('tounicode'), tofile = appmethod('tofile'), - tostring = appmethod('tostring'), + #tostring = appmethod('tostring'), + tostring = interp2app(W_Array.descr_tostring), _setlen = interp2app(W_Array.setlen), ) From antocuni at codespeak.net Thu Jul 8 12:20:46 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 8 Jul 2010 12:20:46 +0200 (CEST) Subject: [pypy-svn] r76014 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100708102046.57CB5282BFA@codespeak.net> Author: antocuni Date: Thu Jul 8 12:20:39 2010 New Revision: 76014 Added: pypy/branch/reflex-support/pypy/module/cppyy/genreflex-methptrgetter.patch Log: (arigo, antocuni) a patch for genreflex.py that adds the MethPtrGetter property to the FunctionMembers of a class Added: pypy/branch/reflex-support/pypy/module/cppyy/genreflex-methptrgetter.patch ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/genreflex-methptrgetter.patch Thu Jul 8 12:20:39 2010 @@ -0,0 +1,100 @@ +Index: cint/reflex/python/genreflex/gendict.py +=================================================================== +--- cint/reflex/python/genreflex/gendict.py (revision 34339) ++++ cint/reflex/python/genreflex/gendict.py (working copy) +@@ -52,6 +52,7 @@ + self.typedefs_for_usr = [] + self.gccxmlvers = gccxmlvers + self.split = opts.get('split', '') ++ self.with_methptrgetter = opts.get('with_methptrgetter', False) + # The next is to avoid a known problem with gccxml that it generates a + # references to id equal '_0' which is not defined anywhere + self.xref['_0'] = {'elem':'Unknown', 'attrs':{'id':'_0','name':''}, 'subelems':[]} +@@ -1939,8 +1940,15 @@ + else : params = '0' + s = ' .AddFunctionMember(%s, Reflex::Literal("%s"), %s%s, 0, %s, %s)' % (self.genTypeID(id), name, type, id, params, mod) + s += self.genCommentProperty(attrs) ++ s += self.genMethPtrGetterProperty(type, attrs) + return s + #---------------------------------------------------------------------------------- ++ def genMethPtrGetterProperty(self, type, attrs): ++ funcname = self.nameOfMethPtrGetter(type, attrs) ++ if funcname is None: ++ return '' ++ return '\n .AddProperty("MethPtrGetter", (void*)%s)' % funcname ++#---------------------------------------------------------------------------------- + def genMCODef(self, type, name, attrs, args): + id = attrs['id'] + cl = self.genTypeName(attrs['context'],colon=True) +@@ -2007,8 +2015,36 @@ + if returns == 'void' : body += ' }\n' + else : body += ' }\n' + body += '}\n' +- return head + body; ++ methptrgetter = self.genMethPtrGetter(type, name, attrs, args) ++ return head + body + methptrgetter + #---------------------------------------------------------------------------------- ++ def nameOfMethPtrGetter(self, type, attrs): ++ id = attrs['id'] ++ if self.with_methptrgetter and 'static' not in attrs and type in ('operator', 'method'): ++ return '%s%s_methptrgetter' % (type, id) ++ return None ++#---------------------------------------------------------------------------------- ++ def genMethPtrGetter(self, type, name, attrs, args): ++ funcname = self.nameOfMethPtrGetter(type, attrs) ++ if funcname is None: ++ return '' ++ id = attrs['id'] ++ cl = self.genTypeName(attrs['context'],colon=True) ++ rettype = self.genTypeName(attrs['returns'],enum=True, const=True, colon=True) ++ arg_type_list = [self.genTypeName(arg['type'], colon=True) for arg in args] ++ lines = [] ++ a = lines.append ++ a('static void* %s(void* o)' % (funcname,)) ++ a('{') ++ # declare a variable "meth" which is a member pointer ++ a(' %s (%s::*meth)(%s);' % (rettype, cl, ', '.join(arg_type_list))) ++ a(' meth = &%s::%s;' % (cl, name)) ++ a(' %s* obj = (%s*)o;' % (cl, cl)) ++ a(' return (void*)(obj->*meth);') ++ a('}') ++ return '\n'.join(lines) ++ ++#---------------------------------------------------------------------------------- + def getDefaultArgs(self, args): + n = 0 + for a in args : +Index: cint/reflex/python/genreflex/genreflex.py +=================================================================== +--- cint/reflex/python/genreflex/genreflex.py (revision 34339) ++++ cint/reflex/python/genreflex/genreflex.py (working copy) +@@ -108,6 +108,10 @@ + Print extra debug information while processing. Keep intermediate files\n + --quiet + Do not print informational messages\n ++ --with-methptrgetter ++ Add the property MethPtrGetter to every FunctionMember. It contains a pointer to a ++ function which you can call to get the actual function pointer of the method that it's ++ stored in the vtable. It works only with gcc. + -h, --help + Print this help\n + """ +@@ -127,7 +131,8 @@ + opts, args = getopt.getopt(options, 'ho:s:c:I:U:D:PC', \ + ['help','debug=', 'output=','selection_file=','pool','dataonly','interpreteronly','deep','gccxmlpath=', + 'capabilities=','rootmap=','rootmap-lib=','comments','iocomments','no_membertypedefs', +- 'fail_on_warnings', 'quiet', 'gccxmlopt=', 'reflex', 'split=','no_templatetypedefs','gccxmlpost=']) ++ 'fail_on_warnings', 'quiet', 'gccxmlopt=', 'reflex', 'split=','no_templatetypedefs','gccxmlpost=', ++ 'with-methptrgetter']) + except getopt.GetoptError, e: + print "--->> genreflex: ERROR:",e + self.usage(2) +@@ -186,6 +191,8 @@ + self.rootmap = a + if o in ('--rootmap-lib',): + self.rootmaplib = a ++ if o in ('--with-methptrgetter',): ++ self.opts['with_methptrgetter'] = True + if o in ('-I', '-U', '-D', '-P', '-C') : + # escape quotes; we need to use " because of windows cmd + poseq = a.find('=') From hakanardo at codespeak.net Thu Jul 8 12:35:54 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 8 Jul 2010 12:35:54 +0200 (CEST) Subject: [pypy-svn] r76015 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100708103554.2588E282BE0@codespeak.net> Author: hakanardo Date: Thu Jul 8 12:35:49 2010 New Revision: 76015 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array_try1.py (props changed) pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: additional tostring test Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Thu Jul 8 12:35:49 2010 @@ -287,6 +287,20 @@ a = self.array('i',[0,0,0]) assert a.tostring() == '\x00'*3*a.itemsize + s = self.array('i', [1, 2, 3]).tostring() + assert '\x00' in s + assert '\x01' in s + assert '\x02' in s + assert '\x03' in s + a=self.array('i', s) + assert a[0]==1 and a[1]==2 and a[2]==3 + + from struct import unpack + values = (-129, 128, -128, 127, 0, 255, -1, 256, -32760, 32760) + s = self.array('i', values).tostring() + a=unpack('i'*len(values), s) + assert a==values + #FXIME: How to test? #from cStringIO import StringIO #f=StringIO() From hakanardo at codespeak.net Thu Jul 8 12:37:38 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 8 Jul 2010 12:37:38 +0200 (CEST) Subject: [pypy-svn] r76016 - in pypy/branch/interplevel-array: . lib-python lib_pypy lib_pypy/pypy_test pypy/jit/backend/llgraph pypy/jit/backend/llsupport pypy/jit/codewriter pypy/jit/codewriter/test pypy/module/test_lib_pypy/ctypes_tests pypy/rlib pypy/rlib/test Message-ID: <20100708103738.8ACE8282BE0@codespeak.net> Author: hakanardo Date: Thu Jul 8 12:37:36 2010 New Revision: 76016 Modified: pypy/branch/interplevel-array/ (props changed) pypy/branch/interplevel-array/lib-python/ (props changed) pypy/branch/interplevel-array/lib_pypy/ (props changed) pypy/branch/interplevel-array/lib_pypy/dbm.py (props changed) pypy/branch/interplevel-array/lib_pypy/pypy_test/test_functools.py (props changed) pypy/branch/interplevel-array/pypy/jit/backend/llgraph/llimpl.py pypy/branch/interplevel-array/pypy/jit/backend/llsupport/descr.py pypy/branch/interplevel-array/pypy/jit/backend/llsupport/support.py pypy/branch/interplevel-array/pypy/jit/codewriter/codewriter.py pypy/branch/interplevel-array/pypy/jit/codewriter/jtransform.py pypy/branch/interplevel-array/pypy/jit/codewriter/test/test_flatten.py pypy/branch/interplevel-array/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py pypy/branch/interplevel-array/pypy/rlib/test/test_rarithmetic.py Log: svn merge -r75941:76012 svn+ssh://hakanardo at codespeak.net/svn/pypy/trunk Modified: pypy/branch/interplevel-array/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llgraph/llimpl.py Thu Jul 8 12:37:36 2010 @@ -1549,6 +1549,8 @@ setannotation(do_getarrayitem_gc_int, annmodel.SomeInteger()) setannotation(do_getarrayitem_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) setannotation(do_getarrayitem_gc_float, annmodel.SomeFloat()) +setannotation(do_getarrayitem_raw_int, annmodel.SomeInteger()) +setannotation(do_getarrayitem_raw_float, annmodel.SomeFloat()) setannotation(do_getfield_gc_int, annmodel.SomeInteger()) setannotation(do_getfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) setannotation(do_getfield_gc_float, annmodel.SomeFloat()) @@ -1560,6 +1562,8 @@ setannotation(do_setarrayitem_gc_int, annmodel.s_None) setannotation(do_setarrayitem_gc_ptr, annmodel.s_None) setannotation(do_setarrayitem_gc_float, annmodel.s_None) +setannotation(do_setarrayitem_raw_int, annmodel.s_None) +setannotation(do_setarrayitem_raw_float, annmodel.s_None) setannotation(do_setfield_gc_int, annmodel.s_None) setannotation(do_setfield_gc_ptr, annmodel.s_None) setannotation(do_setfield_gc_float, annmodel.s_None) Modified: pypy/branch/interplevel-array/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llsupport/descr.py Thu Jul 8 12:37:36 2010 @@ -23,10 +23,10 @@ self._cache_call = {} def init_size_descr(self, STRUCT, sizedescr): - pass + assert isinstance(STRUCT, lltype.GcStruct) def init_array_descr(self, ARRAY, arraydescr): - pass + assert isinstance(ARRAY, lltype.GcArray) # ____________________________________________________________ @@ -205,7 +205,8 @@ assert basesize == arraydescr.get_base_size(False) assert itemsize == arraydescr.get_item_size(False) assert ofslength == arraydescr.get_ofs_length(False) - gccache.init_array_descr(ARRAY, arraydescr) + if isinstance(ARRAY, lltype.GcArray): + gccache.init_array_descr(ARRAY, arraydescr) cache[ARRAY] = arraydescr return arraydescr Modified: pypy/branch/interplevel-array/pypy/jit/backend/llsupport/support.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llsupport/support.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llsupport/support.py Thu Jul 8 12:37:36 2010 @@ -30,7 +30,13 @@ funcobj = get_funcobj(fnptr) if hasattr(funcobj, 'graph'): - llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) + # cache the llinterp; otherwise the remember_malloc/remember_free + # done on the LLInterpreter don't match + try: + llinterp = rtyper._on_top_of_llinterp_llinterp + except AttributeError: + llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) + rtyper._on_top_of_llinterp_llinterp = llinterp def on_top_of_llinterp(*args): real_args = process_args(args) return llinterp.eval_graph(funcobj.graph, real_args) Modified: pypy/branch/interplevel-array/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/interplevel-array/pypy/jit/codewriter/codewriter.py Thu Jul 8 12:37:36 2010 @@ -14,11 +14,12 @@ class CodeWriter(object): callcontrol = None # for tests - def __init__(self, cpu=None, jitdrivers_sd=[]): + def __init__(self, cpu=None, jitdrivers_sd=[], debug=False): self.cpu = cpu self.assembler = Assembler() self.callcontrol = CallControl(cpu, jitdrivers_sd) self._seen_files = set() + self.debug = debug def transform_func_to_jitcode(self, func, values, type_system='lltype'): """For testing.""" @@ -60,7 +61,8 @@ self.assembler.assemble(ssarepr, jitcode) # # print the resulting assembler - self.print_ssa_repr(ssarepr, portal_jd, verbose) + if self.debug: + self.print_ssa_repr(ssarepr, portal_jd, verbose) def make_jitcodes(self, verbose=False): log.info("making JitCodes...") Modified: pypy/branch/interplevel-array/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/interplevel-array/pypy/jit/codewriter/jtransform.py Thu Jul 8 12:37:36 2010 @@ -702,16 +702,40 @@ raise NotImplementedError("cast_ptr_to_int") def rewrite_op_force_cast(self, op): - from pypy.rpython.lltypesystem.rffi import size_and_sign + from pypy.rpython.lltypesystem.rffi import size_and_sign, sizeof from pypy.rlib.rarithmetic import intmask assert not self._is_gc(op.args[0]) size1, unsigned1 = size_and_sign(op.args[0].concretetype) size2, unsigned2 = size_and_sign(op.result.concretetype) - if size1 == size2 and unsigned1 == unsigned2: - return - raise NotImplementedError("cast not supported yet: %s (%s->%s)" % - (op, op.args[0].concretetype, - op.result.concretetype)) + if size2 >= sizeof(lltype.Signed): + return # the target type is LONG or ULONG + # + def bounds(size, unsigned): + if unsigned: + return 0, 1<<(8*size) + else: + return -(1<<(8*size-1)), 1<<(8*size-1) + min1, max1 = bounds(size1, unsigned1) + min2, max2 = bounds(size2, unsigned2) + if min2 <= min1 <= max1 <= max2: + return # the target type includes the source range + # + result = [] + v1 = op.args[0] + if min2: + c_min2 = Constant(min2, lltype.Signed) + v2 = Variable(); v2.concretetype = lltype.Signed + result.append(SpaceOperation('int_sub', [v1, c_min2], v2)) + else: + v2 = v1 + c_mask = Constant(int((1<<(8*size2))-1), lltype.Signed) + v3 = Variable(); v3.concretetype = lltype.Signed + result.append(SpaceOperation('int_and', [v2, c_mask], v3)) + if min2: + result.append(SpaceOperation('int_add', [v3, c_min2], op.result)) + else: + result[-1].result = op.result + return result # ---------- # Renames, from the _old opname to the _new one. Modified: pypy/branch/interplevel-array/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/interplevel-array/pypy/jit/codewriter/test/test_flatten.py Thu Jul 8 12:37:36 2010 @@ -731,25 +731,77 @@ """, transform=True) def test_force_cast(self): - py.test.skip("later") from pypy.rpython.lltypesystem import rffi - def f(n): - c = chr(n) - return rffi.cast(rffi.INT, c) - self.encoding_test(f, [42], """ - int_return %i0 - """, transform=True) - def g(n): - return rffi.cast(rffi.UCHAR, n) - self.encoding_test(g, [42], """ - int_and %i0, $255 -> %i1 - int_return %i1 - """, transform=True) - def h(n): - return rffi.cast(rffi.SCHAR, n) - self.encoding_test(h, [42], """ - ... - """, transform=True) + + for FROM, TO, expected in [ + (rffi.SIGNEDCHAR, rffi.SIGNEDCHAR, ""), + (rffi.SIGNEDCHAR, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.SIGNEDCHAR, rffi.SHORT, ""), + (rffi.SIGNEDCHAR, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.SIGNEDCHAR, rffi.LONG, ""), + (rffi.SIGNEDCHAR, rffi.ULONG, ""), + + (rffi.UCHAR, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.UCHAR, rffi.UCHAR, ""), + (rffi.UCHAR, rffi.SHORT, ""), + (rffi.UCHAR, rffi.USHORT, ""), + (rffi.UCHAR, rffi.LONG, ""), + (rffi.UCHAR, rffi.ULONG, ""), + + (rffi.SHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.SHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.SHORT, rffi.SHORT, ""), + (rffi.SHORT, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.SHORT, rffi.LONG, ""), + (rffi.SHORT, rffi.ULONG, ""), + + (rffi.USHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.USHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.USHORT, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.USHORT, rffi.USHORT, ""), + (rffi.USHORT, rffi.LONG, ""), + (rffi.USHORT, rffi.ULONG, ""), + + (rffi.LONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.LONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.LONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.LONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.LONG, rffi.LONG, ""), + (rffi.LONG, rffi.ULONG, ""), + + (rffi.ULONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.ULONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.ULONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.ULONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.ULONG, rffi.LONG, ""), + (rffi.ULONG, rffi.ULONG, ""), + ]: + expected = [s.strip() for s in expected.splitlines()] + check_force_cast(FROM, TO, expected, 42) + check_force_cast(FROM, TO, expected, -42) + expected.append('int_return %i' + str(len(expected))) + expected = '\n'.join(expected) + # + def f(n): + return rffi.cast(TO, n) + self.encoding_test(f, [rffi.cast(FROM, 42)], expected, + transform=True) def test_force_cast_pointer(self): from pypy.rpython.lltypesystem import rffi @@ -758,3 +810,27 @@ self.encoding_test(h, [lltype.nullptr(rffi.CCHARP.TO)], """ int_return %i0 """, transform=True) + + +def check_force_cast(FROM, TO, operations, value): + """Check that the test is correctly written...""" + from pypy.rpython.lltypesystem import rffi + import re + r = re.compile('(\w+) \%i\d, \$(-?\d+)') + # + value = rffi.cast(FROM, value) + value = rffi.cast(lltype.Signed, value) + # + expected_value = rffi.cast(TO, value) + expected_value = rffi.cast(lltype.Signed, expected_value) + # + for op in operations: + match = r.match(op) + assert match, "line %r does not match regexp" % (op,) + opname = match.group(1) + if opname == 'int_add': value += int(match.group(2)) + elif opname == 'int_sub': value -= int(match.group(2)) + elif opname == 'int_and': value &= int(match.group(2)) + else: assert 0, opname + # + assert rffi.cast(lltype.Signed, value) == expected_value Modified: pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py Thu Jul 8 12:37:36 2010 @@ -113,14 +113,14 @@ "NOT_RPYTHON" return _local_ovfcheck(int(long(a) << b)) -FL_MAXINT = float(LONG_TEST-1) -FL_MININT = float(-LONG_TEST) - def ovfcheck_float_to_int(x): - _, intp = math.modf(x) - if FL_MININT < intp < FL_MAXINT: - return int(intp) - raise OverflowError + try: + result = int(int(x)) # -2147483648.0 => -2147483648L => -2147483648 + except (OverflowError, ValueError): # ValueError for int(nan) on Py>=2.6 + raise OverflowError + if type(result) is not int: + raise OverflowError + return result def compute_restype(self_type, other_type): if self_type is other_type: Modified: pypy/branch/interplevel-array/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/test/test_rarithmetic.py Thu Jul 8 12:37:36 2010 @@ -271,26 +271,29 @@ assert ovfcheck_float_to_int(13.0) == 13 assert ovfcheck_float_to_int(-1.0) == -1 assert ovfcheck_float_to_int(-13.0) == -13 - # strange things happening for float to int on 64 bit - maxint32 = 2 ** 31 - 1 - assert ovfcheck_float_to_int(float(maxint32-1)) == maxint32-1 - #assert ovfcheck_float_to_int(float(maxint32)) == maxint32 - assert ovfcheck_float_to_int(float(-maxint32)) == -maxint32 - #assert ovfcheck_float_to_int(float(-maxint32-1)) == -maxint32-1 - - try: - ovfcheck_float_to_int(float(-sys.maxint-1)-1) - except OverflowError: - pass - else: - assert False - - try: - ovfcheck_float_to_int(float(sys.maxint)+1) - except OverflowError: - pass - else: - assert False + + # strange things happening for float to int on 64 bit: + # int(float(i)) != i because of rounding issues + x = sys.maxint + while int(float(x)) > sys.maxint: + x -= 1 + assert ovfcheck_float_to_int(float(x)) == int(float(x)) + + x = sys.maxint + 1 + while int(float(x)) <= sys.maxint: + x += 1 + py.test.raises(OverflowError, ovfcheck_float_to_int, x) + + x = -sys.maxint-1 + while int(float(x)) < -sys.maxint-1: + x += 1 + assert ovfcheck_float_to_int(float(x)) == int(float(x)) + + x = -sys.maxint-1 + while int(float(x)) >= -sys.maxint-1: + x -= 1 + py.test.raises(OverflowError, ovfcheck_float_to_int, x) + def test_abs(): assert type(abs(r_longlong(1))) is r_longlong From wlav at codespeak.net Thu Jul 8 12:38:19 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Thu, 8 Jul 2010 12:38:19 +0200 (CEST) Subject: [pypy-svn] r76017 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100708103819.8DEC3282C0A@codespeak.net> Author: wlav Date: Thu Jul 8 12:38:17 2010 New Revision: 76017 Modified: pypy/branch/reflex-support/pypy/module/cppyy/__init__.py pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Log: (wlav, cfbolz): start exposing c++ classes and support static methods on them. Modified: pypy/branch/reflex-support/pypy/module/cppyy/__init__.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/__init__.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/__init__.py Thu Jul 8 12:38:17 2010 @@ -4,7 +4,8 @@ """ """ interpleveldefs = { - '_load_lib': 'interp_cppyy.load_lib', + '_load_lib' : 'interp_cppyy.load_lib', + '_type_byname' : 'interp_cppyy.type_byname', } appleveldefs = { Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Thu Jul 8 12:38:17 2010 @@ -20,19 +20,22 @@ return W_CPPLibrary(space, cdll) load_lib.unwrap_spec = [ObjSpace, str] +def type_byname(space, name): + handle = capi.c_cppyy_get_typehandle(name) + if handle: + return W_CPPType(space, name, handle) + raise OperationError(space.w_TypeError, space.wrap("no such C++ class %s" % name)) +type_byname.unwrap_spec = [ObjSpace, str] + + class W_CPPLibrary(Wrappable): _immutable_ = True def __init__(self, space, cdll): self.cdll = cdll self.space = space - def type_byname(self, name): - handle = capi.c_cppyy_get_typehandle(name) - return W_CPPType(self, name, handle) - W_CPPLibrary.typedef = TypeDef( 'CPPLibrary', - type_byname = interp2app(W_CPPLibrary.type_byname, unwrap_spec=['self', str]), ) class CPPMethod(object): @@ -118,7 +121,7 @@ return W_CPPObject(self.cpptype, result) -class CPPOverload(object): +class W_CPPOverload(Wrappable): _immutable_ = True _immutable_fields_ = ["functions[*]"] def __init__(self, space, func_name, functions): @@ -126,6 +129,9 @@ self.func_name = func_name self.functions = debug.make_sure_not_resized(functions) + def is_static(self): + return self.space.wrap(isinstance(self.functions[0], CPPFunction)) + @jit.unroll_safe def call(self, cppthis, args_w): space = self.space @@ -142,15 +148,19 @@ raise OperationError(space.w_TypeError, space.wrap("none of the overloads matched")) def __repr__(self): - return "CPPOverload(%s, %s)" % (self.func_name, self.functions) + return "W_CPPOverload(%s, %s)" % (self.func_name, self.functions) + +W_CPPOverload.typedef = TypeDef( + 'CPPOverload', + is_static = interp2app(W_CPPOverload.is_static, unwrap_spec=['self']), +) class W_CPPType(Wrappable): - _immutable_fields_ = ["cpplib", "name","handle"] + _immutable_fields_ = ["name","handle"] - def __init__(self, cpplib, name, handle): - self.space = cpplib.space - self.cpplib = cpplib + def __init__(self, space, name, handle): + self.space = space self.name = name self.handle = handle self.function_members = {} @@ -165,7 +175,7 @@ overload = args_temp.setdefault(func_member_name, []) overload.append(cppfunction) for name, functions in args_temp.iteritems(): - overload = CPPOverload(self.space, name, functions[:]) + overload = W_CPPOverload(self.space, name, functions[:]) self.function_members[name] = overload def _make_cppfunction(self, method_index): @@ -183,6 +193,9 @@ cls = CPPMethod return cls(self, method_index, result_type, argtypes) + def get_function_members(self): + return self.space.newlist([self.space.wrap(name) for name in self.function_members]) + @jit.purefunction def get_overload(self, name): return self.function_members[name] @@ -197,6 +210,8 @@ W_CPPType.typedef = TypeDef( 'CPPType', + get_function_members = interp2app(W_CPPType.get_function_members, unwrap_spec=['self']), + get_overload = interp2app(W_CPPType.get_overload, unwrap_spec=['self', str]), invoke = interp2app(W_CPPType.invoke, unwrap_spec=['self', str, 'args_w']), construct = interp2app(W_CPPType.construct, unwrap_spec=['self', 'args_w']), ) Modified: pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py Thu Jul 8 12:38:17 2010 @@ -1,25 +1,52 @@ # NOT_RPYTHON import cppyy -class _gbl(object): - """Global C++ namespace, i.e. ::.""" - - def __getattr__(self, attr): - raise AttributeError -class CppyyClass(object): - def __init__(self, cppclass): - # fill dictionary - pass +class CppyyClass(type): + pass +def make_static_function(cpptype, name): + def method(*args): + return cpptype.invoke(name,*args) + method.__name__ = name + return staticmethod(method) +_existing_classes = {} def get_cppclass(name): # lookup class + try: + return _existing_classes[name] + except KeyError: + pass # if failed, create + # TODO: handle base classes + cpptype = cppyy._type_byname(name) + d = {"_cppyyclass" : cpptype} + for f in cpptype.get_function_members(): + cppol = cpptype.get_overload(f) + if cppol.is_static(): + d[f] = make_static_function(cpptype, f) + else: + pass + + pycpptype = CppyyClass(name, (object,), d) - # create base classes - pass + return pycpptype + +# raise TypeError("no such C++ class %s" % name) + + +class _gbl(object): + """Global C++ namespace, i.e. ::.""" + + def __getattr__(self, attr): + try: + cppclass = get_cppclass(attr) + self.__dict__[attr] = cppclass + return cppclass + except TypeError: + raise AttributeError("'gbl' object has no attribute '%s'" % attr) _loaded_shared_libs = {} Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Thu Jul 8 12:38:17 2010 @@ -14,7 +14,7 @@ class TestCPPYYImplementation: def test_class_query(self): lib = interp_cppyy.load_lib(space, shared_lib) - w_cppyyclass = lib.type_byname("example01") + w_cppyyclass = interp_cppyy.type_byname(space, "example01") adddouble = w_cppyyclass.function_members["staticAddToDouble"] func, = adddouble.functions assert isinstance(func.executor, executor.DoubleExecutor) @@ -27,13 +27,14 @@ env = os.environ cls.w_example01 = cls.space.appexec([], """(): import cppyy - return cppyy.load_lib(%r)""" % (shared_lib, )) + cppyy.load_lib(%r) + return cppyy._type_byname('example01')""" % (shared_lib, )) def test_example01static_int(self): """Test passing of an int, returning of an int, and overloading on a differening number of arguments.""" import sys - t = self.example01.type_byname("example01") + t = self.example01 res = t.invoke("staticAddOneToInt", 1) assert res == 2 res = t.invoke("staticAddOneToInt", 1L) @@ -54,14 +55,14 @@ def test_example01static_double(self): """Test passing of a double and returning of a double on a static function.""" - t = self.example01.type_byname("example01") + t = self.example01 res = t.invoke("staticAddToDouble", 0.09) assert res == 0.09 + 0.01 def test_example01static_constcharp(self): """Test passing of a C string and returning of a C string on a static function.""" - t = self.example01.type_byname("example01") + t = self.example01 res = t.invoke("staticAtoi", "1") assert res == 1 @@ -76,7 +77,7 @@ def test_example01method_int(self): """Test passing of a int, returning of a int, and memory cleanup, on a method.""" - t = self.example01.type_byname("example01") + t = self.example01 assert t.invoke("getCount") == 0 instance = t.construct(7) assert t.invoke("getCount") == 1 @@ -98,7 +99,7 @@ def test_example01method_double(self): """Test passing of a double and returning of double on a method""" - t = self.example01.type_byname("example01") + t = self.example01 instance = t.construct(13) res = instance.invoke("addDataToDouble", 16) assert round(res-29, 8) == 0. @@ -110,7 +111,7 @@ """Test passing of a C string and returning of a C string on a method.""" - t = self.example01.type_byname("example01") + t = self.example01 instance = t.construct(42) res = instance.invoke("addDataToAtoi", "13") Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Thu Jul 8 12:38:17 2010 @@ -25,3 +25,35 @@ import cppyy lib2 = cppyy.load_lib(self.shared_lib) assert self.example01 is lib2 + + def testFindingAClass(self): + """Test the lookup of a class, and its caching.""" + import cppyy + example01_class = cppyy.gbl.example01 + cl2 = cppyy.gbl.example01 + assert example01_class is cl2 + + raises(AttributeError, "cppyy.gbl.nonexistingclass") + + def testCallingAStaticFunction(self): + """Test calling of static methods.""" + import cppyy, sys + example01_class = cppyy.gbl.example01 + res = example01_class.staticAddOneToInt(1) + assert res == 2 + + res = example01_class.staticAddOneToInt(1L) + assert res == 2 + res = example01_class.staticAddOneToInt(1, 2) + assert res == 4 + res = example01_class.staticAddOneToInt(-1) + assert res == 0 + res = example01_class.staticAddOneToInt(sys.maxint-1) + assert res == sys.maxint + res = example01_class.staticAddOneToInt(sys.maxint) + assert res == -sys.maxint-1 + + raises(TypeError, 'example01_class.staticAddOneToInt(1, [])') + raises(TypeError, 'example01_class.staticAddOneToInt(1.)') + raises(OverflowError, 'example01_class.staticAddOneToInt(sys.maxint+1)') + From arigo at codespeak.net Thu Jul 8 12:53:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Jul 2010 12:53:21 +0200 (CEST) Subject: [pypy-svn] r76018 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100708105321.AA94B282BE0@codespeak.net> Author: arigo Date: Thu Jul 8 12:53:20 2010 New Revision: 76018 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Log: Support two installations: either genreflex is installed, or ROOT is in $ROOTSYS. Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Thu Jul 8 12:53:20 2010 @@ -6,18 +6,18 @@ srcpath = py.path.local(__file__).dirpath().join("src") incpath = py.path.local(__file__).dirpath().join("include") -try: - rootincpath = os.path.join(os.environ["ROOTSYS"], "include") - rootlibpath = os.path.join(os.environ["ROOTSYS"], "lib") -except KeyError: - print 'please set ROOTSYS envar to the location of the ROOT installation' - raise +if os.environ.get("ROOTSYS"): + rootincpath = [os.path.join(os.environ["ROOTSYS"], "include")] + rootlibpath = [os.path.join(os.environ["ROOTSYS"], "lib")] +else: + rootincpath = [] + rootlibpath = [] eci = ExternalCompilationInfo( separate_module_files=[srcpath.join("reflexcwrapper.cxx")], - include_dirs=[incpath, rootincpath], + include_dirs=[incpath] + rootincpath, includes=["reflexcwrapper.h"], - library_dirs=[rootlibpath], + library_dirs=rootlibpath, libraries=["Reflex"], use_cpp_linker=True, ) Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile Thu Jul 8 12:53:20 2010 @@ -1,8 +1,13 @@ -LD_LIBRARY_PATH := ${LD_LIBRARY_PATH}:${ROOTSYS}/lib ROOTSYS := ${ROOTSYS} -example01Dict.so: example01.cxx - echo $(ROOTSYS) - $(ROOTSYS)/bin/genreflex example01.cxx - g++ -o $@ example01_rflx.cpp -shared -I$(ROOTSYS)/include -L$(ROOTSYS)/lib -lReflex +ifeq ($(ROOTSYS),) + genreflex=genreflex + cppflags= +else + genreflex=$(ROOTSYS)/bin/genreflex + cppflags=-I$(ROOTSYS)/include -L$(ROOTSYS)/lib +endif +example01Dict.so: example01.cxx + $(genreflex) example01.cxx + g++ -o $@ example01_rflx.cpp -shared -lReflex $(cppflags) Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Thu Jul 8 12:53:20 2010 @@ -1,4 +1,4 @@ -import py, os +import py, os, sys from pypy.conftest import gettestobjspace from pypy.module.cppyy import interp_cppyy, executor @@ -9,7 +9,11 @@ space = gettestobjspace(usemodules=['cppyy']) def setup_module(mod): - os.system("make") + if sys.platform == 'win32': + py.test.skip("win32 not supported so far") + err = os.system("cd '%s' && make" % currpath) + if err: + raise OSError("'make' failed (see stderr)") class TestCPPYYImplementation: def test_class_query(self): From wlav at codespeak.net Thu Jul 8 13:12:43 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Thu, 8 Jul 2010 13:12:43 +0200 (CEST) Subject: [pypy-svn] r76019 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100708111243.881E7282BE0@codespeak.net> Author: wlav Date: Thu Jul 8 13:12:42 2010 New Revision: 76019 Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Log: (wlav, cfbolz): add support for constructing an instance of a class and calling methods on it. Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Thu Jul 8 13:12:42 2010 @@ -118,7 +118,7 @@ args = self.prepare_arguments(args_w) result = capi.c_cppyy_construct(self.cpptype.handle, len(args_w), args) self.free_arguments(args) - return W_CPPObject(self.cpptype, result) + return W_CCPInstance(self.cpptype, result) class W_CPPOverload(Wrappable): @@ -216,7 +216,7 @@ construct = interp2app(W_CPPType.construct, unwrap_spec=['self', 'args_w']), ) -class W_CPPObject(Wrappable): +class W_CCPInstance(Wrappable): _immutable_ = True def __init__(self, cppclass, rawobject): self.space = cppclass.space @@ -237,8 +237,8 @@ capi.c_cppyy_destruct(self.cppclass.handle, self.rawobject) self.rawobject = NULL_VOIDP -W_CPPObject.typedef = TypeDef( - 'CPPObject', - invoke = interp2app(W_CPPObject.invoke, unwrap_spec=['self', str, 'args_w']), - destruct = interp2app(W_CPPObject.destruct, unwrap_spec=['self']), +W_CCPInstance.typedef = TypeDef( + 'CPPInstance', + invoke = interp2app(W_CCPInstance.invoke, unwrap_spec=['self', str, 'args_w']), + destruct = interp2app(W_CCPInstance.destruct, unwrap_spec=['self']), ) Modified: pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py Thu Jul 8 13:12:42 2010 @@ -7,10 +7,24 @@ def make_static_function(cpptype, name): def method(*args): - return cpptype.invoke(name,*args) + return cpptype.invoke(name, *args) method.__name__ = name return staticmethod(method) +def make_method(name): + def method(self, *args): + return self._cppinstance.invoke(name, *args) + method.__name__ = name + return method + +class CppyyObject(object): + def __init__(self, *args): + self._cppinstance = self._cppyyclass.construct(*args) + + def destruct(self): + self._cppinstance.destruct() + + _existing_classes = {} def get_cppclass(name): # lookup class @@ -28,9 +42,9 @@ if cppol.is_static(): d[f] = make_static_function(cpptype, f) else: - pass + d[f] = make_method(f) - pycpptype = CppyyClass(name, (object,), d) + pycpptype = CppyyClass(name, (CppyyObject,), d) return pycpptype @@ -45,7 +59,7 @@ cppclass = get_cppclass(attr) self.__dict__[attr] = cppclass return cppclass - except TypeError: + except TypeError, e: raise AttributeError("'gbl' object has no attribute '%s'" % attr) Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Thu Jul 8 13:12:42 2010 @@ -107,9 +107,12 @@ instance = t.construct(13) res = instance.invoke("addDataToDouble", 16) assert round(res-29, 8) == 0. + instance.destruct() instance = t.construct(-13) res = instance.invoke("addDataToDouble", 16) assert round(res-3, 8) == 0. + instance.destruct() + assert t.invoke("getCount") == 0 def test_example01method_constcharp(self): """Test passing of a C string and returning of a C string on a @@ -125,4 +128,6 @@ assert res == "54" res = instance.invoke("addToStringValue", "-12") assert res == "30" + instance.destruct() + assert t.invoke("getCount") == 0 Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Thu Jul 8 13:12:42 2010 @@ -56,4 +56,64 @@ raises(TypeError, 'example01_class.staticAddOneToInt(1, [])') raises(TypeError, 'example01_class.staticAddOneToInt(1.)') raises(OverflowError, 'example01_class.staticAddOneToInt(sys.maxint+1)') + res = example01_class.staticAddToDouble(0.09) + assert res == 0.09 + 0.01 + + res = example01_class.staticAtoi("1") + assert res == 1 + + res = example01_class.staticStrcpy("aap") + assert res == "aap" + + res = example01_class.staticStrcpy(u"aap") + assert res == "aap" + + raises(TypeError, 'example01_class.staticStrcpy(1.)') + + def test_ConstrucingAndCalling(self): + """Test object and method calls.""" + import cppyy, sys + example01_class = cppyy.gbl.example01 + assert example01_class.getCount() == 0 + instance = example01_class(7) + assert example01_class.getCount() == 1 + res = instance.addDataToInt(4) + assert res == 11 + res = instance.addDataToInt(-4) + assert res == 3 + instance.destruct() + assert example01_class.getCount() == 0 + raises(ReferenceError, 'instance.addDataToInt(4)') + return + + instance = example01_class(7) + instance2 = example01_class(8) + assert example01_class.getCount() == 2 + instance.destruct() + assert example01_class.getCount() == 1 + instance2.destruct() + assert example01_class.getCount() == 0 + + t = self.example01 + instance = example01_class(13) + res = instance.addDataToDouble(16) + assert round(res-29, 8) == 0. + instance.destruct() + instance = example01_class(-13) + res = instance.addDataToDouble(16) + assert round(res-3, 8) == 0. + + + t = self.example01 + instance = example01_class(42) + + res = instance.addDataToAtoi("13") + assert res == 55 + + res = instance.addToStringValue("12") + assert res == "54" + res = instance.addToStringValue("-12") + assert res == "30" + instance.destruct() + assert example01_class.getCount() == 0 From arigo at codespeak.net Thu Jul 8 13:25:01 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Jul 2010 13:25:01 +0200 (CEST) Subject: [pypy-svn] r76020 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src test Message-ID: <20100708112501.A00F8282BE0@codespeak.net> Author: arigo Date: Thu Jul 8 13:25:00 2010 New Revision: 76020 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile Log: (antocuni, arigo) Hack: manually do the fast-path construction enabled by --with-methptrgetter for methods of signature 'int m(int)' only. The JIT should speed that case up. Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Thu Jul 8 13:25:00 2010 @@ -22,6 +22,9 @@ use_cpp_linker=True, ) +C_METHPTRGETTER = lltype.FuncType([rffi.VOIDP], rffi.VOIDP) +C_METHPTRGETTER_PTR = lltype.Ptr(C_METHPTRGETTER) + c_cppyy_get_typehandle = rffi.llexternal( "cppyy_get_typehandle", [rffi.CCHARP], rffi.VOIDP, @@ -47,6 +50,10 @@ "cppyy_destruct", [rffi.VOIDP, rffi.VOIDP], lltype.Void, compilation_info=eci) +c_cppyy_get_methptr_getter = rffi.llexternal( + "cppyy_get_methptr_getter", + [rffi.VOIDP, rffi.INT], C_METHPTRGETTER_PTR, + compilation_info=eci) c_num_methods = rffi.llexternal( Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Thu Jul 8 13:25:00 2010 @@ -5,12 +5,15 @@ #ifdef __cplusplus extern "C" { #endif // ifdef __cplusplus + typedef void* (*cppyy_methptrgetter_t)(void*); + void* cppyy_get_typehandle(const char* class_name); void* cppyy_construct(void* handle, int numargs, void* args[]); long cppyy_call_l(void* handle, int method_index, void* self, int numargs, void* args[]); double cppyy_call_d(void* handle, int method_index, void* self, int numargs, void* args[]); void cppyy_destruct(void* handle, void* self); + cppyy_methptrgetter_t cppyy_get_methptr_getter(void* handle, int method_index); int num_methods(void* handle); char* method_name(void* handle, int method_index); Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Thu Jul 8 13:25:00 2010 @@ -50,17 +50,42 @@ self.arg_types = arg_types self.executor = executor.get_executor( result_type ) self.arg_converters = None + # + self.hack_call = arg_types == ['int'] and result_type == 'int' + # def call(self, cppthis, args_w): if self.executor is None: raise OperationError(self.space.w_TypeError, self.space.wrap("return type not handled")) + if self.hack_call: + try: + return self.do_hack_call(cppthis, args_w) + except NotImplementedError: + pass + args = self.prepare_arguments(args_w) try: return self.executor.execute(self.space, self, cppthis, len(args_w), args) finally: self.free_arguments(args) + INT_2_INT_FNPTR = lltype.Ptr(lltype.FuncType([rffi.VOIDP, rffi.INT], + rffi.INT)) + def do_hack_call(self, cppthis, args_w): + # hack: only for methods 'int m(int)' + if len(args_w) != 1: + raise OperationError(space.w_TypeError, space.wrap("wrong number of args")) + arg = self.space.c_int_w(args_w[0]) + methgetter = capi.c_cppyy_get_methptr_getter(self.cpptype.handle, + self.method_index) + if not methgetter: + raise NotImplementedError + funcptr = methgetter(cppthis) + funcptr = rffi.cast(self.INT_2_INT_FNPTR, funcptr) + result = funcptr(cppthis, arg) + return self.space.wrap(rffi.cast(lltype.Signed, result)) + def _build_converters(self): self.arg_converters = [converter.get_converter(arg_type) for arg_type in self.arg_types] @@ -74,7 +99,6 @@ self._build_converters() args = lltype.malloc(rffi.CArray(rffi.VOIDP), len(args_w), flavor='raw') for i in range(len(args_w)): - argtype = self.arg_types[i] conv = self.arg_converters[i] w_arg = args_w[i] try: Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Thu Jul 8 13:25:00 2010 @@ -57,18 +57,24 @@ t.Destruct(self, true); } -typedef void* (*MethPtrGetter)(void*); -static MethPtrGetter get_methptr_getter(Reflex::Member m) +static cppyy_methptrgetter_t get_methptr_getter(Reflex::Member m) { Reflex::PropertyList plist = m.Properties(); if (plist.HasProperty("MethPtrGetter")) { Reflex::Any& value = plist.PropertyValue("MethPtrGetter"); - return (MethPtrGetter)Reflex::any_cast(value); + return (cppyy_methptrgetter_t)Reflex::any_cast(value); } else return 0; } +cppyy_methptrgetter_t cppyy_get_methptr_getter(void* handle, int method_index) +{ + Reflex::Type t((Reflex::TypeName*)handle); + Reflex::Member m = t.FunctionMemberAt(method_index); + return get_methptr_getter(m); +} + int num_methods(void* handle) { Reflex::Type t((Reflex::TypeName*)handle); Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile Thu Jul 8 13:25:00 2010 @@ -8,6 +8,14 @@ cppflags=-I$(ROOTSYS)/include -L$(ROOTSYS)/lib endif +ifeq ($(shell $(genreflex) --help | grep -- --with-methptrgetter),) + genreflexflags= + cppflags2= +else + genreflexflags=--with-methptrgetter + cppflags2=-Wno-pmf-conversions +endif + example01Dict.so: example01.cxx - $(genreflex) example01.cxx - g++ -o $@ example01_rflx.cpp -shared -lReflex $(cppflags) + $(genreflex) example01.cxx $(genreflexflags) + g++ -o $@ example01_rflx.cpp -shared -lReflex $(cppflags) $(cppflags2) From hakanardo at codespeak.net Thu Jul 8 13:28:12 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 8 Jul 2010 13:28:12 +0200 (CEST) Subject: [pypy-svn] r76021 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100708112812.C4874282BE0@codespeak.net> Author: hakanardo Date: Thu Jul 8 13:28:11 2010 New Revision: 76021 Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: some list methods Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/app_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/app_array.py Thu Jul 8 13:28:11 2010 @@ -1,4 +1,5 @@ - +import operator + if True: def initiate(self, initializer): if initializer is not None: @@ -84,7 +85,34 @@ s+=struct.pack(self.typecode, self[i]) return s - + def __repr__(self): + if len(self) == 0: + return "array('%s')" % self.typecode + elif self.typecode == "c": + return "array('%s', %s)" % (self.typecode, repr(self.tostring())) + elif self.typecode == "u": + return "array('%s', %s)" % (self.typecode, repr(self.tounicode())) + else: + return "array('%s', %s)" % (self.typecode, repr(self.tolist())) + ##### list methods + def count(self, x): + """Return number of occurences of x in the array.""" + return operator.countOf(self, x) + def index(self, x): + """Return index of first occurence of x in the array.""" + return operator.indexOf(self, x) + + def remove(self, x): + """Remove the first occurence of x in the array.""" + self.pop(self.index(x)) + + def reverse(self): + """Reverse the order of the items in the array.""" + lst = self.tolist() + lst.reverse() + self._setlen(0) + self.fromlist(lst) + Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Thu Jul 8 13:28:11 2010 @@ -254,10 +254,19 @@ def descr_tostring(self): pbuf = rffi.cast(rffi.CCHARP, self.buffer) s = '' - for i in range(self.len * self.itemsize): + i=0 + while i < self.len * self.itemsize: s += pbuf[i] + i+=1 return self.space.wrap(s) + def descr_buffer(self): + from pypy.interpreter.buffer import StringLikeBuffer + space = self.space + return space.wrap(StringLikeBuffer(space, self.descr_tostring())) + descr_buffer.unwrap_spec = ['self'] + + def descr_itemsize(space, self): return space.wrap(self.itemsize) @@ -280,16 +289,34 @@ fromstring = interp2app(W_Array.descr_fromstring), fromunicode = appmethod('fromunicode'), fromfile = appmethod('fromfile'), + read = appmethod('fromfile'), _fromfile = appmethod('_fromfile'), fromlist = appmethod('fromlist'), tolist = interp2app(W_Array.descr_tolist), tounicode = appmethod('tounicode'), tofile = appmethod('tofile'), + write = appmethod('tofile'), #tostring = appmethod('tostring'), tostring = interp2app(W_Array.descr_tostring), _setlen = interp2app(W_Array.setlen), + __buffer__ = interp2app(W_Array.descr_buffer), + + __repr__ = appmethod('__repr__'), + count = appmethod('count'), + index = appmethod('index'), + remove = appmethod('remove'), + reverse = appmethod('reverse'), + + + # TODO: + # __cmp__ + #byteswap = + #buffer_info = + #__copy__ = + #__reduce__ = + # insert, pop, ) mytype.w_class = W_Array Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Thu Jul 8 13:28:11 2010 @@ -310,6 +310,31 @@ raises(ValueError, self.array('i').tounicode) assert self.array('u', unicode('hello')).tounicode() == unicode('hello') + def test_buffer(self): + assert buffer(self.array('h', 'Hi'))[1] == 'i' + + def test_list_methods(self): + assert repr(self.array('i')) == "array('i')" + assert repr(self.array('i', [1, 2, 3])) == "array('i', [1, 2, 3])" + assert repr(self.array('h')) == "array('h')" + + a=self.array('i', [1, 2, 3, 1, 2, 1]) + assert a.count(1) == 3 + assert a.count(2) == 2 + assert a.index(3) == 2 + assert a.index(2) == 1 + + a.reverse() + assert repr(a) == "array('i', [1, 2, 1, 3, 2, 1])" + + if False: + a.remove(3) + assert repr(a) == "array('i', [1, 2, 1, 2, 1])" + a.remove(1) + assert repr(a) == "array('i', [2, 1, 2, 1])" + + + #FIXME #def test_type(self): # for t in 'bBhHiIlLfdcu': From fijal at codespeak.net Thu Jul 8 15:15:57 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Jul 2010 15:15:57 +0200 (CEST) Subject: [pypy-svn] r76023 - pypy/trunk/pypy/rlib Message-ID: <20100708131557.3C66B282BFB@codespeak.net> Author: fijal Date: Thu Jul 8 15:15:55 2010 New Revision: 76023 Modified: pypy/trunk/pypy/rlib/rarithmetic.py Log: don't do type(int) when we're translated Modified: pypy/trunk/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/trunk/pypy/rlib/rarithmetic.py (original) +++ pypy/trunk/pypy/rlib/rarithmetic.py Thu Jul 8 15:15:55 2010 @@ -118,7 +118,7 @@ result = int(int(x)) # -2147483648.0 => -2147483648L => -2147483648 except (OverflowError, ValueError): # ValueError for int(nan) on Py>=2.6 raise OverflowError - if type(result) is not int: + if not objectmodel.we_are_translated() and type(result) is not int: raise OverflowError return result From hakanardo at codespeak.net Thu Jul 8 15:34:05 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 8 Jul 2010 15:34:05 +0200 (CEST) Subject: [pypy-svn] r76024 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100708133405.2763E282BFB@codespeak.net> Author: hakanardo Date: Thu Jul 8 15:34:03 2010 New Revision: 76024 Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: pickling and comparing Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/app_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/app_array.py Thu Jul 8 15:34:03 2010 @@ -116,3 +116,51 @@ self._setlen(0) self.fromlist(lst) + def __eq__(self, other): + if not self._isarray(other): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) == buffer(other._data) + else: + return self.tolist() == other.tolist() + + def __ne__(self, other): + if not self._isarray(other): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) != buffer(other._data) + else: + return self.tolist() != other.tolist() + + def __lt__(self, other): + if not self._isarray(other): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) < buffer(other._data) + else: + return self.tolist() < other.tolist() + + def __gt__(self, other): + if not self._isarray(other): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) > buffer(other._data) + else: + return self.tolist() > other.tolist() + + def __le__(self, other): + if not self._isarray(other): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) <= buffer(other._data) + else: + return self.tolist() <= other.tolist() + + def __ge__(self, other): + if not self._isarray(other): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) >= buffer(other._data) + else: + return self.tolist() >= other.tolist() + Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Thu Jul 8 15:34:03 2010 @@ -265,7 +265,22 @@ space = self.space return space.wrap(StringLikeBuffer(space, self.descr_tostring())) descr_buffer.unwrap_spec = ['self'] - + + def descr_isarray(self, w_other): + return self.space.wrap(isinstance(w_other, W_ArrayBase)) + + def descr_reduce(self): + space=self.space + if self.len>0: + args=[space.wrap(self.typecode), self.descr_tostring()] + else: + args=[space.wrap(self.typecode)] + from pypy.interpreter.mixedmodule import MixedModule + w_mod = space.getbuiltinmodule('array') + mod = space.interp_w(MixedModule, w_mod) + w_new_inst = mod.get('array') + return space.newtuple([w_new_inst, space.newtuple(args)]) + def descr_itemsize(space, self): @@ -308,6 +323,16 @@ index = appmethod('index'), remove = appmethod('remove'), reverse = appmethod('reverse'), + + __eq__ = appmethod('__eq__'), + __ne__ = appmethod('__ne__'), + __lt__ = appmethod('__lt__'), + __gt__ = appmethod('__gt__'), + __le__ = appmethod('__le__'), + __ge__ = appmethod('__ge__'), + + _isarray = interp2app(W_Array.descr_isarray), + __reduce__ = interp2app(W_Array.descr_reduce), # TODO: Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Thu Jul 8 15:34:03 2010 @@ -332,8 +332,59 @@ assert repr(a) == "array('i', [1, 2, 1, 2, 1])" a.remove(1) assert repr(a) == "array('i', [2, 1, 2, 1])" - - + + def test_compare(self): + a = self.array('i', [1, 2, 3]) + b = self.array('i', [1, 2, 3]) + c = self.array('i', [1, 3, 2]) + + assert (a == a) is True + assert (a == b) is True + assert (b == a) is True + assert (a == c) is False + assert (c == a) is False + + assert (a != a) is False + assert (a != b) is False + assert (b != a) is False + assert (a != c) is True + assert (c != a) is True + + assert (a < a) is False + assert (a < b) is False + assert (b < a) is False + assert (a < c) is True + assert (c < a) is False + + assert (a > a) is False + assert (a > b) is False + assert (b > a) is False + assert (a > c) is False + assert (c > a) is True + + assert (a <= a) is True + assert (a <= b) is True + assert (b <= a) is True + assert (a <= c) is True + assert (c <= a) is False + + assert (a >= a) is True + assert (a >= b) is True + assert (b >= a) is True + assert (a >= c) is False + assert (c >= a) is True + + def test_reduce(self): + import pickle + a = self.array('i', [1, 2, 3]) + s = pickle.dumps(a,1) + b = pickle.loads(s) + assert a == b + + a = self.array('l') + s = pickle.dumps(a,1) + b = pickle.loads(s) + assert len(b) == 0 and b.typecode == 'l' #FIXME #def test_type(self): From hakanardo at codespeak.net Thu Jul 8 15:38:23 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 8 Jul 2010 15:38:23 +0200 (CEST) Subject: [pypy-svn] r76025 - pypy/branch/interplevel-array/pypy/jit/tl Message-ID: <20100708133823.A8F13282BFA@codespeak.net> Author: hakanardo Date: Thu Jul 8 15:38:22 2010 New Revision: 76025 Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py Log: test Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py Thu Jul 8 15:38:22 2010 @@ -44,5 +44,5 @@ sa+=img[i] return sa -img=array(4) +img=array('i',(1,2,3,4)) print f(img) From hakanardo at codespeak.net Thu Jul 8 15:47:39 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 8 Jul 2010 15:47:39 +0200 (CEST) Subject: [pypy-svn] r76026 - in pypy/branch/interplevel-array: . lib-python lib_pypy lib_pypy/pypy_test pypy/module/array pypy/module/test_lib_pypy/ctypes_tests pypy/rlib Message-ID: <20100708134739.5913C282BFA@codespeak.net> Author: hakanardo Date: Thu Jul 8 15:47:36 2010 New Revision: 76026 Modified: pypy/branch/interplevel-array/ (props changed) pypy/branch/interplevel-array/lib-python/ (props changed) pypy/branch/interplevel-array/lib_pypy/ (props changed) pypy/branch/interplevel-array/lib_pypy/dbm.py (props changed) pypy/branch/interplevel-array/lib_pypy/pypy_test/test_functools.py (props changed) pypy/branch/interplevel-array/pypy/module/array/interp_array_try1.py (props changed) pypy/branch/interplevel-array/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py Log: svn merge -r76012:76023 svn+ssh://hakanardo at codespeak.net/svn/pypy/trunk Modified: pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py Thu Jul 8 15:47:36 2010 @@ -118,7 +118,7 @@ result = int(int(x)) # -2147483648.0 => -2147483648L => -2147483648 except (OverflowError, ValueError): # ValueError for int(nan) on Py>=2.6 raise OverflowError - if type(result) is not int: + if not objectmodel.we_are_translated() and type(result) is not int: raise OverflowError return result From benjamin at codespeak.net Thu Jul 8 15:55:19 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 8 Jul 2010 15:55:19 +0200 (CEST) Subject: [pypy-svn] r76027 - in pypy/trunk: lib-python lib_pypy lib_pypy/pypy_test pypy/module/test_lib_pypy/ctypes_tests Message-ID: <20100708135519.E44C7282BFA@codespeak.net> Author: benjamin Date: Thu Jul 8 15:55:18 2010 New Revision: 76027 Modified: pypy/trunk/lib-python/ (props changed) pypy/trunk/lib_pypy/dbm.py (props changed) pypy/trunk/lib_pypy/pypy_test/test_functools.py (props changed) pypy/trunk/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) Log: rip out non-root svn:mergeinfo From benjamin at codespeak.net Thu Jul 8 15:56:27 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 8 Jul 2010 15:56:27 +0200 (CEST) Subject: [pypy-svn] r76028 - pypy/trunk/lib_pypy Message-ID: <20100708135627.43FEF282BFA@codespeak.net> Author: benjamin Date: Thu Jul 8 15:56:25 2010 New Revision: 76028 Modified: pypy/trunk/lib_pypy/ (props changed) Log: kill another svn:mergeinfo From arigo at codespeak.net Thu Jul 8 16:02:22 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Jul 2010 16:02:22 +0200 (CEST) Subject: [pypy-svn] r76029 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100708140222.63902282BFA@codespeak.net> Author: arigo Date: Thu Jul 8 16:02:20 2010 New Revision: 76029 Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Log: Translation fixes. Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Thu Jul 8 16:02:20 2010 @@ -6,7 +6,7 @@ class FunctionExecutor(object): - def execute(self, space, func, num_args, args): + def execute(self, space, func, cppthis, num_args, args): raise NotImplementedError("abstract base class") Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Thu Jul 8 16:02:20 2010 @@ -74,9 +74,10 @@ rffi.INT)) def do_hack_call(self, cppthis, args_w): # hack: only for methods 'int m(int)' + space = self.space if len(args_w) != 1: raise OperationError(space.w_TypeError, space.wrap("wrong number of args")) - arg = self.space.c_int_w(args_w[0]) + arg = space.c_int_w(args_w[0]) methgetter = capi.c_cppyy_get_methptr_getter(self.cpptype.handle, self.method_index) if not methgetter: @@ -84,7 +85,7 @@ funcptr = methgetter(cppthis) funcptr = rffi.cast(self.INT_2_INT_FNPTR, funcptr) result = funcptr(cppthis, arg) - return self.space.wrap(rffi.cast(lltype.Signed, result)) + return space.wrap(rffi.cast(lltype.Signed, result)) def _build_converters(self): self.arg_converters = [converter.get_converter(arg_type) From fijal at codespeak.net Thu Jul 8 16:21:00 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Jul 2010 16:21:00 +0200 (CEST) Subject: [pypy-svn] r76032 - pypy/benchmarks/own/twisted Message-ID: <20100708142100.CA746282BFA@codespeak.net> Author: fijal Date: Thu Jul 8 16:20:58 2010 New Revision: 76032 Modified: pypy/benchmarks/own/twisted/web.py Log: Merge back from upstream Modified: pypy/benchmarks/own/twisted/web.py ============================================================================== --- pypy/benchmarks/own/twisted/web.py (original) +++ pypy/benchmarks/own/twisted/web.py Thu Jul 8 16:20:58 2010 @@ -11,8 +11,6 @@ benchmark vary wildly in their results. """ -from __future__ import division - from twisted.internet.protocol import Protocol from twisted.internet.defer import Deferred from twisted.web.server import Site @@ -20,7 +18,7 @@ from twisted.web.resource import Resource from twisted.web.client import ResponseDone, Agent -from benchlib import Client, driver, rotate_local_intf +from benchlib import Client, driver class BodyConsumer(Protocol): @@ -37,7 +35,7 @@ class Client(Client): def __init__(self, reactor, host, portNumber, agent): - self._requestLocation = 'http://%s:%d/' % (host, portNumber) + self._requestLocation = 'http://%s:%d/' % (host, portNumber,) self._agent = agent super(Client, self).__init__(reactor) @@ -56,16 +54,26 @@ +interface = 0 def main(reactor, duration): + global interface concurrency = 10 root = Resource() root.putChild('', Data("Hello, world", "text/plain")) + + interface += 1 + interface %= 255 port = reactor.listenTCP( - 0, Site(root), backlog=128, interface=rotate_local_intf()) + 0, Site(root), backlog=128, interface='127.0.0.%d' % (interface,)) agent = Agent(reactor) client = Client(reactor, port.getHost().host, port.getHost().port, agent) d = client.run(concurrency, duration) + def cleanup(passthrough): + d = port.stopListening() + d.addCallback(lambda ignored: passthrough) + return d + d.addBoth(cleanup) return d From arigo at codespeak.net Thu Jul 8 16:21:01 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Jul 2010 16:21:01 +0200 (CEST) Subject: [pypy-svn] r76033 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100708142101.D534D282BFA@codespeak.net> Author: arigo Date: Thu Jul 8 16:21:00 2010 New Revision: 76033 Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Log: Translation fix. Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Thu Jul 8 16:21:00 2010 @@ -132,7 +132,8 @@ assert not cppthis args = self.prepare_arguments(args_w) try: - return self.executor.execute(self.space, self, None, len(args_w), args) + return self.executor.execute(self.space, self, NULL_VOIDP, + len(args_w), args) finally: self.free_arguments(args) From arigo at codespeak.net Thu Jul 8 16:53:15 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Jul 2010 16:53:15 +0200 (CEST) Subject: [pypy-svn] r76034 - in pypy/trunk/pypy: rlib translator/c/test Message-ID: <20100708145315.96D0C282BFB@codespeak.net> Author: arigo Date: Thu Jul 8 16:53:14 2010 New Revision: 76034 Modified: pypy/trunk/pypy/rlib/rarithmetic.py pypy/trunk/pypy/translator/c/test/test_typed.py Log: Hopefully fix ovfcheck_float_to_int() by now. Test it carefully in the C backend. Modified: pypy/trunk/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/trunk/pypy/rlib/rarithmetic.py (original) +++ pypy/trunk/pypy/rlib/rarithmetic.py Thu Jul 8 16:53:14 2010 @@ -33,7 +33,7 @@ """ -import math +import sys, math from pypy.rpython import extregistry from pypy.rlib import objectmodel @@ -113,14 +113,23 @@ "NOT_RPYTHON" return _local_ovfcheck(int(long(a) << b)) -def ovfcheck_float_to_int(x): - try: - result = int(int(x)) # -2147483648.0 => -2147483648L => -2147483648 - except (OverflowError, ValueError): # ValueError for int(nan) on Py>=2.6 +# Strange things happening for float to int on 64 bit: +# int(float(i)) != i because of rounding issues. +# These are the minimum and maximum float value that can +# successfully be casted to an int. +if sys.maxint == 2147483647: + def ovfcheck_float_to_int(x): + if -2147483649.0 < x < 2147483648.0: + return int(x) raise OverflowError - if not objectmodel.we_are_translated() and type(result) is not int: +else: + # The following values are not quite +/-sys.maxint. + # Note the "<= x <" here, as opposed to "< x <" above. + # This is justified by test_typed in translator/c/test. + def ovfcheck_float_to_int(x): + if -9223372036854776832.0 <= x < 9223372036854775296.0: + return int(x) raise OverflowError - return result def compute_restype(self_type, other_type): if self_type is other_type: Modified: pypy/trunk/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_typed.py (original) +++ pypy/trunk/pypy/translator/c/test/test_typed.py Thu Jul 8 16:53:14 2010 @@ -789,3 +789,37 @@ return u'hello' + unichr(i) f = self.getcompiled(func, [int]) assert f(0x1234) == u'hello\u1234' + + def test_ovfcheck_float_to_int(self): + from pypy.rlib.rarithmetic import ovfcheck_float_to_int + + def func(fl): + try: + return ovfcheck_float_to_int(fl) + except OverflowError: + return -666 + f = self.getcompiled(func, [float]) + assert f(-123.0) == -123 + + for frac in [0.0, 0.01, 0.99]: + # strange things happening for float to int on 64 bit: + # int(float(i)) != i because of rounding issues + x = sys.maxint + while int(x + frac) > sys.maxint: + x -= 1 + assert f(x + frac) == int(x + frac) + + x = sys.maxint + while int(x - frac) <= sys.maxint: + x += 1 + assert f(x - frac) == -666 + + x = -sys.maxint-1 + while int(x - frac) < -sys.maxint-1: + x += 1 + assert f(x - frac) == int(x - frac) + + x = -sys.maxint-1 + while int(x + frac) >= -sys.maxint-1: + x -= 1 + assert f(x + frac) == -666 From arigo at codespeak.net Thu Jul 8 17:06:41 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Jul 2010 17:06:41 +0200 (CEST) Subject: [pypy-svn] r76035 - pypy/branch/reflex-support/pypy/jit/tl Message-ID: <20100708150641.913CC282BFB@codespeak.net> Author: arigo Date: Thu Jul 8 17:06:40 2010 New Revision: 76035 Modified: pypy/branch/reflex-support/pypy/jit/tl/pypyjit_demo.py Log: Fix with full relative path. Modified: pypy/branch/reflex-support/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/reflex-support/pypy/jit/tl/pypyjit_demo.py Thu Jul 8 17:06:40 2010 @@ -2,7 +2,7 @@ import time import cppyy -lib = cppyy.load_lib("example01Dict.so") +lib = cppyy.load_lib("../../module/cppyy/test/example01Dict.so") cls = lib.type_byname("example01") inst = cls.construct(-17) From antocuni at codespeak.net Thu Jul 8 17:08:23 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 8 Jul 2010 17:08:23 +0200 (CEST) Subject: [pypy-svn] r76036 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src Message-ID: <20100708150823.028DE282BFB@codespeak.net> Author: antocuni Date: Thu Jul 8 17:08:22 2010 New Revision: 76036 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Log: introduce two new types, cppyy_typehandle_t and cppyy_object_t, which are just void* but make the code a bit easier to understand Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Thu Jul 8 17:08:22 2010 @@ -22,67 +22,70 @@ use_cpp_linker=True, ) -C_METHPTRGETTER = lltype.FuncType([rffi.VOIDP], rffi.VOIDP) +C_TYPEHANDLE = rffi.VOIDP +C_OBJECT = rffi.VOIDP + +C_METHPTRGETTER = lltype.FuncType([C_OBJECT], rffi.VOIDP) C_METHPTRGETTER_PTR = lltype.Ptr(C_METHPTRGETTER) c_cppyy_get_typehandle = rffi.llexternal( "cppyy_get_typehandle", - [rffi.CCHARP], rffi.VOIDP, + [rffi.CCHARP], C_TYPEHANDLE, compilation_info=eci) c_callstatic_l = rffi.llexternal( "callstatic_l", - [rffi.VOIDP, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.LONG, + [C_TYPEHANDLE, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) c_cppyy_construct = rffi.llexternal( "cppyy_construct", - [rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.VOIDP, + [C_TYPEHANDLE, rffi.INT, rffi.VOIDPP], C_OBJECT, compilation_info=eci) c_cppyy_call_l = rffi.llexternal( "cppyy_call_l", - [rffi.VOIDP, rffi.INT, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.LONG, + [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) c_cppyy_call_d = rffi.llexternal( "cppyy_call_d", - [rffi.VOIDP, rffi.INT, rffi.VOIDP, rffi.INT, rffi.VOIDPP], rffi.DOUBLE, + [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.DOUBLE, compilation_info=eci) c_cppyy_destruct = rffi.llexternal( "cppyy_destruct", - [rffi.VOIDP, rffi.VOIDP], lltype.Void, + [C_TYPEHANDLE, C_OBJECT], lltype.Void, compilation_info=eci) c_cppyy_get_methptr_getter = rffi.llexternal( "cppyy_get_methptr_getter", - [rffi.VOIDP, rffi.INT], C_METHPTRGETTER_PTR, + [C_TYPEHANDLE, rffi.INT], C_METHPTRGETTER_PTR, compilation_info=eci) c_num_methods = rffi.llexternal( "num_methods", - [rffi.VOIDP], rffi.INT, + [C_TYPEHANDLE], rffi.INT, compilation_info=eci) c_method_name = rffi.llexternal( "method_name", - [rffi.VOIDP, rffi.INT], rffi.CCHARP, + [C_TYPEHANDLE, rffi.INT], rffi.CCHARP, compilation_info=eci) c_result_type_method = rffi.llexternal( "result_type_method", - [rffi.VOIDP, rffi.INT], rffi.CCHARP, + [C_TYPEHANDLE, rffi.INT], rffi.CCHARP, compilation_info=eci) c_num_args_method = rffi.llexternal( "num_args_method", - [rffi.VOIDP, rffi.INT], rffi.INT, + [C_TYPEHANDLE, rffi.INT], rffi.INT, compilation_info=eci) c_arg_type_method = rffi.llexternal( "arg_type_method", - [rffi.VOIDP, rffi.INT, rffi.INT], rffi.CCHARP, + [C_TYPEHANDLE, rffi.INT, rffi.INT], rffi.CCHARP, compilation_info=eci) c_is_constructor = rffi.llexternal( "is_constructor", - [rffi.VOIDP, rffi.INT], rffi.INT, + [C_TYPEHANDLE, rffi.INT], rffi.INT, compilation_info=eci) c_is_static = rffi.llexternal( "is_static", - [rffi.VOIDP, rffi.INT], rffi.INT, + [C_TYPEHANDLE, rffi.INT], rffi.INT, compilation_info=eci) c_myfree = rffi.llexternal( "myfree", Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Thu Jul 8 17:08:22 2010 @@ -5,25 +5,28 @@ #ifdef __cplusplus extern "C" { #endif // ifdef __cplusplus - typedef void* (*cppyy_methptrgetter_t)(void*); + typedef void* cppyy_typehandle_t; + typedef void* cppyy_object_t; + typedef void* (*cppyy_methptrgetter_t)(cppyy_object_t); - void* cppyy_get_typehandle(const char* class_name); + cppyy_typehandle_t cppyy_get_typehandle(const char* class_name); - void* cppyy_construct(void* handle, int numargs, void* args[]); - long cppyy_call_l(void* handle, int method_index, void* self, int numargs, void* args[]); - double cppyy_call_d(void* handle, int method_index, void* self, int numargs, void* args[]); - void cppyy_destruct(void* handle, void* self); - cppyy_methptrgetter_t cppyy_get_methptr_getter(void* handle, int method_index); - - int num_methods(void* handle); - char* method_name(void* handle, int method_index); - char* result_type_method(void* handle, int method_index); - int num_args_method(void* handle, int method_index); - char* arg_type_method(void* handle, int method_index, int index); - int is_constructor(void* handle, int method_index); - int is_static(void* handle, int method_index); + cppyy_object_t cppyy_construct(cppyy_typehandle_t handle, int numargs, void* args[]); + long cppyy_call_l(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); + double cppyy_call_d(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); + void cppyy_destruct(cppyy_typehandle_t handle, cppyy_object_t self); + cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_typehandle_t handle, int method_index); + + int num_methods(cppyy_typehandle_t handle); + char* method_name(cppyy_typehandle_t handle, int method_index); + char* result_type_method(cppyy_typehandle_t handle, int method_index); + int num_args_method(cppyy_typehandle_t handle, int method_index); + char* arg_type_method(cppyy_typehandle_t handle, int method_index, int index); + int is_constructor(cppyy_typehandle_t handle, int method_index); + int is_static(cppyy_typehandle_t handle, int method_index); void myfree(void* ptr); + #ifdef __cplusplus } #endif // ifdef __cplusplus Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Thu Jul 8 17:08:22 2010 @@ -4,12 +4,12 @@ #include -void* cppyy_get_typehandle(const char* class_name) { +cppyy_typehandle_t cppyy_get_typehandle(const char* class_name) { return Reflex::Type::ByName(class_name).Id(); } -void* cppyy_construct(void* handle, int numargs, void* args[]) { +cppyy_object_t cppyy_construct(cppyy_typehandle_t handle, int numargs, void* args[]) { std::vector arguments(args, args+numargs); Reflex::Type t((Reflex::TypeName*)handle); std::vector argtypes; @@ -22,8 +22,8 @@ return t.Construct(constructor_type, arguments).Address(); } -long cppyy_call_l(void* handle, int method_index, - void* self, int numargs, void* args[]) { +long cppyy_call_l(cppyy_typehandle_t handle, int method_index, + cppyy_object_t self, int numargs, void* args[]) { long result; std::vector arguments(args, args+numargs); Reflex::Type t((Reflex::TypeName*)handle); @@ -37,8 +37,8 @@ return result; } -double cppyy_call_d(void* handle, int method_index, - void* self, int numargs, void* args[]) { +double cppyy_call_d(cppyy_typehandle_t handle, int method_index, + cppyy_object_t self, int numargs, void* args[]) { double result; std::vector arguments(args, args+numargs); Reflex::Type t((Reflex::TypeName*)handle); @@ -52,7 +52,7 @@ return result; } -void cppyy_destruct(void* handle, void* self) { +void cppyy_destruct(cppyy_typehandle_t handle, cppyy_object_t self) { Reflex::Type t((Reflex::TypeName*)handle); t.Destruct(self, true); } @@ -68,7 +68,7 @@ return 0; } -cppyy_methptrgetter_t cppyy_get_methptr_getter(void* handle, int method_index) +cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); @@ -76,7 +76,7 @@ } -int num_methods(void* handle) { +int num_methods(cppyy_typehandle_t handle) { Reflex::Type t((Reflex::TypeName*)handle); for (int i = 0; i < (int)t.FunctionMemberSize(); i++) { Reflex::Member m = t.FunctionMemberAt(i); @@ -91,7 +91,7 @@ return t.FunctionMemberSize(); } -char* method_name(void* handle, int method_index) { +char* method_name(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); std::string name = m.Name(); @@ -100,7 +100,7 @@ return name_char; } -char* result_type_method(void* handle, int method_index) { +char* result_type_method(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); Reflex::Type rt = m.TypeOf().ReturnType(); @@ -110,13 +110,13 @@ return name_char; } -int num_args_method(void* handle, int method_index) { +int num_args_method(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); return m.FunctionParameterSize(); } -char* arg_type_method(void* handle, int method_index, int arg_index) { +char* arg_type_method(cppyy_typehandle_t handle, int method_index, int arg_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); @@ -127,13 +127,13 @@ return name_char; } -int is_constructor(void* handle, int method_index) { +int is_constructor(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); return m.IsConstructor(); } -int is_static(void* handle, int method_index) { +int is_static(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); return m.IsStatic(); From arigo at codespeak.net Thu Jul 8 17:23:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Jul 2010 17:23:46 +0200 (CEST) Subject: [pypy-svn] r76038 - pypy/branch/reflex-support/pypy/translator/tool Message-ID: <20100708152346.AA0A5282BFB@codespeak.net> Author: arigo Date: Thu Jul 8 17:23:45 2010 New Revision: 76038 Modified: pypy/branch/reflex-support/pypy/translator/tool/cbuild.py Log: Running some tests depending on cbuild would create externmod_1.so, externmod_2.so, etc., in a random directory, the list growing forever. Attempt to fix that by forcing the externmod_x.so to be created in the usession directory. Modified: pypy/branch/reflex-support/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/reflex-support/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/reflex-support/pypy/translator/tool/cbuild.py Thu Jul 8 17:23:45 2010 @@ -271,15 +271,10 @@ if not self.separate_module_files: return self if outputfilename is None: - # find more or less unique name there - basepath = py.path.local(self.separate_module_files[0]).dirpath() - pth = basepath.join('externmod').new(ext=host.so_ext) - num = 0 - while pth.check(): - pth = basepath.join( - 'externmod_%d' % (num,)).new(ext=host.so_ext) - num += 1 - outputfilename=pth.purebasename + global _counter_so_names + counter = _counter_so_names + _counter_so_names = counter + 1 + outputfilename = str(udir.join('externmod_%d' % counter)) lib = str(host.compile([], self, outputfilename=outputfilename, standalone=False)) d = self._copy_attributes() @@ -288,6 +283,8 @@ d['separate_module_sources'] = () return ExternalCompilationInfo(**d) +_counter_so_names = 0 + # ____________________________________________________________ # From arigo at codespeak.net Thu Jul 8 17:35:14 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Jul 2010 17:35:14 +0200 (CEST) Subject: [pypy-svn] r76039 - pypy/branch/reflex-support/pypy/jit/tl Message-ID: <20100708153514.D5D21282BFB@codespeak.net> Author: arigo Date: Thu Jul 8 17:35:13 2010 New Revision: 76039 Modified: pypy/branch/reflex-support/pypy/jit/tl/pypyjit_demo.py Log: Fixes. Modified: pypy/branch/reflex-support/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/reflex-support/pypy/jit/tl/pypyjit_demo.py Thu Jul 8 17:35:13 2010 @@ -3,12 +3,12 @@ import time import cppyy lib = cppyy.load_lib("../../module/cppyy/test/example01Dict.so") -cls = lib.type_byname("example01") +cls = cppyy._type_byname('example01') inst = cls.construct(-17) t1 = time.time() res = 0 -for i in range(1000000): - res += inst.invoke("add", i) +for i in range(10): + res += inst.invoke("addDataToInt", i) t2 = time.time() print t2 - t1 From wlav at codespeak.net Thu Jul 8 17:43:22 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Thu, 8 Jul 2010 17:43:22 +0200 (CEST) Subject: [pypy-svn] r76040 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src test Message-ID: <20100708154322.82D93282BFB@codespeak.net> Author: wlav Date: Thu Jul 8 17:43:20 2010 New Revision: 76040 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Log: (cfbolz, wlav) Implementation of overloads for constructors from the python side. Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Thu Jul 8 17:43:20 2010 @@ -33,13 +33,17 @@ [rffi.CCHARP], C_TYPEHANDLE, compilation_info=eci) -c_callstatic_l = rffi.llexternal( - "callstatic_l", - [C_TYPEHANDLE, rffi.INT, rffi.INT, rffi.VOIDPP], rffi.LONG, - compilation_info=eci) -c_cppyy_construct = rffi.llexternal( - "cppyy_construct", - [C_TYPEHANDLE, rffi.INT, rffi.VOIDPP], C_OBJECT, +c_cppyy_allocate = rffi.llexternal( + "cppyy_allocate", + [C_TYPEHANDLE], rffi.VOIDP, + compilation_info=eci) +c_cppyy_deallocate = rffi.llexternal( + "cppyy_deallocate", + [C_TYPEHANDLE, C_OBJECT], rffi.VOIDP, + compilation_info=eci) +c_cppyy_call_v = rffi.llexternal( + "cppyy_call_v", + [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], lltype.Void, compilation_info=eci) c_cppyy_call_l = rffi.llexternal( "cppyy_call_l", Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Thu Jul 8 17:43:20 2010 @@ -10,6 +10,11 @@ raise NotImplementedError("abstract base class") +class VoidExecutor(object): + def execute(self, space, func, cppthis, num_args, args): + capi.c_cppyy_call_v(func.cpptype.handle, func.method_index, cppthis, num_args, args) + return space.w_None + class LongExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): result = capi.c_cppyy_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) @@ -37,6 +42,7 @@ # raise TypeError("no clue what %s is" % name) +_executors["void"] = VoidExecutor() _executors["int"] = LongExecutor() _executors["long"] = LongExecutor() _executors["double"] = DoubleExecutor() Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Thu Jul 8 17:43:20 2010 @@ -11,7 +11,10 @@ cppyy_typehandle_t cppyy_get_typehandle(const char* class_name); - cppyy_object_t cppyy_construct(cppyy_typehandle_t handle, int numargs, void* args[]); + void* cppyy_allocate(cppyy_typehandle_t handle); + void cppyy_deallocate(cppyy_typehandle_t handle, cppyy_object_t instance); + + void cppyy_call_v(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); long cppyy_call_l(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); double cppyy_call_d(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); void cppyy_destruct(cppyy_typehandle_t handle, cppyy_object_t self); Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Thu Jul 8 17:43:20 2010 @@ -138,13 +138,16 @@ self.free_arguments(args) -class CPPConstructor(CPPFunction): +class CPPConstructor(CPPMethod): def call(self, cppthis, args_w): assert not cppthis - args = self.prepare_arguments(args_w) - result = capi.c_cppyy_construct(self.cpptype.handle, len(args_w), args) - self.free_arguments(args) - return W_CCPInstance(self.cpptype, result) + newthis = capi.c_cppyy_allocate(self.cpptype.handle) + try: + CPPMethod.call(self, newthis, args_w) + except Exception, e: + capi.c_cppyy_deallocate(self.cpptype.handle, newthis) + raise + return W_CCPInstance(self.cpptype, newthis) class W_CPPOverload(Wrappable): Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Thu Jul 8 17:43:20 2010 @@ -9,17 +9,25 @@ } -cppyy_object_t cppyy_construct(cppyy_typehandle_t handle, int numargs, void* args[]) { +void* cppyy_allocate(cppyy_typehandle_t handle) { + return Reflex::Type((Reflex::TypeName*)handle).Allocate(); +} + +void cppyy_deallocate(cppyy_typehandle_t handle, cppyy_object_t instance) { + Reflex::Type((Reflex::TypeName*)handle).Deallocate(instance); +} + +void cppyy_call_v(cppyy_typehandle_t handle, int method_index, + cppyy_object_t self, int numargs, void* args[]) { std::vector arguments(args, args+numargs); Reflex::Type t((Reflex::TypeName*)handle); - std::vector argtypes; - argtypes.reserve(numargs); - for (int i = 0; i < numargs; i++) { - argtypes.push_back(Reflex::Type::ByName("int")); + Reflex::Member m = t.FunctionMemberAt(method_index); + if (self) { + Reflex::Object o(t, self); + m.Invoke(o, 0, arguments); + } else { + m.Invoke(0, arguments); } - Reflex::Type constructor_type = Reflex::FunctionTypeBuilder( - Reflex::Type::ByName("void"), argtypes); - return t.Construct(constructor_type, arguments).Address(); } long cppyy_call_l(cppyy_typehandle_t handle, int method_index, Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx Thu Jul 8 17:43:20 2010 @@ -4,6 +4,22 @@ #include #include + +class payload { +public: + payload(double d) : m_data(d) {} + +// cctor needed as long as we have no converter for const& objects + payload( const payload& p ) : m_data(p.m_data) {} + + double getData() { return m_data; } + void setData(double d) { m_data = d; } + +private: + double m_data; +}; + + class example01 { public: static int count; @@ -48,6 +64,9 @@ ::strcpy(strout, strin); return strout; } + static void setPayload( payload* p, double d ) { + p->setData(d); + } static int getCount() { std::cout << "getcount called" << std::endl; Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Thu Jul 8 17:43:20 2010 @@ -70,9 +70,9 @@ raises(TypeError, 'example01_class.staticStrcpy(1.)') - def test_ConstrucingAndCalling(self): + def test_ConstructingAndCalling(self): """Test object and method calls.""" - import cppyy, sys + import cppyy example01_class = cppyy.gbl.example01 assert example01_class.getCount() == 0 instance = example01_class(7) @@ -114,6 +114,22 @@ assert res == "54" res = instance.addToStringValue("-12") assert res == "30" + + res = instance.staticAddOneToInt(1L) + assert res == 2 + instance.destruct() assert example01_class.getCount() == 0 + def testPassingOfAnObjectByPointer(self): + import cppyy + example01_class = cppyy.gbl.example01 + payload_class = cppyy.gbl.payload + + e = example01_class(14) + pl = payload_class(3.14) + + + pl.destruct() + e.destruct() + assert example01_class.getCount() == 0 From arigo at codespeak.net Thu Jul 8 18:01:44 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Jul 2010 18:01:44 +0200 (CEST) Subject: [pypy-svn] r76041 - in pypy/branch/reflex-support/pypy/jit: metainterp tl Message-ID: <20100708160144.06E8E282BFA@codespeak.net> Author: arigo Date: Thu Jul 8 18:01:43 2010 New Revision: 76041 Modified: pypy/branch/reflex-support/pypy/jit/metainterp/pyjitpl.py pypy/branch/reflex-support/pypy/jit/metainterp/warmspot.py pypy/branch/reflex-support/pypy/jit/tl/pypyjit_child.py Log: Update pypyjit_child.py to force the jitcodes to be dumped to disk in that case. Modified: pypy/branch/reflex-support/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/reflex-support/pypy/jit/metainterp/pyjitpl.py Thu Jul 8 18:01:43 2010 @@ -1099,10 +1099,12 @@ """The 'residual_call' operation is emitted in two cases: when we have to generate a residual CALL operation, but also to handle an indirect_call that may need to be inlined.""" - assert isinstance(funcbox, Const) - sd = self.metainterp.staticdata - key = sd.cpu.ts.getaddr_for_box(funcbox) - jitcode = sd.bytecode_for_address(key) + if isinstance(funcbox, Const): + sd = self.metainterp.staticdata + key = sd.cpu.ts.getaddr_for_box(funcbox) + jitcode = sd.bytecode_for_address(key) + else: + jitcode = None # rare case (e.g. indirect low-level calls) if jitcode is not None: # we should follow calls to this graph return self.metainterp.perform_call(jitcode, argboxes) Modified: pypy/branch/reflex-support/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/reflex-support/pypy/jit/metainterp/warmspot.py Thu Jul 8 18:01:43 2010 @@ -66,11 +66,14 @@ def jittify_and_run(interp, graph, args, repeat=1, backendopt=False, trace_limit=sys.maxint, - debug_level=DEBUG_STEPS, inline=False, **kwds): + debug_level=DEBUG_STEPS, inline=False, + write_jitcodes_directory=False, **kwds): translator = interp.typer.annotator.translator translator.config.translation.gc = "boehm" translator.config.translation.list_comprehension_operations = True - warmrunnerdesc = WarmRunnerDesc(translator, backendopt=backendopt, **kwds) + warmrunnerdesc = WarmRunnerDesc(translator, backendopt=backendopt, + write_jitcodes_directory=write_jitcodes_directory, + **kwds) for jd in warmrunnerdesc.jitdrivers_sd: jd.warmstate.set_param_threshold(3) # for tests jd.warmstate.set_param_trace_eagerness(2) # for tests @@ -144,7 +147,8 @@ class WarmRunnerDesc(object): def __init__(self, translator, policy=None, backendopt=True, CPUClass=None, - optimizer=None, ProfilerClass=EmptyProfiler, **kwds): + optimizer=None, ProfilerClass=EmptyProfiler, + write_jitcodes_directory=False, **kwds): pyjitpl._warmrunnerdesc = self # this is a global for debugging only! self.set_translator(translator) self.build_cpu(CPUClass, **kwds) @@ -173,6 +177,8 @@ self.rewrite_jit_merge_points(policy) verbose = not self.cpu.translate_support_code + if write_jitcodes_directory: + verbose = False self.codewriter.make_jitcodes(verbose=verbose) self.rewrite_can_enter_jits() self.rewrite_set_param() Modified: pypy/branch/reflex-support/pypy/jit/tl/pypyjit_child.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/tl/pypyjit_child.py (original) +++ pypy/branch/reflex-support/pypy/jit/tl/pypyjit_child.py Thu Jul 8 18:01:43 2010 @@ -35,5 +35,6 @@ warmspot.jittify_and_run(interp, graph, [], policy=policy, listops=True, CPUClass=CPUClass, backendopt=True, inline=True, + write_jitcodes_directory=True, optimizer=OPTIMIZER_FULL) From antocuni at codespeak.net Thu Jul 8 18:04:44 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 8 Jul 2010 18:04:44 +0200 (CEST) Subject: [pypy-svn] r76042 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src Message-ID: <20100708160444.3DE34282BFA@codespeak.net> Author: antocuni Date: Thu Jul 8 18:04:42 2010 New Revision: 76042 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Log: (antocuni, arigo around) huge pile of custom hacks to convince the jit that the function pointer to the method that we get is constant, given the dynamic class of the cpp object Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Thu Jul 8 18:04:42 2010 @@ -91,6 +91,11 @@ "is_static", [C_TYPEHANDLE, rffi.INT], rffi.INT, compilation_info=eci) +c_dynamic_type = rffi.llexternal( + "dynamic_type", + [C_TYPEHANDLE, C_OBJECT], C_TYPEHANDLE, + compilation_info=eci) + c_myfree = rffi.llexternal( "myfree", [rffi.VOIDP], lltype.Void, Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Thu Jul 8 18:04:42 2010 @@ -27,6 +27,7 @@ char* arg_type_method(cppyy_typehandle_t handle, int method_index, int index); int is_constructor(cppyy_typehandle_t handle, int method_index); int is_static(cppyy_typehandle_t handle, int method_index); + cppyy_typehandle_t dynamic_type(cppyy_typehandle_t handle, cppyy_object_t self); void myfree(void* ptr); Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Thu Jul 8 18:04:42 2010 @@ -78,11 +78,11 @@ if len(args_w) != 1: raise OperationError(space.w_TypeError, space.wrap("wrong number of args")) arg = space.c_int_w(args_w[0]) - methgetter = capi.c_cppyy_get_methptr_getter(self.cpptype.handle, - self.method_index) - if not methgetter: - raise NotImplementedError - funcptr = methgetter(cppthis) + + cpptype = jit.hint(self.cpptype, promote=True) + cpp_dynamic_type = capi.c_dynamic_type(cpptype.handle, cppthis) + cppthis_holder.cppthis = cppthis + funcptr = cpptype.get_methptr(cpp_dynamic_type, self.method_index) funcptr = rffi.cast(self.INT_2_INT_FNPTR, funcptr) result = funcptr(cppthis, arg) return space.wrap(rffi.cast(lltype.Signed, result)) @@ -184,6 +184,10 @@ is_static = interp2app(W_CPPOverload.is_static, unwrap_spec=['self']), ) +# hack hack hack (see get_methptr) +class _CppthisHolder(object): + pass +cppthis_holder = _CppthisHolder() class W_CPPType(Wrappable): _immutable_fields_ = ["name","handle"] @@ -229,6 +233,18 @@ def get_overload(self, name): return self.function_members[name] + @jit.purefunction + def get_methptr(self, cpp_dynamic_type, method_index): + # hack hack hack: we can't really pass cppthis as an argument as it's + # not constant, but as long as it corresponds to cpp_dynamic_type, the + # function is still pure + cppthis = cppthis_holder.cppthis + methgetter = capi.c_cppyy_get_methptr_getter(self.handle, method_index) + if not methgetter: + raise NotImplementedError + return methgetter(cppthis) + + def invoke(self, name, args_w): overload = self.get_overload(name) return overload.call(NULL_VOIDP, args_w) Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Thu Jul 8 18:04:42 2010 @@ -147,6 +147,12 @@ return m.IsStatic(); } +cppyy_typehandle_t dynamic_type(cppyy_typehandle_t handle, cppyy_object_t self) { + Reflex::Type t((Reflex::TypeName*)handle); + const Reflex::Object* obj = (const Reflex::Object*)self; + return t.DynamicType((*obj)).Id(); +} + void myfree(void* ptr) { free(ptr); } From antocuni at codespeak.net Thu Jul 8 18:06:03 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 8 Jul 2010 18:06:03 +0200 (CEST) Subject: [pypy-svn] r76043 - pypy/branch/reflex-support/pypy/config Message-ID: <20100708160603.8A313282BFA@codespeak.net> Author: antocuni Date: Thu Jul 8 18:06:02 2010 New Revision: 76043 Modified: pypy/branch/reflex-support/pypy/config/pypyoption.py Log: always enable this module on the branch Modified: pypy/branch/reflex-support/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/reflex-support/pypy/config/pypyoption.py (original) +++ pypy/branch/reflex-support/pypy/config/pypyoption.py Thu Jul 8 18:06:02 2010 @@ -30,7 +30,7 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl", "cpyext"] + "thread", "itertools", "pyexpat", "_ssl", "cpyext", "cppyy"] )) working_oo_modules = default_modules.copy() From arigo at codespeak.net Thu Jul 8 18:17:19 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Jul 2010 18:17:19 +0200 (CEST) Subject: [pypy-svn] r76044 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100708161719.2AD94282BFA@codespeak.net> Author: arigo Date: Thu Jul 8 18:17:14 2010 New Revision: 76044 Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py Log: Translation fix. Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Thu Jul 8 18:17:14 2010 @@ -10,7 +10,7 @@ raise NotImplementedError("abstract base class") -class VoidExecutor(object): +class VoidExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): capi.c_cppyy_call_v(func.cpptype.handle, func.method_index, cppthis, num_args, args) return space.w_None From hakanardo at codespeak.net Thu Jul 8 19:04:09 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 8 Jul 2010 19:04:09 +0200 (CEST) Subject: [pypy-svn] r76045 - pypy/branch/interplevel-array/pypy/module/array/test Message-ID: <20100708170409.BF1DF282BFA@codespeak.net> Author: hakanardo Date: Thu Jul 8 19:04:08 2010 New Revision: 76045 Added: pypy/branch/interplevel-array/pypy/module/array/test/sum.c pypy/branch/interplevel-array/pypy/module/array/test/sumtst.c Modified: pypy/branch/interplevel-array/pypy/module/array/test/Makefile pypy/branch/interplevel-array/pypy/module/array/test/inttst.py pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py Log: performace tests Modified: pypy/branch/interplevel-array/pypy/module/array/test/Makefile ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/Makefile (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/Makefile Thu Jul 8 19:04:08 2010 @@ -4,3 +4,5 @@ gcc -o $@ $^ loop: loop.o gcc -o $@ $^ +sum: sumtst.o sum.o + gcc -o $@ $^ \ No newline at end of file Modified: pypy/branch/interplevel-array/pypy/module/array/test/inttst.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/inttst.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/inttst.py Thu Jul 8 19:04:08 2010 @@ -1,19 +1,26 @@ #!/usr/bin/python from time import time -from array import array -#img=array('B',(0,)*640*480); -#intimg=array('I',(0,)*640*480); -img=array(640*480); -intimg=array(640*480); +from array import array #, simple_array -def f(): +def f(img, intimg): l=0 - for i in xrange(640,640*480): + i=640 + while i<640*480: l+=img[i] intimg[i]=intimg[i-640]+l + i+=1 + return l + +if True: + img=array('d','\x00'*640*480*8) + intimg=array('d','\x00'*640*480*8) +else: + img=simple_array(640*480) + intimg=simple_array(640*480) + start=time() -for l in range(500): f() +for l in range(500): f(img, intimg) print time()-start Added: pypy/branch/interplevel-array/pypy/module/array/test/sum.c ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/test/sum.c Thu Jul 8 19:04:08 2010 @@ -0,0 +1,7 @@ +double sum(double *img) { + int l=0; + for (int i=0; i<640*480; i++) { + l+=img[i]; + } + return l; +} Added: pypy/branch/interplevel-array/pypy/module/array/test/sumtst.c ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/test/sumtst.c Thu Jul 8 19:04:08 2010 @@ -0,0 +1,8 @@ + +double sum(double *img); + +void main() { + double *img=malloc(640*480*4*sizeof(double)); + int sa=0; + for (int l=0; l<500; l++) sum(img); +} Modified: pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py Thu Jul 8 19:04:08 2010 @@ -1,8 +1,8 @@ #!/usr/bin/python -from array import array +from array import array, simple_array -img=array(640*480); -def f(): +#img=array('d',(0,)*640*480); +def f(img): l=0 i=0; while i<640*480: @@ -10,6 +10,14 @@ i+=1 return l +if False: + img=array('d', '\x00'*640*480*8) +else: + img=simple_array(640*480) + +for l in range(500): f(img) +#print f(img) -#for l in range(500): f() -print f() +# C pypy-simple pypy cpython +# sumtst: 0m0.630s 0m0.659s 0m9.185s 0m33.447s +# intimg: 0m0.646s 0m1.404s 0m26.850s 1m0.279s From wlav at codespeak.net Thu Jul 8 19:37:04 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Thu, 8 Jul 2010 19:37:04 +0200 (CEST) Subject: [pypy-svn] r76046 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src test Message-ID: <20100708173704.134B3282BFA@codespeak.net> Author: wlav Date: Thu Jul 8 19:37:02 2010 New Revision: 76046 Added: pypy/branch/reflex-support/pypy/module/cppyy/helper.py (contents, props changed) pypy/branch/reflex-support/pypy/module/cppyy/test/test_helper.py (contents, props changed) Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Log: (wlav, cfbolz): work enough to be able to pass an instance as an argument. Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Thu Jul 8 19:37:02 2010 @@ -91,6 +91,10 @@ "is_static", [C_TYPEHANDLE, rffi.INT], rffi.INT, compilation_info=eci) +c_is_subtype = rffi.llexternal( + "is_subtype", + [C_TYPEHANDLE, C_TYPEHANDLE], rffi.INT, + compilation_info=eci) c_dynamic_type = rffi.llexternal( "dynamic_type", [C_TYPEHANDLE, C_OBJECT], C_TYPEHANDLE, Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Thu Jul 8 19:37:02 2010 @@ -1,5 +1,8 @@ +from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cppyy import helper, capi + _converters = {} class TypeConverter(object): @@ -30,13 +33,49 @@ x = rffi.str2charp(arg) return rffi.cast(rffi.VOIDP, x) -def get_converter(name): +class InstancePtrConverter(TypeConverter): + _immutable_ = True + def __init__(self, space, cpptype): + self.cpptype = cpptype + + def convert_argument(self, space, w_obj): + from pypy.module.cppyy import interp_cppyy + w_cppinstance = space.findattr(w_obj, space.wrap("_cppinstance")) + if w_cppinstance is not None: + w_obj = w_cppinstance + obj = space.interpclass_w(w_obj) + if isinstance(obj, interp_cppyy.W_CCPInstance): + if capi.c_is_subtype(obj.cppclass.handle, self.cpptype.handle): + return obj.rawobject + raise OperationError(space.w_TypeError, + space.wrap("cannot pass %s as %s" % ( + space.type(w_obj).getname(space, "?"), + self.cpptype.name))) + def free_argument(self, arg): + pass + + +def get_converter(space, name): + from pypy.module.cppyy import interp_cppyy + # The matching of the name to a converter should follow: + # 1) full, exact match + # 2) match of decorated, unqualified type + # 3) accept const ref as by value + # 4) accept ref as pointer + # 5) generalized cases (covers basically all user classes) + try: return _converters[name] except KeyError: pass - - raise TypeError("no clue what %s is" % name) + compound = helper.compound(name) + cpptype = interp_cppyy.type_byname(space, helper.clean_type(name)) + if compound == "*": + return InstancePtrConverter(space, cpptype) + + print name + + raise OperationError(space.w_TypeError, space.wrap("no clue what %s is" % name)) _converters["int"] = IntConverter() _converters["double"] = DoubleConverter() Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Thu Jul 8 19:37:02 2010 @@ -32,7 +32,7 @@ result = capi.charp2str_free(ccpresult) return space.wrap(result) -def get_executor(name): +def get_executor(space, name): try: return _executors[name] except KeyError: Added: pypy/branch/reflex-support/pypy/module/cppyy/helper.py ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/helper.py Thu Jul 8 19:37:02 2010 @@ -0,0 +1,19 @@ +from pypy.rlib import rstring + +def compound(name): + name = "".join(rstring.split(name, "const")) # poor man's replace + i = _find_qualifier_index(name) + return "".join(name[i:].split(" ")) + +def _find_qualifier_index(name): + i = len(name) + for i in range(len(name) - 1, -1, -1): + c = name[i] + if c.isalnum() or c == ">": + break + return i + 1 + +def clean_type(name): + assert name.find("const") == -1 + i = _find_qualifier_index(name) + return name[:i].strip() Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Thu Jul 8 19:37:02 2010 @@ -27,6 +27,8 @@ char* arg_type_method(cppyy_typehandle_t handle, int method_index, int index); int is_constructor(cppyy_typehandle_t handle, int method_index); int is_static(cppyy_typehandle_t handle, int method_index); + int is_subtype(cppyy_typehandle_t h1, cppyy_typehandle_t h2); + cppyy_typehandle_t dynamic_type(cppyy_typehandle_t handle, cppyy_object_t self); void myfree(void* ptr); Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Thu Jul 8 19:37:02 2010 @@ -20,10 +20,23 @@ return W_CPPLibrary(space, cdll) load_lib.unwrap_spec = [ObjSpace, str] +class State(object): + def __init__(self, space): + self.cpptype_cache = {} + def type_byname(space, name): + state = space.fromcache(State) + try: + return state.cpptype_cache[name] + except KeyError: + pass + handle = capi.c_cppyy_get_typehandle(name) if handle: - return W_CPPType(space, name, handle) + cpptype = W_CPPType(space, name, handle) + state.cpptype_cache[name] = cpptype + return cpptype + raise OperationError(space.w_TypeError, space.wrap("no such C++ class %s" % name)) type_byname.unwrap_spec = [ObjSpace, str] @@ -48,7 +61,7 @@ self.space = cpptype.space self.method_index = method_index self.arg_types = arg_types - self.executor = executor.get_executor( result_type ) + self.executor = executor.get_executor(self.space, result_type) self.arg_converters = None # self.hack_call = arg_types == ['int'] and result_type == 'int' @@ -88,7 +101,7 @@ return space.wrap(rffi.cast(lltype.Signed, result)) def _build_converters(self): - self.arg_converters = [converter.get_converter(arg_type) + self.arg_converters = [converter.get_converter(self.space, arg_type) for arg_type in self.arg_types] @jit.unroll_safe @@ -261,6 +274,8 @@ construct = interp2app(W_CPPType.construct, unwrap_spec=['self', 'args_w']), ) + + class W_CCPInstance(Wrappable): _immutable_ = True def __init__(self, cppclass, rawobject): Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Thu Jul 8 19:37:02 2010 @@ -147,6 +147,14 @@ return m.IsStatic(); } +int is_subtype(cppyy_typehandle_t h1, cppyy_typehandle_t h2) { + if (h1 == h2) + return 1; + Reflex::Type t1((Reflex::TypeName*)h1); + Reflex::Type t2((Reflex::TypeName*)h2); + return (int)t2.HasBase(t1); +} + cppyy_typehandle_t dynamic_type(cppyy_typehandle_t handle, cppyy_object_t self) { Reflex::Type t((Reflex::TypeName*)handle); const Reflex::Object* obj = (const Reflex::Object*)self; Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Thu Jul 8 19:37:02 2010 @@ -19,6 +19,8 @@ def test_class_query(self): lib = interp_cppyy.load_lib(space, shared_lib) w_cppyyclass = interp_cppyy.type_byname(space, "example01") + w_cppyyclass2 = interp_cppyy.type_byname(space, "example01") + assert space.is_w(w_cppyyclass, w_cppyyclass2) adddouble = w_cppyyclass.function_members["staticAddToDouble"] func, = adddouble.functions assert isinstance(func.executor, executor.DoubleExecutor) Added: pypy/branch/reflex-support/pypy/module/cppyy/test/test_helper.py ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_helper.py Thu Jul 8 19:37:02 2010 @@ -0,0 +1,11 @@ +from pypy.module.cppyy import helper + +def test_compound(): + assert helper.compound("int*") == "*" + assert helper.compound("int* const *&") == "**&" + assert helper.compound("std::vector*") == "*" + + +def test_clean_type(): + assert helper.clean_type(" int***") == "int" + assert helper.clean_type("std::vector&") == "std::vector" Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Thu Jul 8 19:37:02 2010 @@ -128,7 +128,12 @@ e = example01_class(14) pl = payload_class(3.14) + assert round(pl.getData()-3.14, 8) == 0 + example01_class.setPayload(pl._cppinstance, 41.) + assert pl.getData() == 41. + example01_class.setPayload(pl, 43.) + assert pl.getData() == 43. pl.destruct() e.destruct() From jcreigh at codespeak.net Thu Jul 8 20:03:06 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 8 Jul 2010 20:03:06 +0200 (CEST) Subject: [pypy-svn] r76047 - pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform Message-ID: <20100708180306.B3FD2282BFA@codespeak.net> Author: jcreigh Date: Thu Jul 8 20:03:04 2010 New Revision: 76047 Modified: pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py Log: make _compare_gcmap_entries return INT, not Signed (translation fix) Modified: pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py Thu Jul 8 20:03:04 2010 @@ -415,11 +415,12 @@ key1 = addr1.address[0] key2 = addr2.address[0] if key1 < key2: - return -1 + result = -1 elif key1 == key2: - return 0 + result = 0 else: - return 1 + result = 1 + return rffi.cast(rffi.INT, result) # ____________________________________________________________ From wlav at codespeak.net Thu Jul 8 20:04:22 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Thu, 8 Jul 2010 20:04:22 +0200 (CEST) Subject: [pypy-svn] r76048 - pypy/branch/reflex-support/pypy/module/cppyy/test Message-ID: <20100708180422.394C3282BFA@codespeak.net> Author: wlav Date: Thu Jul 8 20:04:20 2010 New Revision: 76048 Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Log: Some more tests. Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Thu Jul 8 20:04:20 2010 @@ -31,10 +31,10 @@ def setup_class(cls): cls.space = space env = os.environ - cls.w_example01 = cls.space.appexec([], """(): + cls.w_example01, cls.w_payload = cls.space.unpackiterable(cls.space.appexec([], """(): import cppyy cppyy.load_lib(%r) - return cppyy._type_byname('example01')""" % (shared_lib, )) + return cppyy._type_byname('example01'), cppyy._type_byname('payload')""" % (shared_lib, ))) def test_example01static_int(self): """Test passing of an int, returning of an int, and overloading on a @@ -133,3 +133,16 @@ instance.destruct() assert t.invoke("getCount") == 0 + def testPassingOfAnObjectByPointer(self): + """Test passing of an instance as an argument.""" + + t = self.example01 + + pl = self.payload.construct(3.14) + assert round(pl.invoke("getData")-3.14, 8) == 0 + + t.invoke("setPayload", pl, 41.) # now pl is a CPPInstance + assert pl.invoke("getData") == 41. + + pl.destruct() + assert t.invoke("getCount") == 0 Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Thu Jul 8 20:04:20 2010 @@ -134,6 +134,8 @@ assert pl.getData() == 41. example01_class.setPayload(pl, 43.) assert pl.getData() == 43. + e.setPayload(pl, 45.) + assert pl.getData() == 45. pl.destruct() e.destruct() From getxsick at codespeak.net Thu Jul 8 20:06:49 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 8 Jul 2010 20:06:49 +0200 (CEST) Subject: [pypy-svn] r76050 - in pypy/branch/fast-ctypes/pypy: module/jitffi rlib rlib/test Message-ID: <20100708180649.2DA1B282BFA@codespeak.net> Author: getxsick Date: Thu Jul 8 20:06:47 2010 New Revision: 76050 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: drop boxing for returned results. push results to app-level by the caller Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Thu Jul 8 20:06:47 2010 @@ -91,8 +91,7 @@ space.wrap('Unsupported type of argument: %s' % self.args_type[0])) i += 1 - res = self.rget.call() - return space.wrap(res.value) + return self.rget.call(space.wrap) def W_Get___new__(space, w_type, cpu, lib, func, args_type, res_type): try: Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Thu Jul 8 20:06:47 2010 @@ -98,20 +98,20 @@ cache.append(_Func(self.args_type, self.res_type, self.looptoken)) self.setup_stack() - def call(self): + def call(self, push_result): res = self.cpu.execute_token(self.looptoken) if self.res_type == 'i': - r = ReturnInt(self.cpu.get_latest_value_int(0)) + r = push_result(self.cpu.get_latest_value_int(0)) elif self.res_type == 'f': - r = ReturnFloat(self.cpu.get_latest_value_float(0)) + r = push_result(self.cpu.get_latest_value_float(0)) elif self.res_type == 'p': - r = ReturnPtr(self.cpu.get_latest_value_ref(0)) + r = push_result(self.cpu.get_latest_value_ref(0)) elif self.res_type == 'v': - r = ReturnNone(None) + r = None else: raise ValueError(self.res_type) - + self.setup_stack() # clean up the stack return r @@ -140,19 +140,3 @@ self.args_type = args_type self.res_type = res_type self.looptoken = looptoken - -class Return(object): - def __init__(self, value): - self.value = value - -class ReturnInt(Return): - pass - -class ReturnFloat(Return): - pass - -class ReturnPtr(Return): - pass - -class ReturnNone(Return): - pass Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Thu Jul 8 20:06:47 2010 @@ -62,6 +62,9 @@ def setup_class(cls): cls.lib_name = cls.preprare_c_example() + def push_result(self, value): # mock function + return value + def test_missing_lib(self): py.test.raises(OSError, rjitffi.CDLL, 'xxxfoo888baryyy') @@ -71,44 +74,44 @@ func = lib.get('add_integers', ['i', 'i'], 'i') func.push_int(1) func.push_int(2) - assert func.call().value == 3 + assert func.call(self.push_result) == 3 func = lib.get('add_integers', ['i', 'i'], 'i') func.push_int(-1) func.push_int(2) - assert func.call().value == 1 + assert func.call(self.push_result) == 1 func = lib.get('add_integers', ['i', 'i'], 'i') func.push_int(0) func.push_int(0) - assert func.call().value == 0 + assert func.call(self.push_result) == 0 func = lib.get('max3', ['i', 'i', 'i'], 'i') func.push_int(2) func.push_int(8) func.push_int(3) - assert func.call().value == 8 + assert func.call(self.push_result) == 8 func = lib.get('add_floats', ['f', 'f'], 'f') func.push_float(1.2) func.push_float(1.5) - assert func.call().value == 2.7 + assert func.call(self.push_result) == 2.7 def test_get_void(self): lib = rjitffi.CDLL(self.lib_name) func = lib.get('fvoid', [], 'i') - assert func.call().value == 1 + assert func.call(self.push_result) == 1 func = lib.get('return_void', ['i', 'i'], 'v') func.push_int(1) func.push_int(2) - assert func.call().value is None + assert func.call(self.push_result) is None func = lib.get('return_void', ['i', 'i']) func.push_int(1) func.push_int(2) - assert func.call().value is None + assert func.call(self.push_result) is None def test_various_type_args(self): lib = rjitffi.CDLL(self.lib_name) @@ -116,12 +119,12 @@ func = lib.get('add_intfloat', ['i', 'f'], 'i') func.push_int(1) func.push_float(2.9) - assert func.call().value == 3 + assert func.call(self.push_result) == 3 # stack is cleaned up after calling func.push_int(0) func.push_float(1.3) - assert func.call().value == 1 + assert func.call(self.push_result) == 1 def test_undefined_func(self): lib = rjitffi.CDLL(self.lib_name) From arigo at codespeak.net Thu Jul 8 20:23:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Jul 2010 20:23:59 +0200 (CEST) Subject: [pypy-svn] r76051 - pypy/branch/reflex-support/pypy/translator/c Message-ID: <20100708182359.018F6282BFA@codespeak.net> Author: arigo Date: Thu Jul 8 20:23:57 2010 New Revision: 76051 Modified: pypy/branch/reflex-support/pypy/translator/c/genc.py Log: Tentatively fix the Makefile to not call trackgcroot.py on some C files, and on all other files (.cpp, .cxx...) Modified: pypy/branch/reflex-support/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/reflex-support/pypy/translator/c/genc.py (original) +++ pypy/branch/reflex-support/pypy/translator/c/genc.py Thu Jul 8 20:23:57 2010 @@ -572,16 +572,15 @@ mk.rule(*rule) if self.config.translation.gcrootfinder == 'asmgcc': - trackgcfiles = [cfile[:-2] for cfile in mk.cfiles] - if self.translator.platform.name == 'msvc': + trackgcfiles = [cfile[:-2] for cfile in mk.cfiles + if cfile.endswith('.c')] + if 1: # XXX do that more cleanly trackgcfiles = [f for f in trackgcfiles if f.startswith(('implement', 'testing', '../module_cache/module'))] sfiles = ['%s.s' % (c,) for c in trackgcfiles] - lblsfiles = ['%s.lbl.s' % (c,) for c in trackgcfiles] gcmapfiles = ['%s.gcmap' % (c,) for c in trackgcfiles] mk.definition('ASMFILES', sfiles) - mk.definition('ASMLBLFILES', lblsfiles) mk.definition('GCMAPFILES', gcmapfiles) mk.definition('DEBUGFLAGS', '-O2 -fomit-frame-pointer -g') @@ -596,17 +595,32 @@ python = '' if self.translator.platform.name == 'msvc': - lblofiles = [] - for cfile in mk.cfiles: - f = cfile[:-2] - if f in trackgcfiles: - ofile = '%s.lbl.obj' % (f,) - else: - ofile = '%s.obj' % (f,) - - lblofiles.append(ofile) - mk.definition('ASMLBLOBJFILES', lblofiles) - mk.definition('OBJECTS', 'gcmaptable.obj $(ASMLBLOBJFILES)') + o_ext = '.obj' + lbl_ext = '.lbl.obj' + else: + o_ext = '.o' + lbl_ext = '.lbl.s' + + objectfiles = [] + nontrackgcexts = set() + for cfile in mk.cfiles: + f = cfile[:-2] + if f in trackgcfiles: + ofile = f + lbl_ext + else: + f, ext = os.path.splitext(cfile) + nontrackgcexts.add(ext) + ofile = f + o_ext + objectfiles.append(ofile) + if self.translator.platform.name == 'msvc': + objectfiles.append('gcmaptable.obj') + else: + objectfiles.append('gcmaptable.s') + mk.definition('OBJECTS', objectfiles) + nontrackgcexts = list(nontrackgcexts) + nontrackgcexts.sort() + + if self.translator.platform.name == 'msvc': # /Oi (enable intrinsics) and /Ob1 (some inlining) are mandatory # even in debug builds mk.definition('ASM_CFLAGS', '$(CFLAGS) $(CFLAGSEXTRA) /Oi /Ob1') @@ -619,14 +633,19 @@ ) mk.rule('gcmaptable.c', '$(GCMAPFILES)', 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc $(GCMAPFILES) > $@') + for ext in nontrackgcexts: + mk.rule(ext + '.obj', '', + '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) /Fo$@ /c $< $(INCLUDEDIRS)') else: - mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') mk.rule('%.lbl.s %.gcmap', '%.s', python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -m$(PYPY_MAIN_FUNCTION) -t $< > $*.gcmap') mk.rule('gcmaptable.s', '$(GCMAPFILES)', python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') + for ext in nontrackgcexts: + mk.rule('%.o', '%' + ext, + '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)') else: mk.definition('DEBUGFLAGS', '-O1 -g') From wlav at codespeak.net Thu Jul 8 20:30:37 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Thu, 8 Jul 2010 20:30:37 +0200 (CEST) Subject: [pypy-svn] r76053 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src Message-ID: <20100708183037.A284A282BFA@codespeak.net> Author: wlav Date: Thu Jul 8 20:30:36 2010 New Revision: 76053 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Log: Make C API consistent. Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Thu Jul 8 20:30:36 2010 @@ -28,84 +28,85 @@ C_METHPTRGETTER = lltype.FuncType([C_OBJECT], rffi.VOIDP) C_METHPTRGETTER_PTR = lltype.Ptr(C_METHPTRGETTER) -c_cppyy_get_typehandle = rffi.llexternal( +c_get_typehandle = rffi.llexternal( "cppyy_get_typehandle", [rffi.CCHARP], C_TYPEHANDLE, compilation_info=eci) -c_cppyy_allocate = rffi.llexternal( +c_allocate = rffi.llexternal( "cppyy_allocate", [C_TYPEHANDLE], rffi.VOIDP, compilation_info=eci) -c_cppyy_deallocate = rffi.llexternal( +c_deallocate = rffi.llexternal( "cppyy_deallocate", [C_TYPEHANDLE, C_OBJECT], rffi.VOIDP, compilation_info=eci) -c_cppyy_call_v = rffi.llexternal( +c_call_v = rffi.llexternal( "cppyy_call_v", [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], lltype.Void, compilation_info=eci) -c_cppyy_call_l = rffi.llexternal( +c_call_l = rffi.llexternal( "cppyy_call_l", [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) -c_cppyy_call_d = rffi.llexternal( +c_call_d = rffi.llexternal( "cppyy_call_d", [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.DOUBLE, compilation_info=eci) -c_cppyy_destruct = rffi.llexternal( +c_destruct = rffi.llexternal( "cppyy_destruct", [C_TYPEHANDLE, C_OBJECT], lltype.Void, compilation_info=eci) -c_cppyy_get_methptr_getter = rffi.llexternal( + +c_get_methptr_getter = rffi.llexternal( "cppyy_get_methptr_getter", [C_TYPEHANDLE, rffi.INT], C_METHPTRGETTER_PTR, compilation_info=eci) - c_num_methods = rffi.llexternal( - "num_methods", + "cppyy_num_methods", [C_TYPEHANDLE], rffi.INT, compilation_info=eci) c_method_name = rffi.llexternal( - "method_name", + "cppyy_method_name", [C_TYPEHANDLE, rffi.INT], rffi.CCHARP, compilation_info=eci) c_result_type_method = rffi.llexternal( - "result_type_method", + "cppyy_result_type_method", [C_TYPEHANDLE, rffi.INT], rffi.CCHARP, compilation_info=eci) c_num_args_method = rffi.llexternal( - "num_args_method", + "cppyy_num_args_method", [C_TYPEHANDLE, rffi.INT], rffi.INT, compilation_info=eci) c_arg_type_method = rffi.llexternal( - "arg_type_method", + "cppyy_arg_type_method", [C_TYPEHANDLE, rffi.INT, rffi.INT], rffi.CCHARP, compilation_info=eci) + c_is_constructor = rffi.llexternal( - "is_constructor", + "cppyy_is_constructor", [C_TYPEHANDLE, rffi.INT], rffi.INT, compilation_info=eci) c_is_static = rffi.llexternal( - "is_static", + "cppyy_is_static", [C_TYPEHANDLE, rffi.INT], rffi.INT, compilation_info=eci) c_is_subtype = rffi.llexternal( - "is_subtype", + "cppyy_is_subtype", [C_TYPEHANDLE, C_TYPEHANDLE], rffi.INT, compilation_info=eci) c_dynamic_type = rffi.llexternal( - "dynamic_type", + "cppyy_dynamic_type", [C_TYPEHANDLE, C_OBJECT], C_TYPEHANDLE, compilation_info=eci) -c_myfree = rffi.llexternal( - "myfree", +c_free = rffi.llexternal( + "cppyy_free", [rffi.VOIDP], lltype.Void, compilation_info=eci) def charp2str_free(charp): string = rffi.charp2str(charp) - c_myfree(charp) + c_free(charp) return string Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Thu Jul 8 20:30:36 2010 @@ -12,22 +12,22 @@ class VoidExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): - capi.c_cppyy_call_v(func.cpptype.handle, func.method_index, cppthis, num_args, args) + capi.c_call_v(func.cpptype.handle, func.method_index, cppthis, num_args, args) return space.w_None class LongExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): - result = capi.c_cppyy_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) + result = capi.c_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) return space.wrap(result) class DoubleExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): - result = capi.c_cppyy_call_d(func.cpptype.handle, func.method_index, cppthis, num_args, args) + result = capi.c_call_d(func.cpptype.handle, func.method_index, cppthis, num_args, args) return space.wrap(result) class CStringExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): - lresult = capi.c_cppyy_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) + lresult = capi.c_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) ccpresult = rffi.cast(rffi.CCHARP, lresult) result = capi.charp2str_free(ccpresult) return space.wrap(result) Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Thu Jul 8 20:30:36 2010 @@ -20,18 +20,19 @@ void cppyy_destruct(cppyy_typehandle_t handle, cppyy_object_t self); cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_typehandle_t handle, int method_index); - int num_methods(cppyy_typehandle_t handle); - char* method_name(cppyy_typehandle_t handle, int method_index); - char* result_type_method(cppyy_typehandle_t handle, int method_index); - int num_args_method(cppyy_typehandle_t handle, int method_index); - char* arg_type_method(cppyy_typehandle_t handle, int method_index, int index); - int is_constructor(cppyy_typehandle_t handle, int method_index); - int is_static(cppyy_typehandle_t handle, int method_index); - int is_subtype(cppyy_typehandle_t h1, cppyy_typehandle_t h2); + int cppyy_num_methods(cppyy_typehandle_t handle); + char* cppyy_method_name(cppyy_typehandle_t handle, int method_index); + char* cppyy_result_type_method(cppyy_typehandle_t handle, int method_index); + int cppyy_num_args_method(cppyy_typehandle_t handle, int method_index); + char* cppyy_arg_type_method(cppyy_typehandle_t handle, int method_index, int index); - cppyy_typehandle_t dynamic_type(cppyy_typehandle_t handle, cppyy_object_t self); + int cppyy_is_constructor(cppyy_typehandle_t handle, int method_index); + int cppyy_is_static(cppyy_typehandle_t handle, int method_index); + int cppyy_is_subtype(cppyy_typehandle_t h1, cppyy_typehandle_t h2); - void myfree(void* ptr); + cppyy_typehandle_t cppyy_dynamic_type(cppyy_typehandle_t handle, cppyy_object_t self); + + void cppyy_free(void* ptr); #ifdef __cplusplus } Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Thu Jul 8 20:30:36 2010 @@ -31,7 +31,7 @@ except KeyError: pass - handle = capi.c_cppyy_get_typehandle(name) + handle = capi.c_get_typehandle(name) if handle: cpptype = W_CPPType(space, name, handle) state.cpptype_cache[name] = cpptype @@ -154,11 +154,11 @@ class CPPConstructor(CPPMethod): def call(self, cppthis, args_w): assert not cppthis - newthis = capi.c_cppyy_allocate(self.cpptype.handle) + newthis = capi.c_allocate(self.cpptype.handle) try: CPPMethod.call(self, newthis, args_w) except Exception, e: - capi.c_cppyy_deallocate(self.cpptype.handle, newthis) + capi.c_deallocate(self.cpptype.handle, newthis) raise return W_CCPInstance(self.cpptype, newthis) @@ -252,7 +252,7 @@ # not constant, but as long as it corresponds to cpp_dynamic_type, the # function is still pure cppthis = cppthis_holder.cppthis - methgetter = capi.c_cppyy_get_methptr_getter(self.handle, method_index) + methgetter = capi.c_get_methptr_getter(self.handle, method_index) if not methgetter: raise NotImplementedError return methgetter(cppthis) @@ -294,7 +294,7 @@ return overload.call(self.rawobject, args_w) def destruct(self): - capi.c_cppyy_destruct(self.cppclass.handle, self.rawobject) + capi.c_destruct(self.cppclass.handle, self.rawobject) self.rawobject = NULL_VOIDP W_CCPInstance.typedef = TypeDef( Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Thu Jul 8 20:30:36 2010 @@ -17,6 +17,7 @@ Reflex::Type((Reflex::TypeName*)handle).Deallocate(instance); } + void cppyy_call_v(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]) { std::vector arguments(args, args+numargs); @@ -65,8 +66,8 @@ t.Destruct(self, true); } -static cppyy_methptrgetter_t get_methptr_getter(Reflex::Member m) -{ + +static cppyy_methptrgetter_t get_methptr_getter(Reflex::Member m) { Reflex::PropertyList plist = m.Properties(); if (plist.HasProperty("MethPtrGetter")) { Reflex::Any& value = plist.PropertyValue("MethPtrGetter"); @@ -84,7 +85,7 @@ } -int num_methods(cppyy_typehandle_t handle) { +int cppyy_num_methods(cppyy_typehandle_t handle) { Reflex::Type t((Reflex::TypeName*)handle); for (int i = 0; i < (int)t.FunctionMemberSize(); i++) { Reflex::Member m = t.FunctionMemberAt(i); @@ -99,7 +100,7 @@ return t.FunctionMemberSize(); } -char* method_name(cppyy_typehandle_t handle, int method_index) { +char* cppyy_method_name(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); std::string name = m.Name(); @@ -108,7 +109,7 @@ return name_char; } -char* result_type_method(cppyy_typehandle_t handle, int method_index) { +char* cppyy_result_type_method(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); Reflex::Type rt = m.TypeOf().ReturnType(); @@ -118,14 +119,13 @@ return name_char; } -int num_args_method(cppyy_typehandle_t handle, int method_index) { +int cppyy_num_args_method(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); return m.FunctionParameterSize(); } -char* arg_type_method(cppyy_typehandle_t handle, int method_index, int arg_index) { - +char* cppyy_arg_type_method(cppyy_typehandle_t handle, int method_index, int arg_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); Reflex::Type at = m.TypeOf().FunctionParameterAt(arg_index); @@ -135,19 +135,20 @@ return name_char; } -int is_constructor(cppyy_typehandle_t handle, int method_index) { + +int cppyy_is_constructor(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); return m.IsConstructor(); } -int is_static(cppyy_typehandle_t handle, int method_index) { +int cppyy_is_static(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); return m.IsStatic(); } -int is_subtype(cppyy_typehandle_t h1, cppyy_typehandle_t h2) { +int cppyy_is_subtype(cppyy_typehandle_t h1, cppyy_typehandle_t h2) { if (h1 == h2) return 1; Reflex::Type t1((Reflex::TypeName*)h1); @@ -155,12 +156,13 @@ return (int)t2.HasBase(t1); } -cppyy_typehandle_t dynamic_type(cppyy_typehandle_t handle, cppyy_object_t self) { +cppyy_typehandle_t cppyy_dynamic_type(cppyy_typehandle_t handle, cppyy_object_t self) { Reflex::Type t((Reflex::TypeName*)handle); const Reflex::Object* obj = (const Reflex::Object*)self; return t.DynamicType((*obj)).Id(); } -void myfree(void* ptr) { + +void cppyy_free(void* ptr) { free(ptr); } From getxsick at codespeak.net Thu Jul 8 21:00:33 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 8 Jul 2010 21:00:33 +0200 (CEST) Subject: [pypy-svn] r76054 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100708190033.D9F17282BFA@codespeak.net> Author: getxsick Date: Thu Jul 8 21:00:31 2010 New Revision: 76054 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: use dict-cache instead list-cache to be more effective Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Thu Jul 8 21:00:31 2010 @@ -6,7 +6,7 @@ from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.typesystem import deref -cache = [] # XXX global! +cache = {} # XXX global! class CDLL(object): def __init__(self, name, load=True): @@ -37,7 +37,6 @@ self.args_type = args_type self.res_type = res_type self.cpu = cpu - self.looptoken = None lib = lib.handler bargs = [] @@ -47,14 +46,10 @@ raise ValueError("Cannot find symbol %s", func) bargs.append(BoxInt()) - # check if it's not already compiled - for func in cache: - if self.args_type == func.args_type and \ - self.res_type == func.res_type: - self.looptoken = func.looptoken - break - - if self.looptoken is None: + # grab from the cache if possible + try: + self.looptoken = cache[self.res_type][tuple(self.args_type)] + except KeyError: args = [] for arg in self.args_type: if arg == 'i': @@ -95,7 +90,7 @@ self.cpu.compile_loop(bargs, oplist, self.looptoken) # add to the cache - cache.append(_Func(self.args_type, self.res_type, self.looptoken)) + cache[self.res_type] = { tuple(self.args_type) : self.looptoken } self.setup_stack() def call(self, push_result): @@ -134,9 +129,3 @@ def push_ref(self, value): self.cpu.set_future_value_ref(self.esp, value) self.esp += 1 - -class _Func(object): - def __init__(self, args_type, res_type, looptoken): - self.args_type = args_type - self.res_type = res_type - self.looptoken = looptoken From hakanardo at codespeak.net Fri Jul 9 07:06:35 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Fri, 9 Jul 2010 07:06:35 +0200 (CEST) Subject: [pypy-svn] r76055 - in pypy/branch/interplevel-array/pypy: jit/tl module/array module/array/test Message-ID: <20100709050635.5E34B282C00@codespeak.net> Author: hakanardo Date: Fri Jul 9 07:06:32 2010 New Revision: 76055 Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/inttst.py pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py Log: jit-able getitem/setitem Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py Fri Jul 9 07:06:32 2010 @@ -39,9 +39,11 @@ from array import array def f(img): + i=0 sa=0 - for i in xrange(4): + while i<4: sa+=img[i] + i+=1 return sa img=array('i',(1,2,3,4)) Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Fri Jul 9 07:06:32 2010 @@ -132,6 +132,17 @@ descr_len.unwrap_spec = ['self'] + def descr_getslice(self, start, stop, step): + size = (stop - start) / step + if (stop - start) % step > 0: size += 1 + w_a=mytype.w_class(self.space) + w_a.setlen(size) + j=0 + for i in range(start, stop, step): + w_a.buffer[j]=self.buffer[i] + j+=1 + return w_a + def descr_getitem(self, w_idx): space=self.space start, stop, step = space.decode_index(w_idx, self.len) @@ -144,15 +155,7 @@ item = float(item) return self.space.wrap(item) else: - size = (stop - start) / step - if (stop - start) % step > 0: size += 1 - w_a=mytype.w_class(self.space) - w_a.setlen(size) - j=0 - for i in range(start, stop, step): - w_a.buffer[j]=self.buffer[i] - j+=1 - return w_a + return self.descr_getslice(start, stop, step) descr_getitem.unwrap_spec = ['self', W_Root] @@ -199,28 +202,30 @@ self.descr_append(w_item) descr_extend.unwrap_spec = ['self', W_Root] - + def descr_setslice(self, start, stop, step, w_item): + if isinstance(w_item, W_Array): # Implies mytype.typecode == w_item.typecode + size = (stop - start) / step + if (stop - start) % step > 0: size += 1 + if w_item.len != size: # FIXME: Support for step=1 + msg = ('attempt to assign array of size %d to ' + + 'slice of size %d') % (w_item.len, size) + raise OperationError(self.space.w_ValueError, + self.space.wrap(msg)) + j=0 + for i in range(start, stop, step): + self.buffer[i]=w_item.buffer[j] + j+=1 + return + msg='can only assign array to array slice' + raise OperationError(self.space.w_TypeError, self.space.wrap(msg)) + def descr_setitem(self, w_idx, w_item): start, stop, step = self.space.decode_index(w_idx, self.len) if step==0: item = self.item_w(w_item) self.buffer[start] = item else: - if isinstance(w_item, W_Array): # Implies mytype.typecode == w_item.typecode - size = (stop - start) / step - if (stop - start) % step > 0: size += 1 - if w_item.len != size: # FIXME: Support for step=1 - msg = ('attempt to assign array of size %d to ' + - 'slice of size %d') % (w_item.len, size) - raise OperationError(self.space.w_ValueError, - self.space.wrap(msg)) - j=0 - for i in range(start, stop, step): - self.buffer[i]=w_item.buffer[j] - j+=1 - return - msg='can only assign array to array slice' - raise OperationError(self.space.w_TypeError, self.space.wrap(msg)) + self.descr_setslice(start, stop, step, w_item) descr_setitem.unwrap_spec = ['self', W_Root, W_Root] def descr_fromstring(self, s): Modified: pypy/branch/interplevel-array/pypy/module/array/test/inttst.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/inttst.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/inttst.py Fri Jul 9 07:06:32 2010 @@ -1,7 +1,7 @@ #!/usr/bin/python from time import time -from array import array #, simple_array +from array import array, simple_array def f(img, intimg): l=0 Modified: pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py Fri Jul 9 07:06:32 2010 @@ -10,7 +10,7 @@ i+=1 return l -if False: +if True: img=array('d', '\x00'*640*480*8) else: img=simple_array(640*480) @@ -19,5 +19,5 @@ #print f(img) # C pypy-simple pypy cpython -# sumtst: 0m0.630s 0m0.659s 0m9.185s 0m33.447s -# intimg: 0m0.646s 0m1.404s 0m26.850s 1m0.279s +# sumtst: 0m0.630s 0m0.659s 0m0.762s 0m33.447s +# intimg: 0m0.646s 0m1.078s 0m1.357s 1m0.279s From wlav at codespeak.net Fri Jul 9 09:58:42 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Fri, 9 Jul 2010 09:58:42 +0200 (CEST) Subject: [pypy-svn] r76056 - pypy/extradoc/sprintinfo/cern2010 Message-ID: <20100709075842.64C98282C00@codespeak.net> Author: wlav Date: Fri Jul 9 09:58:40 2010 New Revision: 76056 Modified: pypy/extradoc/sprintinfo/cern2010/planning.txt Log: A few more detailed TODOs. Modified: pypy/extradoc/sprintinfo/cern2010/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/cern2010/planning.txt (original) +++ pypy/extradoc/sprintinfo/cern2010/planning.txt Fri Jul 9 09:58:40 2010 @@ -11,3 +11,6 @@ Detailed TODO list: - Deal with opened cpplibs: ownership and possible offloading + - Interacting of cache from a get_cppclass-like UI and gbl getattr + - Need a "counter" mixin baseclass for memory tracking + - Better error reporting when overloads fail From hakanardo at codespeak.net Fri Jul 9 10:14:33 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Fri, 9 Jul 2010 10:14:33 +0200 (CEST) Subject: [pypy-svn] r76057 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100709081433.66293282C00@codespeak.net> Author: hakanardo Date: Fri Jul 9 10:14:31 2010 New Revision: 76057 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: Support for architectures where unsisgned int is unsigned long Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Fri Jul 9 10:14:31 2010 @@ -43,6 +43,12 @@ self.canoverflow = canoverflow self.w_class = None + assert self.bytes <= rffi.sizeof(rffi.ULONG) + if self.bytes == rffi.sizeof(rffi.ULONG) and not signed and self.unwrap == 'int_w': + # Treat this type as a ULONG + self.unwrap = 'bigint_w' + self.canoverflow = False + def _freeze_(self): # hint for the annotator: track individual constant instances Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Fri Jul 9 10:14:31 2010 @@ -92,6 +92,17 @@ except OverflowError: pass + for tc in 'BHIL': + a = self.array(tc) + vals=[0, 2 ** a.itemsize - 1] + a.fromlist(vals) + assert a.tolist() == vals + + a = self.array(tc.lower()) + vals=[ -1 * (2 ** a.itemsize) / 2, (2 ** a.itemsize) / 2 - 1] + a.fromlist(vals) + assert a.tolist() == vals + def test_float(self): values = [0, 1, 2.5, -4.25] for tc in 'fd': From antocuni at codespeak.net Fri Jul 9 10:44:17 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 9 Jul 2010 10:44:17 +0200 (CEST) Subject: [pypy-svn] r76058 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src Message-ID: <20100709084417.0954B282C00@codespeak.net> Author: antocuni Date: Fri Jul 9 10:44:15 2010 New Revision: 76058 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Log: (arigo, antocuni): revert r76042 as it makes things slower. Moreover, turn get_methptr_getter into a pure function, to help the jit Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Fri Jul 9 10:44:15 2010 @@ -96,10 +96,6 @@ "cppyy_is_subtype", [C_TYPEHANDLE, C_TYPEHANDLE], rffi.INT, compilation_info=eci) -c_dynamic_type = rffi.llexternal( - "cppyy_dynamic_type", - [C_TYPEHANDLE, C_OBJECT], C_TYPEHANDLE, - compilation_info=eci) c_free = rffi.llexternal( "cppyy_free", Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Fri Jul 9 10:44:15 2010 @@ -30,8 +30,6 @@ int cppyy_is_static(cppyy_typehandle_t handle, int method_index); int cppyy_is_subtype(cppyy_typehandle_t h1, cppyy_typehandle_t h2); - cppyy_typehandle_t cppyy_dynamic_type(cppyy_typehandle_t handle, cppyy_object_t self); - void cppyy_free(void* ptr); #ifdef __cplusplus Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Fri Jul 9 10:44:15 2010 @@ -51,6 +51,11 @@ 'CPPLibrary', ) + at jit.purefunction +def get_methptr_getter(handle, method_index): + return capi.c_get_methptr_getter(handle, method_index) + + class CPPMethod(object): """ A concrete function after overloading has been resolved """ _immutable_ = True @@ -91,11 +96,11 @@ if len(args_w) != 1: raise OperationError(space.w_TypeError, space.wrap("wrong number of args")) arg = space.c_int_w(args_w[0]) - - cpptype = jit.hint(self.cpptype, promote=True) - cpp_dynamic_type = capi.c_dynamic_type(cpptype.handle, cppthis) - cppthis_holder.cppthis = cppthis - funcptr = cpptype.get_methptr(cpp_dynamic_type, self.method_index) + methgetter = get_methptr_getter(self.cpptype.handle, + self.method_index) + if not methgetter: + raise NotImplementedError + funcptr = methgetter(cppthis) funcptr = rffi.cast(self.INT_2_INT_FNPTR, funcptr) result = funcptr(cppthis, arg) return space.wrap(rffi.cast(lltype.Signed, result)) @@ -197,10 +202,6 @@ is_static = interp2app(W_CPPOverload.is_static, unwrap_spec=['self']), ) -# hack hack hack (see get_methptr) -class _CppthisHolder(object): - pass -cppthis_holder = _CppthisHolder() class W_CPPType(Wrappable): _immutable_fields_ = ["name","handle"] @@ -246,18 +247,6 @@ def get_overload(self, name): return self.function_members[name] - @jit.purefunction - def get_methptr(self, cpp_dynamic_type, method_index): - # hack hack hack: we can't really pass cppthis as an argument as it's - # not constant, but as long as it corresponds to cpp_dynamic_type, the - # function is still pure - cppthis = cppthis_holder.cppthis - methgetter = capi.c_get_methptr_getter(self.handle, method_index) - if not methgetter: - raise NotImplementedError - return methgetter(cppthis) - - def invoke(self, name, args_w): overload = self.get_overload(name) return overload.call(NULL_VOIDP, args_w) Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Fri Jul 9 10:44:15 2010 @@ -156,13 +156,6 @@ return (int)t2.HasBase(t1); } -cppyy_typehandle_t cppyy_dynamic_type(cppyy_typehandle_t handle, cppyy_object_t self) { - Reflex::Type t((Reflex::TypeName*)handle); - const Reflex::Object* obj = (const Reflex::Object*)self; - return t.DynamicType((*obj)).Id(); -} - - void cppyy_free(void* ptr) { free(ptr); } From wlav at codespeak.net Fri Jul 9 10:45:57 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Fri, 9 Jul 2010 10:45:57 +0200 (CEST) Subject: [pypy-svn] r76059 - pypy/branch/reflex-support/pypy/module/cppyy/test Message-ID: <20100709084557.CF369282C00@codespeak.net> Author: wlav Date: Fri Jul 9 10:45:56 2010 New Revision: 76059 Added: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.h (contents, props changed) Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx Log: (cfbolz, wlav) Cleanup example01 C++ code in test. Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile Fri Jul 9 10:45:56 2010 @@ -16,6 +16,6 @@ cppflags2=-Wno-pmf-conversions endif -example01Dict.so: example01.cxx - $(genreflex) example01.cxx $(genreflexflags) - g++ -o $@ example01_rflx.cpp -shared -lReflex $(cppflags) $(cppflags2) +example01Dict.so: example01.cxx example01.h + $(genreflex) example01.h $(genreflexflags) + g++ -o $@ example01_rflx.cpp example01.cxx -shared -lReflex $(cppflags) $(cppflags2) Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx Fri Jul 9 10:45:56 2010 @@ -4,98 +4,86 @@ #include #include +#include "example01.h" -class payload { -public: - payload(double d) : m_data(d) {} - -// cctor needed as long as we have no converter for const& objects - payload( const payload& p ) : m_data(p.m_data) {} - - double getData() { return m_data; } - void setData(double d) { m_data = d; } - -private: - double m_data; -}; - - -class example01 { -public: - static int count; - int somedata; - - example01() : somedata(-99) { - count++; - } - example01(int a) : somedata(a) { - count++; - std::cout << "constructor called" << std::endl; - } - example01(const example01& e) : somedata(e.somedata) { - count++; - std::cout << "copy constructor called" << std::endl; - } - example01& operator=(const example01& e) { - if (this != &e) { - somedata = e.somedata; - } - return *this; - } - ~example01() { - count--; - } +//=========================================================================== +payload::payload(double d) : m_data(d) {} +payload::payload(const payload& p) : m_data(p.m_data) {} + +double payload::getData() { return m_data; } +void payload::setData(double d) { m_data = d; } + + +//=========================================================================== +example01::example01() : somedata(-99) { + count++; +} +example01::example01(int a) : somedata(a) { + count++; + std::cout << "constructor called" << std::endl; +} +example01::example01(const example01& e) : somedata(e.somedata) { + count++; + std::cout << "copy constructor called" << std::endl; +} +example01& example01::operator=(const example01& e) { + if (this != &e) { + somedata = e.somedata; + } + return *this; +} +example01::~example01() { + count--; +} // class methods - static int staticAddOneToInt(int a) { - return a + 1; - } - static int staticAddOneToInt(int a, int b) { - return a + b + 1; - } - static double staticAddToDouble(double a) { - return a + 0.01; - } - static int staticAtoi(const char* str) { - return ::atoi(str); - } - static char* staticStrcpy(const char* strin) { - char* strout = (char*)malloc(::strlen(strin + 1)); - ::strcpy(strout, strin); - return strout; - } - static void setPayload( payload* p, double d ) { - p->setData(d); - } - - static int getCount() { - std::cout << "getcount called" << std::endl; - return count; - } +int example01::staticAddOneToInt(int a) { + return a + 1; +} +int example01::staticAddOneToInt(int a, int b) { + return a + b + 1; +} +double example01::staticAddToDouble(double a) { + return a + 0.01; +} +int example01::staticAtoi(const char* str) { + return ::atoi(str); +} +char* example01::staticStrcpy(const char* strin) { + char* strout = (char*)malloc(::strlen(strin + 1)); + ::strcpy(strout, strin); + return strout; +} +void example01::setPayload( payload* p, double d ) { + p->setData(d); +} + +int example01::getCount() { + std::cout << "getcount called" << std::endl; + return count; +} // instance methods - int addDataToInt(int a) { - return somedata + a; - } - - double addDataToDouble(double a) { - return somedata + a; - } - - int addDataToAtoi(const char* str) { - return ::atoi(str) + somedata; - } - - char* addToStringValue(const char* str) { - int out = ::atoi(str) + somedata; - std::ostringstream ss; - ss << out << std::ends; - std::string result = ss.str(); - char* cresult = (char*)malloc(result.size()+1); - ::strcpy(cresult, result.c_str()); - return cresult; - } - -}; +int example01::addDataToInt(int a) { + return somedata + a; +} + +double example01::addDataToDouble(double a) { + return somedata + a; +} + +int example01::addDataToAtoi(const char* str) { + return ::atoi(str) + somedata; +} + +char* example01::addToStringValue(const char* str) { + int out = ::atoi(str) + somedata; + std::ostringstream ss; + ss << out << std::ends; + std::string result = ss.str(); + char* cresult = (char*)malloc(result.size()+1); + ::strcpy(cresult, result.c_str()); + return cresult; +} int example01::count = 0; Added: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.h ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.h Fri Jul 9 10:45:56 2010 @@ -0,0 +1,39 @@ +class payload { +public: + payload(double d); + payload(const payload& p); + + double getData(); + void setData(double d); + +private: + double m_data; +}; + + +class example01 { +public: + static int count; + int somedata; + + example01(); + example01(int a); + example01(const example01& e); + example01& operator=(const example01& e); + ~example01(); + +// class methods + static int staticAddOneToInt(int a); + static int staticAddOneToInt(int a, int b); + static double staticAddToDouble(double a); + static int staticAtoi(const char* str); + static char* staticStrcpy(const char* strin); + static void setPayload( payload* p, double d ); + static int getCount(); + +// instance methods + int addDataToInt(int a); + double addDataToDouble(double a); + int addDataToAtoi(const char* str); + char* addToStringValue(const char* str); +}; From arigo at codespeak.net Fri Jul 9 10:49:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Jul 2010 10:49:10 +0200 (CEST) Subject: [pypy-svn] r76060 - pypy/branch/kill-caninline/pypy/jit/metainterp Message-ID: <20100709084910.D8A85282C00@codespeak.net> Author: arigo Date: Fri Jul 9 10:49:09 2010 New Revision: 76060 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py Log: Just kill can_inline. Incomplete. Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py Fri Jul 9 10:49:09 2010 @@ -399,14 +399,6 @@ maybe_enter_jit._always_inline_ = True jd._maybe_enter_jit_fn = maybe_enter_jit - can_inline = state.can_inline_greenargs - num_green_args = jd.num_green_args - def maybe_enter_from_start(*args): - if can_inline is not None and not can_inline(*args[:num_green_args]): - maybe_compile_and_run(*args) - maybe_enter_from_start._always_inline_ = True - jd._maybe_enter_from_start_fn = maybe_enter_from_start - def make_driverhook_graphs(self): from pypy.rlib.jit import BaseJitCell bk = self.rtyper.annotator.bookkeeper @@ -423,8 +415,6 @@ s_BaseJitCell_not_None) jd._get_jitcell_at_ptr = self._make_hook_graph(jd, annhelper, jd.jitdriver.get_jitcell_at, s_BaseJitCell_or_None) - jd._can_inline_ptr = self._make_hook_graph(jd, - annhelper, jd.jitdriver.can_inline, annmodel.s_Bool) jd._get_printable_location_ptr = self._make_hook_graph(jd, annhelper, jd.jitdriver.get_printable_location, s_Str) jd._confirm_enter_jit_ptr = self._make_hook_graph(jd, @@ -582,7 +572,6 @@ def ll_portal_runner(*args): while 1: try: - jd._maybe_enter_from_start_fn(*args) return support.maybe_on_top_of_llinterp(rtyper, portal_ptr)(*args) except self.ContinueRunningNormally, e: Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py Fri Jul 9 10:49:09 2010 @@ -316,9 +316,9 @@ return self.jit_getter # if self.jitdriver_sd._get_jitcell_at_ptr is None: - jit_getter = self._make_jitcell_getter_default() + jit_getter_maybe, jit_getter = self._make_jitcell_getter_default() else: - jit_getter = self._make_jitcell_getter_custom() + jit_getter_maybe, jit_getter = self._make_jitcell_getter_custom() # unwrap_greenkey = self.make_unwrap_greenkey() # @@ -326,6 +326,7 @@ greenargs = unwrap_greenkey(greenkey) return jit_getter(*greenargs) self.jit_cell_at_key = jit_cell_at_key + self.jit_getter_maybe = jit_getter_maybe self.jit_getter = jit_getter # return jit_getter @@ -356,13 +357,15 @@ jitcell_dict = r_dict(comparekey, hashkey) # def get_jitcell(*greenargs): + return jitcell_dict.get(greenargs, None) + def get_or_build_jitcell(*greenargs): try: cell = jitcell_dict[greenargs] except KeyError: cell = JitCell() jitcell_dict[greenargs] = cell return cell - return get_jitcell + return get_jitcell, get_or_build_jitcell def _make_jitcell_getter_custom(self): "NOT_RPYTHON" @@ -388,6 +391,9 @@ else: cell = None # + return cell + def get_or_build_jitcell(*greenargs): + cell = get_jitcell_or_none(greenargs) if cell is None: cell = JitCell() # @@ -408,7 +414,7 @@ set_jitcell_at_ptr) fn(cellref, *greenargs) return cell - return get_jitcell + return get_jitcell, get_or_build_jitcell # ---------- @@ -468,29 +474,17 @@ if hasattr(self, 'get_location_str'): return # - can_inline_ptr = self.jitdriver_sd._can_inline_ptr unwrap_greenkey = self.make_unwrap_greenkey() jit_getter = self.make_jitcell_getter() - if can_inline_ptr is None: - def can_inline_callable(*greenargs): - # XXX shouldn't it be False by default? - return True - else: - rtyper = self.warmrunnerdesc.rtyper - # - def can_inline_callable(*greenargs): - fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) - return fn(*greenargs) - def can_inline(*greenargs): - cell = jit_getter(*greenargs) - if cell.dont_trace_here: - return False - return can_inline_callable(*greenargs) - self.can_inline_greenargs = can_inline - def can_inline_greenkey(greenkey): + jit_getter_maybe = self.jit_getter_maybe + + def can_inline_callable(greenkey): greenargs = unwrap_greenkey(greenkey) - return can_inline(*greenargs) - self.can_inline_callable = can_inline_greenkey + cell = jit_getter_maybe(*greenargs) + if cell is not None and cell.dont_trace_here: + return False + return True + self.can_inline_callable = can_inline_callable def get_assembler_token(greenkey): greenargs = unwrap_greenkey(greenkey) From antocuni at codespeak.net Fri Jul 9 11:01:35 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 9 Jul 2010 11:01:35 +0200 (CEST) Subject: [pypy-svn] r76061 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100709090135.9CD41282C00@codespeak.net> Author: antocuni Date: Fri Jul 9 11:01:34 2010 New Revision: 76061 Modified: pypy/branch/reflex-support/pypy/module/cppyy/helper.py Log: translation fix Modified: pypy/branch/reflex-support/pypy/module/cppyy/helper.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/helper.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/helper.py Fri Jul 9 11:01:34 2010 @@ -16,4 +16,4 @@ def clean_type(name): assert name.find("const") == -1 i = _find_qualifier_index(name) - return name[:i].strip() + return name[:i].strip(' ') From arigo at codespeak.net Fri Jul 9 11:10:29 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Jul 2010 11:10:29 +0200 (CEST) Subject: [pypy-svn] r76062 - pypy/branch/reflex-support/pypy/module/cppyy/test Message-ID: <20100709091029.24CC1282C00@codespeak.net> Author: arigo Date: Fri Jul 9 11:10:27 2010 New Revision: 76062 Added: pypy/branch/reflex-support/pypy/module/cppyy/test/bench1.py Log: A benchmark. Added: pypy/branch/reflex-support/pypy/module/cppyy/test/bench1.py ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/bench1.py Fri Jul 9 11:10:27 2010 @@ -0,0 +1,26 @@ + +import time +import cppyy +lib = cppyy.load_lib("./example01Dict.so") +cls = cppyy._type_byname("example01") +inst = cls.construct(0) + +def g(): + res = 0 + for i in range(10000000): + i + +def f(): + res = 0 + for i in range(10000000): + #inst.invoke("addDataToDouble", float(i)) + inst.invoke("addDataToInt", i) + +g(); f(); +t1 = time.time() +g() +t2 = time.time() +f() +t3 = time.time() +print t3 - t2, t2 - t1 +print (t3 - t2) - (t2 - t1) From wlav at codespeak.net Fri Jul 9 11:38:49 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Fri, 9 Jul 2010 11:38:49 +0200 (CEST) Subject: [pypy-svn] r76064 - pypy/branch/reflex-support/pypy/module/cppyy/test Message-ID: <20100709093849.328C7282C00@codespeak.net> Author: wlav Date: Fri Jul 9 11:38:47 2010 New Revision: 76064 Added: pypy/branch/reflex-support/pypy/module/cppyy/test/bench1.cxx (contents, props changed) Log: Initial C++ benchmark. Added: pypy/branch/reflex-support/pypy/module/cppyy/test/bench1.cxx ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/bench1.cxx Fri Jul 9 11:38:47 2010 @@ -0,0 +1,45 @@ +#include +#include +#include +#include + +#include "example01.h" + +static double gTicks = 0; + +double get_cputime() { + struct tms cpt; + times(&cpt); + return (double)(cpt.tms_utime+cpt.tms_stime) / gTicks; +} + +int g() { + int i = 0; + for ( ; i < 10000000; ++i) + ; + return i; +} + +int f() { + int i = 0; + example01 e; + for ( ; i < 10000000; ++i) + e.addDataToInt(i); + return i; +} + + +int main() { + gTicks = (double)sysconf(_SC_CLK_TCK); + double t1 = get_cputime(); + g(); + double t2 = get_cputime(); + f(); + double t3 = get_cputime(); + + std::cout << std::setprecision( 8 ); + std::cout << (t3 - t2) << " " << (t2 - t1) << std::endl; + std::cout << (t3-t2) - (t2 - t1) << std::endl; + + return 0; +} From arigo at codespeak.net Fri Jul 9 12:35:03 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Jul 2010 12:35:03 +0200 (CEST) Subject: [pypy-svn] r76065 - pypy/branch/kill-caninline/pypy/jit/backend/llgraph Message-ID: <20100709103503.C8C7E282C00@codespeak.net> Author: arigo Date: Fri Jul 9 12:35:02 2010 New Revision: 76065 Modified: pypy/branch/kill-caninline/pypy/jit/backend/llgraph/llimpl.py Log: Fix the llgraph backend: there is a case in which self.loop.operations becomes different from operations. Fixed by killing the local vars and only using the self.* version. Modified: pypy/branch/kill-caninline/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/llgraph/llimpl.py Fri Jul 9 12:35:02 2010 @@ -421,11 +421,9 @@ global _last_exception assert _last_exception is None, "exception left behind" verbose = True - operations = self.loop.operations - opindex = 0 + self.opindex = 0 while True: - self.opindex = opindex - op = operations[opindex] + op = self.loop.operations[self.opindex] args = [self.getenv(v) for v in op.args] if not op.is_final(): try: @@ -439,8 +437,8 @@ args = [self.getenv(v) for v in op.fail_args if v] assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) - operations = op.jump_target.operations - opindex = 0 + self.loop = op.jump_target + self.opindex = 0 continue else: self._populate_fail_args(op) @@ -465,14 +463,13 @@ raise Exception("op.result.concretetype is %r" % (RESTYPE,)) self.env[op.result] = x - opindex += 1 + self.opindex += 1 continue if op.opnum == rop.JUMP: assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) self.loop = op.jump_target - operations = self.loop.operations - opindex = 0 + self.opindex = 0 _stats.exec_jumps += 1 elif op.opnum == rop.FINISH: if self.verbose: From arigo at codespeak.net Fri Jul 9 12:38:06 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Jul 2010 12:38:06 +0200 (CEST) Subject: [pypy-svn] r76066 - in pypy/branch/kill-caninline/pypy: jit/codewriter jit/metainterp jit/metainterp/test rlib Message-ID: <20100709103806.A47B9282C00@codespeak.net> Author: arigo Date: Fri Jul 9 12:38:04 2010 New Revision: 76066 Modified: pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py pypy/branch/kill-caninline/pypy/jit/metainterp/history.py pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_basic.py pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_jitdriver.py pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py pypy/branch/kill-caninline/pypy/rlib/jit.py Log: Progress. Modified: pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py Fri Jul 9 12:38:04 2010 @@ -804,7 +804,9 @@ self.make_three_lists(op.args[2:2+num_green_args]) + self.make_three_lists(op.args[2+num_green_args:])) op1 = SpaceOperation('jit_merge_point', args, None) - return ops + [op1] + op2 = SpaceOperation('-live-', [], None) + # ^^^ we need a -live- for the case of do_recursive_call() + return ops + [op1, op2] def handle_jit_marker__can_enter_jit(self, op, jitdriver): jd = self.callcontrol.jitdriver_sd_from_jitdriver(jitdriver) Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/history.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/history.py Fri Jul 9 12:38:04 2010 @@ -919,11 +919,12 @@ "found %d %r, expected %d" % (found, insn, expected_count)) return insns - def check_loops(self, expected=None, **check): + def check_loops(self, expected=None, everywhere=False, **check): insns = {} for loop in self.loops: - if getattr(loop, '_ignore_during_counting', False): - continue + if not everywhere: + if getattr(loop, '_ignore_during_counting', False): + continue insns = loop.summary(adding_insns=insns) if expected is not None: insns.pop('debug_merge_point', None) Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py Fri Jul 9 12:38:04 2010 @@ -149,8 +149,6 @@ assert oldbox not in registers[count:] def make_result_of_lastop(self, resultbox): - if resultbox is None: - return target_index = ord(self.bytecode[self.pc-1]) if resultbox.type == history.INT: self.registers_i[target_index] = resultbox @@ -685,11 +683,11 @@ def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes): targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex] allboxes = greenboxes + redboxes - portal_code = targetjitdriver_sd.mainjitcode warmrunnerstate = targetjitdriver_sd.warmstate token = None if warmrunnerstate.inlining: if warmrunnerstate.can_inline_callable(greenboxes): + portal_code = targetjitdriver_sd.mainjitcode return self.metainterp.perform_call(portal_code, allboxes, greenkey=greenboxes) token = warmrunnerstate.get_assembler_token(greenboxes) @@ -697,6 +695,10 @@ # that assembler that we call is still correct self.verify_green_args(targetjitdriver_sd, greenboxes) # + return self.do_recursive_call(targetjitdriver_sd, allboxes, token) + + def do_recursive_call(self, targetjitdriver_sd, allboxes, token=None): + portal_code = targetjitdriver_sd.mainjitcode k = targetjitdriver_sd.portal_runner_adr funcbox = ConstInt(heaptracker.adr2int(k)) return self.do_residual_call(funcbox, portal_code.calldescr, @@ -787,12 +789,7 @@ @arguments("int") def opimpl_can_enter_jit(self, jdindex): - if self.metainterp.in_recursion: - from pypy.jit.metainterp.warmspot import CannotInlineCanEnterJit - raise CannotInlineCanEnterJit() - assert jdindex == self.metainterp.jitdriver_sd.index, ( - "found a can_enter_jit that does not match the current jitdriver") - self.metainterp.seen_can_enter_jit = True + self.metainterp.seen_can_enter_jit_for_jdindex = jdindex def verify_green_args(self, jitdriver_sd, varargs): num_green_args = jitdriver_sd.num_green_args @@ -806,13 +803,15 @@ self.verify_green_args(jitdriver_sd, greenboxes) # xxx we may disable the following line in some context later self.debug_merge_point(jitdriver_sd, greenboxes) - if self.metainterp.seen_can_enter_jit: - self.metainterp.seen_can_enter_jit = False - # Assert that it's impossible to arrive here with in_recursion - # set to a non-zero value: seen_can_enter_jit can only be set - # to True by opimpl_can_enter_jit, which should be executed - # just before opimpl_jit_merge_point (no recursion inbetween). - assert not self.metainterp.in_recursion + if self.metainterp.seen_can_enter_jit_for_jdindex < 0: + return + # + assert self.metainterp.seen_can_enter_jit_for_jdindex == jdindex, ( + "found a can_enter_jit for a JitDriver that does not match " + "the following jit_merge_point's") + self.metainterp.seen_can_enter_jit_for_jdindex = -1 + # + if not self.metainterp.in_recursion: assert jitdriver_sd is self.metainterp.jitdriver_sd # Set self.pc to point to jit_merge_point instead of just after: # if reached_can_enter_jit() raises SwitchToBlackhole, then the @@ -822,6 +821,15 @@ self.pc = orgpc self.metainterp.reached_can_enter_jit(greenboxes, redboxes) self.pc = saved_pc + else: + warmrunnerstate = jitdriver_sd.warmstate + token = warmrunnerstate.get_assembler_token(greenboxes) + resbox = self.do_recursive_call(jitdriver_sd, + greenboxes + redboxes, + token) + # in case of exception, do_recursive_call() stops by raising + # the ChangeFrame exception already. + self.metainterp.finishframe(resbox) def debug_merge_point(self, jitdriver_sd, greenkey): # debugging: produce a DEBUG_MERGE_POINT operation @@ -1018,9 +1026,10 @@ self.metainterp.clear_exception() resbox = self.metainterp.execute_and_record_varargs(opnum, argboxes, descr=descr) - self.make_result_of_lastop(resbox) - # ^^^ this is done before handle_possible_exception() because we need - # the box to show up in get_list_of_active_boxes() + if resbox is not None: + self.make_result_of_lastop(resbox) + # ^^^ this is done before handle_possible_exception() because we + # need the box to show up in get_list_of_active_boxes() if exc: self.metainterp.handle_possible_exception() else: @@ -1323,7 +1332,8 @@ self.last_exc_value_box = None self.popframe() if self.framestack: - self.framestack[-1].make_result_of_lastop(resultbox) + if resultbox is not None: + self.framestack[-1].make_result_of_lastop(resultbox) raise ChangeFrame else: try: @@ -1552,7 +1562,7 @@ redkey = original_boxes[num_green_args:] self.resumekey = compile.ResumeFromInterpDescr(original_greenkey, redkey) - self.seen_can_enter_jit = False + self.seen_can_enter_jit_for_jdindex = -1 try: self.interpret() except GenerateMergePoint, gmp: @@ -1579,7 +1589,7 @@ # because we cannot reconstruct the beginning of the proper loop self.current_merge_points = [(original_greenkey, -1)] self.resumekey = key - self.seen_can_enter_jit = False + self.seen_can_enter_jit_for_jdindex = -1 try: self.prepare_resume_from_failure(key.guard_opnum) self.interpret() @@ -2232,7 +2242,8 @@ else: resultbox = unboundmethod(self, *args) # - self.make_result_of_lastop(resultbox) + if resultbox is not None: + self.make_result_of_lastop(resultbox) # unboundmethod = getattr(MIFrame, 'opimpl_' + name).im_func argtypes = unrolling_iterable(unboundmethod.argtypes) Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_basic.py Fri Jul 9 12:38:04 2010 @@ -116,8 +116,9 @@ class JitMixin: basic = True - def check_loops(self, expected=None, **check): - get_stats().check_loops(expected=expected, **check) + def check_loops(self, expected=None, everywhere=False, **check): + get_stats().check_loops(expected=expected, everywhere=everywhere, + **check) def check_loop_count(self, count): """NB. This is a hack; use check_tree_loop_count() or check_enter_count() for the real thing. Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_jitdriver.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_jitdriver.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_jitdriver.py Fri Jul 9 12:38:04 2010 @@ -14,7 +14,6 @@ def test_simple(self): myjitdriver1 = JitDriver(greens=[], reds=['n', 'm'], - can_inline = lambda *args: False, get_printable_location = getloc1) myjitdriver2 = JitDriver(greens=['g'], reds=['r'], get_printable_location = getloc2) Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py Fri Jul 9 12:38:04 2010 @@ -5,7 +5,7 @@ from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.rpython.annlowlevel import hlstr -from pypy.jit.metainterp.warmspot import CannotInlineCanEnterJit, get_stats +from pypy.jit.metainterp.warmspot import get_stats class RecursiveTests: @@ -968,6 +968,94 @@ assert res == portal(2, 0) self.check_loops(call_assembler=2) + def test_inline_without_hitting_the_loop(self): + driver = JitDriver(greens = ['codeno'], reds = ['i'], + get_printable_location = lambda codeno : str(codeno)) + + def portal(codeno): + i = 0 + while True: + driver.jit_merge_point(codeno=codeno, i=i) + if codeno < 10: + i += portal(20) + codeno += 1 + elif codeno == 10: + if i > 63: + return i + codeno = 0 + driver.can_enter_jit(codeno=codeno, i=i) + else: + return 1 + + assert portal(0) == 70 + res = self.meta_interp(portal, [0], inline=True) + assert res == 70 + self.check_loops(call_assembler=0) + + def test_inline_with_hitting_the_loop_sometimes(self): + driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], + get_printable_location = lambda codeno : str(codeno)) + + def portal(codeno, k): + if k > 2: + return 1 + i = 0 + while True: + driver.jit_merge_point(codeno=codeno, i=i, k=k) + if codeno < 10: + i += portal(codeno + 5, k+1) + codeno += 1 + elif codeno == 10: + if i > [-1, 2000, 63][k]: + return i + codeno = 0 + driver.can_enter_jit(codeno=codeno, i=i, k=k) + else: + return 1 + + assert portal(0, 1) == 2095 + res = self.meta_interp(portal, [0, 1], inline=True) + assert res == 2095 + self.check_loops(call_assembler=6, everywhere=True) + + def test_inline_with_hitting_the_loop_sometimes_exc(self): + driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], + get_printable_location = lambda codeno : str(codeno)) + class GotValue(Exception): + def __init__(self, result): + self.result = result + + def portal(codeno, k): + if k > 2: + raise GotValue(1) + i = 0 + while True: + driver.jit_merge_point(codeno=codeno, i=i, k=k) + if codeno < 10: + try: + portal(codeno + 5, k+1) + except GotValue, e: + i += e.result + codeno += 1 + elif codeno == 10: + if i > [-1, 2000, 63][k]: + raise GotValue(i) + codeno = 0 + driver.can_enter_jit(codeno=codeno, i=i, k=k) + else: + raise GotValue(1) + + def main(codeno, k): + try: + portal(codeno, k) + except GotValue, e: + return e.result + + assert main(0, 1) == 2095 + res = self.meta_interp(main, [0, 1], inline=True) + assert res == 2095 + self.check_loops(call_assembler=6, everywhere=True) + # There is a test which I fail to write. # * what happens if we call recursive_call while blackholing # this seems to be completely corner case and not really happening Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py Fri Jul 9 12:38:04 2010 @@ -136,9 +136,6 @@ class ContinueRunningNormallyBase(JitException): pass -class CannotInlineCanEnterJit(JitException): - pass - # ____________________________________________________________ class WarmRunnerDesc(object): Modified: pypy/branch/kill-caninline/pypy/rlib/jit.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rlib/jit.py (original) +++ pypy/branch/kill-caninline/pypy/rlib/jit.py Fri Jul 9 12:38:04 2010 @@ -253,8 +253,7 @@ def __init__(self, greens=None, reds=None, virtualizables=None, get_jitcell_at=None, set_jitcell_at=None, - can_inline=None, get_printable_location=None, - confirm_enter_jit=None): + get_printable_location=None, confirm_enter_jit=None): if greens is not None: self.greens = greens if reds is not None: @@ -270,7 +269,6 @@ self.get_jitcell_at = get_jitcell_at self.set_jitcell_at = set_jitcell_at self.get_printable_location = get_printable_location - self.can_inline = can_inline self.confirm_enter_jit = confirm_enter_jit def _freeze_(self): @@ -384,7 +382,6 @@ self.annotate_hook(driver.get_jitcell_at, driver.greens, **kwds_s) self.annotate_hook(driver.set_jitcell_at, driver.greens, [s_jitcell], **kwds_s) - self.annotate_hook(driver.can_inline, driver.greens, **kwds_s) self.annotate_hook(driver.get_printable_location, driver.greens, **kwds_s) def annotate_hook(self, func, variables, args_s=[], **kwds_s): From arigo at codespeak.net Fri Jul 9 12:39:17 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Jul 2010 12:39:17 +0200 (CEST) Subject: [pypy-svn] r76067 - pypy/branch/kill-caninline/pypy/jit/metainterp/test Message-ID: <20100709103917.CAA78282C00@codespeak.net> Author: arigo Date: Fri Jul 9 12:39:16 2010 New Revision: 76067 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_jitdriver.py Log: Fix test. Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_jitdriver.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_jitdriver.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_jitdriver.py Fri Jul 9 12:39:16 2010 @@ -45,7 +45,6 @@ # this is not an example of reasonable code: loop1() is unrolled # 'n/m' times, where n and m are given as red arguments. myjitdriver1 = JitDriver(greens=[], reds=['n', 'm'], - can_inline = lambda *args: True, get_printable_location = getloc1) myjitdriver2 = JitDriver(greens=['g'], reds=['r'], get_printable_location = getloc2) From arigo at codespeak.net Fri Jul 9 12:40:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Jul 2010 12:40:45 +0200 (CEST) Subject: [pypy-svn] r76068 - pypy/trunk/pypy/jit/backend/llgraph Message-ID: <20100709104045.8AE3D282C00@codespeak.net> Author: arigo Date: Fri Jul 9 12:40:44 2010 New Revision: 76068 Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Log: Merge r76065. Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Fri Jul 9 12:40:44 2010 @@ -421,11 +421,9 @@ global _last_exception assert _last_exception is None, "exception left behind" verbose = True - operations = self.loop.operations - opindex = 0 + self.opindex = 0 while True: - self.opindex = opindex - op = operations[opindex] + op = self.loop.operations[self.opindex] args = [self.getenv(v) for v in op.args] if not op.is_final(): try: @@ -439,8 +437,8 @@ args = [self.getenv(v) for v in op.fail_args if v] assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) - operations = op.jump_target.operations - opindex = 0 + self.loop = op.jump_target + self.opindex = 0 continue else: self._populate_fail_args(op) @@ -465,14 +463,13 @@ raise Exception("op.result.concretetype is %r" % (RESTYPE,)) self.env[op.result] = x - opindex += 1 + self.opindex += 1 continue if op.opnum == rop.JUMP: assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) self.loop = op.jump_target - operations = self.loop.operations - opindex = 0 + self.opindex = 0 _stats.exec_jumps += 1 elif op.opnum == rop.FINISH: if self.verbose: From cfbolz at codespeak.net Fri Jul 9 12:53:47 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 9 Jul 2010 12:53:47 +0200 (CEST) Subject: [pypy-svn] r76069 - in pypy/branch/reflex-support/pypy/module/cppyy: . include test Message-ID: <20100709105347.62299282C00@codespeak.net> Author: cfbolz Date: Fri Jul 9 12:53:46 2010 New Revision: 76069 Modified: pypy/branch/reflex-support/pypy/module/cppyy/ (props changed) pypy/branch/reflex-support/pypy/module/cppyy/include/ (props changed) pypy/branch/reflex-support/pypy/module/cppyy/include/cppyy.h (props changed) pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (props changed) pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py (props changed) pypy/branch/reflex-support/pypy/module/cppyy/test/ (props changed) pypy/branch/reflex-support/pypy/module/cppyy/test/bench1.py (props changed) Log: fixeol and fix svn properties From antocuni at codespeak.net Fri Jul 9 13:02:30 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 9 Jul 2010 13:02:30 +0200 (CEST) Subject: [pypy-svn] r76070 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100709110230.8C9F8282C00@codespeak.net> Author: antocuni Date: Fri Jul 9 13:02:29 2010 New Revision: 76070 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py Log: fix incorrect signature Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Fri Jul 9 13:02:29 2010 @@ -39,7 +39,7 @@ compilation_info=eci) c_deallocate = rffi.llexternal( "cppyy_deallocate", - [C_TYPEHANDLE, C_OBJECT], rffi.VOIDP, + [C_TYPEHANDLE, C_OBJECT], rffi.VOID, compilation_info=eci) c_call_v = rffi.llexternal( "cppyy_call_v", From arigo at codespeak.net Fri Jul 9 14:59:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Jul 2010 14:59:45 +0200 (CEST) Subject: [pypy-svn] r76071 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20100709125945.DE79C282C00@codespeak.net> Author: arigo Date: Fri Jul 9 14:59:43 2010 New Revision: 76071 Modified: pypy/trunk/pypy/jit/metainterp/test/test_recursive.py Log: This test makes no sense as it is written. Modified: pypy/trunk/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_recursive.py Fri Jul 9 14:59:43 2010 @@ -523,7 +523,7 @@ def test_trace_from_start(self): def p(pc, code): code = hlstr(code) - return "%s %d %s" % (code, pc, code[pc]) + return "'%s' at %d: %s" % (code, pc, code[pc]) def c(pc, code): return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], @@ -556,6 +556,7 @@ result = 0 for i in range(m): result += f('+-cl--', i) + g(50) # <--- this test is broken self.meta_interp(g, [50], backendopt=True) self.check_tree_loop_count(3) self.check_history(int_add=1) From arigo at codespeak.net Fri Jul 9 15:00:39 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Jul 2010 15:00:39 +0200 (CEST) Subject: [pypy-svn] r76072 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20100709130039.C1061282C00@codespeak.net> Author: arigo Date: Fri Jul 9 15:00:38 2010 New Revision: 76072 Modified: pypy/trunk/pypy/jit/metainterp/test/test_recursive.py Log: Fix the test. Modified: pypy/trunk/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_recursive.py Fri Jul 9 15:00:38 2010 @@ -537,9 +537,9 @@ op = code[pc] if op == "+": n += 7 - if op == "-": + elif op == "-": n -= 1 - if op == "c": + elif op == "c": n = f('---', n) elif op == "l": if n > 0: @@ -556,7 +556,7 @@ result = 0 for i in range(m): result += f('+-cl--', i) - g(50) # <--- this test is broken + g(50) self.meta_interp(g, [50], backendopt=True) self.check_tree_loop_count(3) self.check_history(int_add=1) From hakanardo at codespeak.net Fri Jul 9 15:52:32 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Fri, 9 Jul 2010 15:52:32 +0200 (CEST) Subject: [pypy-svn] r76073 - pypy/branch/interplevel-array/pypy/module/array/test Message-ID: <20100709135232.8C77F282C09@codespeak.net> Author: hakanardo Date: Fri Jul 9 15:52:31 2010 New Revision: 76073 Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: more testing Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Fri Jul 9 15:52:31 2010 @@ -18,11 +18,15 @@ class AppTestArray: def setup_class(cls): - cls.space = gettestobjspace(usemodules=('array',)) + cls.space = gettestobjspace(usemodules=('array','struct')) cls.w_array = cls.space.appexec([], """(): import array return array.array """) + cls.w_unpack = cls.space.appexec([], """(): + import struct + return struct.unpack + """) def test_ctor(self): raises(TypeError, self.array, 'hi') @@ -306,13 +310,24 @@ a=self.array('i', s) assert a[0]==1 and a[1]==2 and a[2]==3 - from struct import unpack + unpack=self.unpack values = (-129, 128, -128, 127, 0, 255, -1, 256, -32760, 32760) s = self.array('i', values).tostring() a=unpack('i'*len(values), s) assert a==values + + for tcodes, values in (('bhilfd', (-128, 127, 0, 1, 7, -10)), + ('BHILfd', (127, 0, 1, 7, 255, 169)), + ('hilHILfd', (32760, 30123, 3422, 23244))): + for tc in tcodes: + values += ((2 ** self.array(tc).itemsize) / 2 - 1, ) + s = self.array(tc, values).tostring() + a=unpack(tc*len(values), s) + assert a==values + + - #FXIME: How to test? + #FIXME: How to test? #from cStringIO import StringIO #f=StringIO() #self.array('c', ('h', 'i')).tofile(f) From getxsick at codespeak.net Fri Jul 9 16:36:36 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 9 Jul 2010 16:36:36 +0200 (CEST) Subject: [pypy-svn] r76078 - pypy/trunk/pypy/rlib Message-ID: <20100709143636.DC461282C07@codespeak.net> Author: getxsick Date: Fri Jul 9 16:36:35 2010 New Revision: 76078 Modified: pypy/trunk/pypy/rlib/rdynload.py Log: kill redundant statement Modified: pypy/trunk/pypy/rlib/rdynload.py ============================================================================== --- pypy/trunk/pypy/rlib/rdynload.py (original) +++ pypy/trunk/pypy/rlib/rdynload.py Fri Jul 9 16:36:35 2010 @@ -18,8 +18,6 @@ if _WIN32: from pypy.rlib import rwin32 - -if _WIN32: includes = ['windows.h'] else: includes = ['dlfcn.h'] From benjamin at codespeak.net Fri Jul 9 20:17:56 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 9 Jul 2010 20:17:56 +0200 (CEST) Subject: [pypy-svn] r76083 - in pypy/branch/fast-forward/pypy/module/struct: . test Message-ID: <20100709181756.2171A282C00@codespeak.net> Author: benjamin Date: Fri Jul 9 20:17:54 2010 New Revision: 76083 Modified: pypy/branch/fast-forward/pypy/module/struct/formatiterator.py pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py Log: warnings about type coerion Why it took me all morning to get this to work is a mystery. Modified: pypy/branch/fast-forward/pypy/module/struct/formatiterator.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/struct/formatiterator.py (original) +++ pypy/branch/fast-forward/pypy/module/struct/formatiterator.py Fri Jul 9 20:17:54 2010 @@ -76,7 +76,12 @@ if not e.match(space, space.w_TypeError): raise e if not space.is_true(space.isinstance(w_obj, space.w_float)): + # CPython silliness. Have a warning, and then raise the error. + space.warn("integer argument expected, got non-integer", + space.w_DeprecationWarning) raise e + space.warn("struct: integer argument expected, got float", + space.w_DeprecationWarning) return space.int(w_obj) # wrapped float -> wrapped int or long else: Modified: pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py (original) +++ pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py Fri Jul 9 20:17:54 2010 @@ -55,6 +55,17 @@ assert calcsize('=bQ3i') == 1 + 8 + 3*4 + def test_deprecation_warning(self): + import warnings + for code in 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q': + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + raises(TypeError, self.struct.pack, code, 3j) + assert len(w) == 1 + assert str(w[0].message) == "integer argument expected, got non-integer" + assert w[0].category is DeprecationWarning + + def test_pack_standard_little(self): """ Check packing with the '<' format specifier. From hakanardo at codespeak.net Fri Jul 9 20:49:45 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Fri, 9 Jul 2010 20:49:45 +0200 (CEST) Subject: [pypy-svn] r76084 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100709184945.0776A282C00@codespeak.net> Author: hakanardo Date: Fri Jul 9 20:49:44 2010 New Revision: 76084 Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: a few more methods Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/app_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/app_array.py Fri Jul 9 20:49:44 2010 @@ -116,6 +116,12 @@ self._setlen(0) self.fromlist(lst) + def insert(self, i, x): + lst=self.tolist() + lst.insert(i, x) + self._setlen(0) + self.fromlist(lst) + def __eq__(self, other): if not self._isarray(other): return NotImplemented Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Fri Jul 9 20:49:44 2010 @@ -234,6 +234,9 @@ self.descr_setslice(start, stop, step, w_item) descr_setitem.unwrap_spec = ['self', W_Root, W_Root] + def charbuf(self): + return rffi.cast(rffi.CCHARP, self.buffer) + def descr_fromstring(self, s): if len(s)%mytype.bytes !=0: msg = 'string length not a multiple of item size' @@ -248,7 +251,7 @@ #self.buffer[oldlen + i]=self.item_w(self.space.wrap(item)) self.buffer[oldlen + i]=rffi.cast(mytype.itemtype, item) else: - pbuf = rffi.cast(rffi.CCHARP, self.buffer) + pbuf =self.charbuf() for i in range(len(s)): pbuf[oldlen * mytype.bytes + i] = s[i] @@ -263,10 +266,10 @@ descr_tolist.unwrap_spec = ['self'] def descr_tostring(self): - pbuf = rffi.cast(rffi.CCHARP, self.buffer) + pbuf = self.charbuf() s = '' i=0 - while i < self.len * self.itemsize: + while i < self.len * mytype.bytes: s += pbuf[i] i+=1 return self.space.wrap(s) @@ -292,12 +295,51 @@ w_new_inst = mod.get('array') return space.newtuple([w_new_inst, space.newtuple(args)]) + def descr_pop(self, i=-1): + if i < 0: i += self.len + if i < 0 or i >= self.len: + msg = 'pop index out of range' + raise OperationError(self.space.w_IndexError, self.space.wrap(msg)) + val = self.buffer[i] + while i < self.len - 1: + self.buffer[i] = self.buffer[i + 1] + i += 1 + self.setlen(self.len-1) + return self.space.wrap(val) + descr_pop.unwrap_spec = ['self', int] + + + def descr_copy(self): + w_a=mytype.w_class(self.space) + w_a.descr_fromsequence(self) + return w_a + + def descr_buffer_info(self): + space = self.space + w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer)) + w_len = space.wrap(self.len) + return space.newtuple([w_ptr, w_len]) + + def descr_byteswap(self): + if mytype.bytes not in [1, 2, 4, 8]: + msg="byteswap not supported for this array" + raise OperationError(self.space.w_RuntimeError, self.space.wrap(msg)) + bytes = self.charbuf() + tmp = [0] * mytype.bytes + for start in range(0, self.len * mytype.bytes, mytype.bytes): + stop = start + mytype.bytes - 1 + for i in range(mytype.bytes): + tmp[i] = bytes[start + i] + for i in range(mytype.bytes): + bytes[stop - i] = tmp[i] + + def descr_itemsize(space, self): - return space.wrap(self.itemsize) + return space.wrap(mytype.bytes) def descr_typecode(space, self): - return space.wrap(self.typecode) + return space.wrap(mytype.typecode) W_Array.__name__ = 'W_ArrayType_'+mytype.typecode W_Array.typedef = TypeDef( @@ -334,6 +376,8 @@ index = appmethod('index'), remove = appmethod('remove'), reverse = appmethod('reverse'), + insert = appmethod('insert'), + pop = interp2app(W_Array.descr_pop), __eq__ = appmethod('__eq__'), __ne__ = appmethod('__ne__'), @@ -344,15 +388,11 @@ _isarray = interp2app(W_Array.descr_isarray), __reduce__ = interp2app(W_Array.descr_reduce), + __copy__ = interp2app(W_Array.descr_copy), + + buffer_info = interp2app(W_Array.descr_buffer_info), - - # TODO: - # __cmp__ - #byteswap = - #buffer_info = - #__copy__ = - #__reduce__ = - # insert, pop, + byteswap = interp2app(W_Array.descr_byteswap), ) mytype.w_class = W_Array Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Fri Jul 9 20:49:44 2010 @@ -337,8 +337,12 @@ assert self.array('u', unicode('hello')).tounicode() == unicode('hello') def test_buffer(self): - assert buffer(self.array('h', 'Hi'))[1] == 'i' - + a = self.array('h', 'Hi') + buf = buffer(a) + assert buf[1] == 'i' + raises(TypeError, buf.__setitem__, 1, 'o') + + def test_list_methods(self): assert repr(self.array('i')) == "array('i')" assert repr(self.array('i', [1, 2, 3])) == "array('i', [1, 2, 3])" @@ -353,11 +357,24 @@ a.reverse() assert repr(a) == "array('i', [1, 2, 1, 3, 2, 1])" - if False: - a.remove(3) - assert repr(a) == "array('i', [1, 2, 1, 2, 1])" - a.remove(1) - assert repr(a) == "array('i', [2, 1, 2, 1])" + a.remove(3) + assert repr(a) == "array('i', [1, 2, 1, 2, 1])" + a.remove(1) + assert repr(a) == "array('i', [2, 1, 2, 1])" + + a.pop() + assert repr(a) == "array('i', [2, 1, 2])" + + a.pop(1) + assert repr(a) == "array('i', [2, 2])" + + a.pop(-2) + assert repr(a) == "array('i', [2])" + + a.insert(1,7) + a.insert(0,8) + a.insert(-1,9) + assert repr(a) == "array('i', [8, 2, 9, 7])" def test_compare(self): a = self.array('i', [1, 2, 3]) @@ -400,6 +417,12 @@ assert (a >= c) is False assert (c >= a) is True + assert cmp(a, a) == 0 + assert cmp(a, b) == 0 + assert cmp(a, c) < 0 + assert cmp(b, a) == 0 + assert cmp(c, a) > 0 + def test_reduce(self): import pickle a = self.array('i', [1, 2, 3]) @@ -412,6 +435,31 @@ b = pickle.loads(s) assert len(b) == 0 and b.typecode == 'l' + def test_misc(self): + a = self.array('i', [1, 2, 3]) + from copy import copy + b = copy(a) + a[1] = 7 + assert repr(b) == "array('i', [1, 2, 3])" + + bi=b.buffer_info() + assert bi[0] != 0 # FIXME: How can the address be tested? + assert bi[1] == 3 + + for tc in 'bhilBHIL': + a=self.array(tc, [1, 2, 3]) + a.byteswap() + assert len(a)==3 + assert a[0] == 1 * (256 ** (a.itemsize-1)) + assert a[1] == 2 * (256 ** (a.itemsize-1)) + assert a[2] == 3 * (256 ** (a.itemsize-1)) + a.byteswap() + assert len(a)==3 + assert a[0] == 1 + assert a[1] == 2 + assert a[2] == 3 + + #FIXME #def test_type(self): # for t in 'bBhHiIlLfdcu': From benjamin at codespeak.net Sat Jul 10 01:02:23 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 10 Jul 2010 01:02:23 +0200 (CEST) Subject: [pypy-svn] r76087 - pypy/trunk/pypy/jit/metainterp Message-ID: <20100709230223.8E93C282BD4@codespeak.net> Author: benjamin Date: Sat Jul 10 01:02:21 2010 New Revision: 76087 Modified: pypy/trunk/pypy/jit/metainterp/executor.py Log: want a pointer not an integer Modified: pypy/trunk/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/executor.py (original) +++ pypy/trunk/pypy/jit/metainterp/executor.py Sat Jul 10 01:02:21 2010 @@ -91,7 +91,7 @@ return BoxInt(cpu.bh_getarrayitem_gc_i(arraydescr, array, index)) def do_getarrayitem_raw(cpu, _, arraybox, indexbox, arraydescr): - array = arraybox.getint() + array = arraybox.getref_base() index = indexbox.getint() assert not arraydescr.is_array_of_pointers() if arraydescr.is_array_of_floats(): From benjamin at codespeak.net Sat Jul 10 16:44:09 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 10 Jul 2010 16:44:09 +0200 (CEST) Subject: [pypy-svn] r76089 - in pypy/branch/fast-forward/pypy/module/struct: . test Message-ID: <20100710144409.4A77C282BD4@codespeak.net> Author: benjamin Date: Sat Jul 10 16:44:07 2010 New Revision: 76089 Modified: pypy/branch/fast-forward/pypy/module/struct/formatiterator.py pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py Log: fun with deprecation warnings Modified: pypy/branch/fast-forward/pypy/module/struct/formatiterator.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/struct/formatiterator.py (original) +++ pypy/branch/fast-forward/pypy/module/struct/formatiterator.py Sat Jul 10 16:44:07 2010 @@ -1,10 +1,11 @@ from pypy.interpreter.error import OperationError +from pypy.rlib.objectmodel import specialize from pypy.rlib.rstruct.error import StructError from pypy.rlib.rstruct.standardfmttable import PACK_ACCEPTS_BROKEN_INPUT -from pypy.rlib.rstruct.formatiterator import FormatIterator, \ - CalcSizeFormatIterator +from pypy.rlib.rstruct.formatiterator import (FormatIterator, + CalcSizeFormatIterator) class PackFormatIterator(FormatIterator): @@ -44,44 +45,46 @@ # permissive version - accepts float arguments too def accept_int_arg(self): - w_obj = self.accept_obj_arg() - try: - return self.space.int_w(w_obj) - except OperationError, e: - return self.space.int_w(self._maybe_float(e, w_obj)) + return self._accept_integral("int_w") def accept_uint_arg(self): - w_obj = self.accept_obj_arg() - try: - return self.space.uint_w(w_obj) - except OperationError, e: - return self.space.uint_w(self._maybe_float(e, w_obj)) + return self._accept_integral("uint_w") def accept_longlong_arg(self): - w_obj = self.accept_obj_arg() - try: - return self.space.r_longlong_w(w_obj) - except OperationError, e: - return self.space.r_longlong_w(self._maybe_float(e, w_obj)) + return self._accept_integral("r_longlong_w") def accept_ulonglong_arg(self): + return self._accept_integral("r_ulonglong_w") + + @specialize.arg(1) + def _accept_integral(self, meth): + space = self.space w_obj = self.accept_obj_arg() - try: - return self.space.r_ulonglong_w(w_obj) - except OperationError, e: - return self.space.r_ulonglong_w(self._maybe_float(e, w_obj)) + if (space.isinstance_w(w_obj, space.w_int) or + space.isinstance_w(w_obj, space.w_long)): + w_index = w_obj + else: + w_index = None + w_index_method = space.lookup(w_obj, "__index__") + if w_index_method is not None: + try: + w_index = space.index(w_obj) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + pass + if w_index is None: + w_index = self._maybe_float(w_obj) + return getattr(space, meth)(w_index) - def _maybe_float(self, e, w_obj): + def _maybe_float(self, w_obj): space = self.space - if not e.match(space, space.w_TypeError): - raise e - if not space.is_true(space.isinstance(w_obj, space.w_float)): - # CPython silliness. Have a warning, and then raise the error. + if space.is_true(space.isinstance(w_obj, space.w_float)): + space.warn("struct: integer argument expected, got float", + space.w_DeprecationWarning) + else: space.warn("integer argument expected, got non-integer", space.w_DeprecationWarning) - raise e - space.warn("struct: integer argument expected, got float", - space.w_DeprecationWarning) return space.int(w_obj) # wrapped float -> wrapped int or long else: Modified: pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py (original) +++ pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py Sat Jul 10 16:44:07 2010 @@ -55,6 +55,13 @@ assert calcsize('=bQ3i') == 1 + 8 + 3*4 + def test_index(self): + class X(object): + def __index__(self): + return 3 + assert self.struct.unpack("i", self.struct.pack("i", X()))[0] == 3 + + def test_deprecation_warning(self): import warnings for code in 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q': From benjamin at codespeak.net Sat Jul 10 18:39:14 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 10 Jul 2010 18:39:14 +0200 (CEST) Subject: [pypy-svn] r76090 - in pypy/branch/fast-forward/pypy: objspace/std rlib/rstruct Message-ID: <20100710163914.5CFED282BD4@codespeak.net> Author: benjamin Date: Sat Jul 10 18:39:12 2010 New Revision: 76090 Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py pypy/branch/fast-forward/pypy/objspace/std/floattype.py pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Log: use rarithmetic.INFINITY instead of float('inf') Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Sat Jul 10 18:39:12 2010 @@ -9,7 +9,8 @@ from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.longobject import W_LongObject from pypy.rlib.rarithmetic import ovfcheck_float_to_int, intmask, isinf, isnan -from pypy.rlib.rarithmetic import formatd, LONG_BIT, FL_MAXINT, FL_MININT +from pypy.rlib.rarithmetic import (formatd, LONG_BIT, FL_MAXINT, FL_MININT, + INFINITY) from pypy.rlib.rbigint import rbigint from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rfloat @@ -438,7 +439,7 @@ elif x == 0.0: if y < 0.0: if isinf(y): - return space.wrap(float("inf")) + return space.wrap(INFINITY) raise OperationError(space.w_ZeroDivisionError, space.wrap("0.0 cannot be raised to " "a negative power")) Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floattype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floattype.py Sat Jul 10 18:39:12 2010 @@ -1,7 +1,7 @@ import math import sys from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib import rfloat +from pypy.rlib import rfloat, rarithmetic from pypy.interpreter import gateway from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.error import OperationError @@ -99,7 +99,7 @@ i += 1 if length - i >= 2 and s[i:i + 2].lower() == "nf": i += 2 - value = float("inf") + value = rarithmetic.INFINITY if length - i >= 5 and s[i:i + 5].lower() == "inity": i += 5 elif s[i] == "n" or s[i] == "N": Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py Sat Jul 10 18:39:12 2010 @@ -128,7 +128,7 @@ unpack = unpack_bool else: cpython_checks_range = fmtchar in 'bBhH' - pack = std.make_int_packer(size, signed, cpython_checks_range) + pack = std.make_int_packer(size, signed, True) unpack = std.make_int_unpacker(size, signed) native_fmttable[fmtchar] = {'size': size, Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Sat Jul 10 18:39:12 2010 @@ -238,9 +238,8 @@ for c, size in [('b', 1), ('h', 2), ('i', 4), ('l', 4), ('q', 8)]: standard_fmttable[c] = {'size': size, - 'pack': make_int_packer(size, True, False), + 'pack': make_int_packer(size, True, True), 'unpack': make_int_unpacker(size, True)} standard_fmttable[c.upper()] = {'size': size, - 'pack': make_int_packer(size, False, - False), + 'pack': make_int_packer(size, False, True), 'unpack': make_int_unpacker(size, False)} From benjamin at codespeak.net Sat Jul 10 18:44:33 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 10 Jul 2010 18:44:33 +0200 (CEST) Subject: [pypy-svn] r76091 - pypy/branch/fast-forward/pypy/rlib/rstruct Message-ID: <20100710164433.5C7B5282BD4@codespeak.net> Author: benjamin Date: Sat Jul 10 18:44:30 2010 New Revision: 76091 Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Log: revert changes I'm working on Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py Sat Jul 10 18:44:30 2010 @@ -128,7 +128,7 @@ unpack = unpack_bool else: cpython_checks_range = fmtchar in 'bBhH' - pack = std.make_int_packer(size, signed, True) + pack = std.make_int_packer(size, signed, cpython_checks_range) unpack = std.make_int_unpacker(size, signed) native_fmttable[fmtchar] = {'size': size, Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Sat Jul 10 18:44:30 2010 @@ -238,8 +238,9 @@ for c, size in [('b', 1), ('h', 2), ('i', 4), ('l', 4), ('q', 8)]: standard_fmttable[c] = {'size': size, - 'pack': make_int_packer(size, True, True), + 'pack': make_int_packer(size, True, False), 'unpack': make_int_unpacker(size, True)} standard_fmttable[c.upper()] = {'size': size, - 'pack': make_int_packer(size, False, True), + 'pack': make_int_packer(size, False, + False), 'unpack': make_int_unpacker(size, False)} From benjamin at codespeak.net Sat Jul 10 19:03:31 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 10 Jul 2010 19:03:31 +0200 (CEST) Subject: [pypy-svn] r76092 - pypy/branch/fast-forward/pypy/rlib/rstruct Message-ID: <20100710170331.791A0282B9D@codespeak.net> Author: benjamin Date: Sat Jul 10 19:03:30 2010 New Revision: 76092 Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Log: Cpython is now good and checks ranges Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py Sat Jul 10 19:03:30 2010 @@ -127,8 +127,7 @@ pack = pack_bool unpack = unpack_bool else: - cpython_checks_range = fmtchar in 'bBhH' - pack = std.make_int_packer(size, signed, cpython_checks_range) + pack = std.make_int_packer(size, signed, True) unpack = std.make_int_unpacker(size, signed) native_fmttable[fmtchar] = {'size': size, Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Sat Jul 10 19:03:30 2010 @@ -238,9 +238,8 @@ for c, size in [('b', 1), ('h', 2), ('i', 4), ('l', 4), ('q', 8)]: standard_fmttable[c] = {'size': size, - 'pack': make_int_packer(size, True, False), + 'pack': make_int_packer(size, True, True), 'unpack': make_int_unpacker(size, True)} standard_fmttable[c.upper()] = {'size': size, - 'pack': make_int_packer(size, False, - False), + 'pack': make_int_packer(size, False, True), 'unpack': make_int_unpacker(size, False)} From benjamin at codespeak.net Sat Jul 10 19:06:44 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 10 Jul 2010 19:06:44 +0200 (CEST) Subject: [pypy-svn] r76093 - in pypy/branch/fast-forward/pypy/rpython: . lltypesystem/module Message-ID: <20100710170644.CED5E282B9D@codespeak.net> Author: benjamin Date: Sat Jul 10 19:06:43 2010 New Revision: 76093 Modified: pypy/branch/fast-forward/pypy/rpython/extfuncregistry.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Log: register new math functions Modified: pypy/branch/fast-forward/pypy/rpython/extfuncregistry.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/extfuncregistry.py (original) +++ pypy/branch/fast-forward/pypy/rpython/extfuncregistry.py Sat Jul 10 19:06:43 2010 @@ -22,7 +22,10 @@ # and are part of math.h for name in ll_math.unary_math_functions: llimpl = getattr(ll_math, 'll_math_%s' % name, None) - f = getattr(math, name) + try: + f = getattr(math, name) + except AttributeError: + f = getattr(rarithmetic, name) register_external(f, [float], float, export_name="ll_math.ll_math_%s" % name, sandboxsafe=True, llimpl=llimpl) Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/module/ll_math.py Sat Jul 10 19:06:43 2010 @@ -316,7 +316,7 @@ 'acos', 'asin', 'atan', 'ceil', 'cos', 'cosh', 'exp', 'fabs', 'floor', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', 'log', 'log10', - 'acosh', 'asinh', 'atanh', # -- added in Python 2.6 + 'acosh', 'asinh', 'atanh', 'log1p', 'expm1' # -- added in Python 2.6 ] unary_math_functions_can_overflow = [ 'cosh', 'exp', 'log1p', 'sinh', 'expm1', From benjamin at codespeak.net Sat Jul 10 19:18:38 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 10 Jul 2010 19:18:38 +0200 (CEST) Subject: [pypy-svn] r76096 - in pypy/branch/fast-forward/pypy/rlib: . test Message-ID: <20100710171838.6A002282B9D@codespeak.net> Author: benjamin Date: Sat Jul 10 19:18:37 2010 New Revision: 76096 Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py Log: add replacement for libm round() Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Sat Jul 10 19:18:37 2010 @@ -143,6 +143,15 @@ return (u - 1.) * x / math.log(u) return math.exp(x) - 1. +def round_away(x): + # round() from libm + absx = abs(x) + if absx - math.floor(absx) >= .5: + r = math.ceil(absx) + else: + r = math.floor(absx) + return copysign(r, x) + def intmask(n): if isinstance(n, int): return int(n) # possibly bool->int Modified: pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py Sat Jul 10 19:18:37 2010 @@ -379,3 +379,13 @@ assert copysign(-1, -1) == -1 assert copysign(1, -1) == -1 assert copysign(1, -0.) == -1 + +def test_round_away(): + assert round_away(.1) == 0. + assert round_away(.5) == 1. + assert round_away(.7) == 1. + assert round_away(1.) == 1. + assert round_away(-.5) == -1. + assert round_away(-.1) == 0. + assert round_away(-.7) == -1. + assert round_away(0.) == 0. From benjamin at codespeak.net Sat Jul 10 19:22:17 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 10 Jul 2010 19:22:17 +0200 (CEST) Subject: [pypy-svn] r76097 - pypy/branch/fast-forward/pypy/module/math Message-ID: <20100710172217.BCE87282B9D@codespeak.net> Author: benjamin Date: Sat Jul 10 19:22:15 2010 New Revision: 76097 Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py Log: use round replacement Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Sat Jul 10 19:22:15 2010 @@ -522,7 +522,7 @@ def _sinpi(x): y = math.fmod(abs(x), 2.) - n = int(round(2. * y)) + n = int(rarithmetic.round_away(2. * y)) if n == 0: r = math.sin(math.pi * y) elif n == 1: From benjamin at codespeak.net Sat Jul 10 19:55:40 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 10 Jul 2010 19:55:40 +0200 (CEST) Subject: [pypy-svn] r76099 - in pypy/branch/fast-forward/pypy/module/math: . test Message-ID: <20100710175540.297B8282B9D@codespeak.net> Author: benjamin Date: Sat Jul 10 19:55:39 2010 New Revision: 76099 Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py pypy/branch/fast-forward/pypy/module/math/test/test_direct.py Log: fix log of negatives Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Sat Jul 10 19:55:39 2010 @@ -228,7 +228,7 @@ base = _get_double(space, w_base) if base <= 0.0: # just for raising the proper errors - return math1(space, math.log, base) + return math1(space, math.log, w_base) return _log_any(space, w_x, base) log.unwrap_spec = [ObjSpace, W_Root, W_Root] Modified: pypy/branch/fast-forward/pypy/module/math/test/test_direct.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/test/test_direct.py (original) +++ pypy/branch/fast-forward/pypy/module/math/test/test_direct.py Sat Jul 10 19:55:39 2010 @@ -55,6 +55,7 @@ ('pow', (10.0, 40000.0), OverflowError), ('ldexp', (10.0, 40000), OverflowError), ('log', (0.0,), ValueError), + ('log', (-1.,), ValueError), ('log10', (0.0,), ValueError), ] From benjamin at codespeak.net Sat Jul 10 20:33:11 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 10 Jul 2010 20:33:11 +0200 (CEST) Subject: [pypy-svn] r76100 - pypy/branch/fast-forward/pypy/module/math Message-ID: <20100710183311.3A26A282B9D@codespeak.net> Author: benjamin Date: Sat Jul 10 20:33:09 2010 New Revision: 76100 Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py Log: rewrite so the annotator doesn't think we're dividing by zero Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Sat Jul 10 20:33:09 2010 @@ -1,7 +1,7 @@ import math import sys -from pypy.rlib import rarithmetic +from pypy.rlib import rarithmetic, unroll from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped @@ -557,6 +557,8 @@ _lanczos_den_coeffs = [ 0.0, 39916800.0, 120543840.0, 150917976.0, 105258076.0, 45995730.0, 13339535.0, 2637558.0, 357423.0, 32670.0, 1925.0, 66.0, 1.0] +LANCZOS_N = len(_lanczos_den_coeffs) +_lanczos_n_iter = unroll.unrolling_iterable(range(LANCZOS_N)) _gamma_integrals = [ 1.0, 1.0, 2.0, 6.0, 24.0, 120.0, 720.0, 5040.0, 40320.0, 362880.0, 3628800.0, 39916800.0, 479001600.0, 6227020800.0, 87178291200.0, @@ -569,11 +571,11 @@ den = 0. assert x > 0. if x < 5.: - for i in range(len(_lanczos_den_coeffs) - 1, -1, -1): + for i in _lanczos_n_iter: num = num * x + _lanczos_num_coeffs[i] den = den * x + _lanczos_den_coeffs[i] else: - for i in range(len(_lanczos_den_coeffs)): + for i in _lanczos_n_iter: num = num / x + _lanczos_num_coeffs[i] den = den / x + _lanczos_den_coeffs[i] return num / den From benjamin at codespeak.net Sat Jul 10 20:39:06 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 10 Jul 2010 20:39:06 +0200 (CEST) Subject: [pypy-svn] r76101 - pypy/branch/fast-forward/pypy/module/math Message-ID: <20100710183906.ED6F3282B9D@codespeak.net> Author: benjamin Date: Sat Jul 10 20:39:05 2010 New Revision: 76101 Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py Log: must go backwards here Modified: pypy/branch/fast-forward/pypy/module/math/interp_math.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/math/interp_math.py (original) +++ pypy/branch/fast-forward/pypy/module/math/interp_math.py Sat Jul 10 20:39:05 2010 @@ -559,6 +559,7 @@ 13339535.0, 2637558.0, 357423.0, 32670.0, 1925.0, 66.0, 1.0] LANCZOS_N = len(_lanczos_den_coeffs) _lanczos_n_iter = unroll.unrolling_iterable(range(LANCZOS_N)) +_lanczos_n_iter_back = unroll.unrolling_iterable(range(LANCZOS_N - 1, -1, -1)) _gamma_integrals = [ 1.0, 1.0, 2.0, 6.0, 24.0, 120.0, 720.0, 5040.0, 40320.0, 362880.0, 3628800.0, 39916800.0, 479001600.0, 6227020800.0, 87178291200.0, @@ -571,7 +572,7 @@ den = 0. assert x > 0. if x < 5.: - for i in _lanczos_n_iter: + for i in _lanczos_n_iter_back: num = num * x + _lanczos_num_coeffs[i] den = den * x + _lanczos_den_coeffs[i] else: From benjamin at codespeak.net Sat Jul 10 21:54:30 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 10 Jul 2010 21:54:30 +0200 (CEST) Subject: [pypy-svn] r76102 - pypy/branch/fast-forward/pypy/module/exceptions Message-ID: <20100710195430.DAC17282B9D@codespeak.net> Author: benjamin Date: Sat Jul 10 21:54:28 2010 New Revision: 76102 Modified: pypy/branch/fast-forward/pypy/module/exceptions/interp_exceptions.py Log: be constant enough Modified: pypy/branch/fast-forward/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/fast-forward/pypy/module/exceptions/interp_exceptions.py Sat Jul 10 21:54:28 2010 @@ -119,7 +119,8 @@ def descr_unicode(self, space): w_str = space.lookup(self, "__str__") - w_base_str = space.wrap(W_BaseException.typedef.rawdict["__str__"]) + w_base_type = space.gettypeobject(W_BaseException.typedef) + w_base_str = w_base_type.dict_w["__str__"] if not space.is_w(w_str, w_base_str): w_as_str = space.get_and_call_function(w_str, space.wrap(self)) return space.call_function(space.w_unicode, w_as_str) From benjamin at codespeak.net Sun Jul 11 03:11:40 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Jul 2010 03:11:40 +0200 (CEST) Subject: [pypy-svn] r76103 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100711011140.855F02A2B3D@codespeak.net> Author: benjamin Date: Sun Jul 11 03:11:33 2010 New Revision: 76103 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py Log: remove unneeded error check Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Sun Jul 11 03:11:33 2010 @@ -588,9 +588,6 @@ if self._sign != "\0": msg = "sign not allowed with 'c' presentation type" raise OperationError(space.w_ValueError, space.wrap(msg)) - if self._thousands_sep: - msg = "thousands separator not allowed with 'c' presentation" - raise Operationerror(space.w_ValueError, space.wrap(msg)) value = space.int_w(w_num) if self.is_unicode: result = runicode.UNICHR(value) From benjamin at codespeak.net Sun Jul 11 03:15:06 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Jul 2010 03:15:06 +0200 (CEST) Subject: [pypy-svn] r76104 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100711011506.2F8B72A2B3D@codespeak.net> Author: benjamin Date: Sun Jul 11 03:15:04 2010 New Revision: 76104 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py Log: fix for non-latin 1 keys Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Sun Jul 11 03:15:04 2010 @@ -168,7 +168,15 @@ index = self.auto_numbering self.auto_numbering += 1 if index == -1: - arg_key = name[:i] + kwarg = name[:i] + if self.is_unicode: + try: + arg_key = kwarg.encode("latin-1") + except UnicodeEncodeError: + # Not going to be found in a dict of strings. + raise OperationError(space.w_KeyError, space.wrap(kwarg)) + else: + arg_key = kwarg try: w_arg = self.kwargs[arg_key] except KeyError: Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py Sun Jul 11 03:15:04 2010 @@ -112,6 +112,9 @@ assert self.s("{!s}").format(x()) == self.s("42") assert self.s("{!r}").format(x()) == self.s("32") + def test_non_latin1_key(self): + raises(KeyError, self.s("{\u1000}").format) + class AppTestStringFormat(BaseStringFormatTests): From benjamin at codespeak.net Sun Jul 11 06:16:30 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Jul 2010 06:16:30 +0200 (CEST) Subject: [pypy-svn] r76105 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100711041630.B42D12A2B3D@codespeak.net> Author: benjamin Date: Sun Jul 11 06:16:28 2010 New Revision: 76105 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py Log: tiptoe around unicode annotation Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Sun Jul 11 06:16:28 2010 @@ -401,8 +401,15 @@ return rstring.StringBuilder() def _unknown_presentation(self, tp): - msg = "unknown presentation for %s: '%s'" % (tp, self._type) - raise OperationError(self.space.w_ValueError, self.space.wrap(msg)) + msg = "unknown presentation for %s: '%s'" + if self.is_unicode: + the_msg = unicode(msg) + the_tp = tp.decode("ascii") + else: + the_msg = msg + the_tp = tp + w_msg = self.space.wrap(the_msg % (the_tp, self._type)) + raise OperationError(self.space.w_ValueError, w_msg) def format_string(self, string): space = self.space From benjamin at codespeak.net Sun Jul 11 18:30:18 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Jul 2010 18:30:18 +0200 (CEST) Subject: [pypy-svn] r76106 - pypy/branch/fast-forward/pypy/rlib/rstruct Message-ID: <20100711163018.21E06282B9C@codespeak.net> Author: benjamin Date: Sun Jul 11 18:30:15 2010 New Revision: 76106 Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py Log: reuse standfmt impl Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/nativefmttable.py Sun Jul 11 18:30:15 2010 @@ -59,13 +59,6 @@ doubleval = float(floatval) fmtiter.appendobj(doubleval) -def pack_bool(fmtiter): - truth = fmtiter.accept_bool_arg() - fmtiter.result.append('\x01' if truth else '\x00') - -def unpack_bool(fmtiter): - fmtiter.appendobj(bool(ord(fmtiter.read(1)))) - # ____________________________________________________________ # # Use rffi_platform to get the native sizes and alignments from the C compiler @@ -124,8 +117,8 @@ pack = pack_double unpack = unpack_double elif fmtchar == '?': - pack = pack_bool - unpack = unpack_bool + pack = std.pack_bool + unpack = std.unpack_bool else: pack = std.make_int_packer(size, signed, True) unpack = std.make_int_unpacker(size, signed) From benjamin at codespeak.net Sun Jul 11 18:31:16 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Jul 2010 18:31:16 +0200 (CEST) Subject: [pypy-svn] r76107 - pypy/branch/fast-forward/pypy/module/struct/test Message-ID: <20100711163116.BD0FE282B9C@codespeak.net> Author: benjamin Date: Sun Jul 11 18:31:15 2010 New Revision: 76107 Modified: pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py Log: remove test that is no longer applicable Modified: pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py (original) +++ pypy/branch/fast-forward/pypy/module/struct/test/test_struct.py Sun Jul 11 18:31:15 2010 @@ -363,18 +363,6 @@ raises(someerror, calcsize, "%dci" % (sys.maxint,)) - def test_broken_input(self): - """ - For compatibility: check that we also accept inputs that are - wrongly accepted by CPython 2.4. - """ - pack = self.struct.pack - assert pack("!b", 0xa0) == '\xa0' - assert pack("!B", -1.1) == '\xff' - assert pack("!h", 0xa000) == '\xa0\x00' - assert pack("!H", -2.2) == '\xff\xfe' - - def test_unicode(self): """ A PyPy extension: accepts the 'u' format character in native mode, From benjamin at codespeak.net Sun Jul 11 19:05:05 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Jul 2010 19:05:05 +0200 (CEST) Subject: [pypy-svn] r76108 - pypy/branch/fast-forward/pypy/rlib/rstruct Message-ID: <20100711170505.28162282B9C@codespeak.net> Author: benjamin Date: Sun Jul 11 19:05:03 2010 New Revision: 76108 Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Log: get a character for ord() Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/standardfmttable.py Sun Jul 11 19:05:03 2010 @@ -150,7 +150,8 @@ @specialize.argtype(0) def unpack_bool(fmtiter): - fmtiter.appendobj(bool(ord(fmtiter.read(1)))) + c = ord(fmtiter.read(1)[0]) + fmtiter.appendobj(bool(c)) @specialize.argtype(0) def unpack_string(fmtiter, count): From benjamin at codespeak.net Sun Jul 11 23:40:25 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Jul 2010 23:40:25 +0200 (CEST) Subject: [pypy-svn] r76109 - pypy/branch/fast-forward/pypy/interpreter/astcompiler Message-ID: <20100711214025.0814E282B9C@codespeak.net> Author: benjamin Date: Sun Jul 11 23:40:23 2010 New Revision: 76109 Modified: pypy/branch/fast-forward/pypy/interpreter/astcompiler/codegen.py Log: have to get lineno Modified: pypy/branch/fast-forward/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/astcompiler/codegen.py Sun Jul 11 23:40:23 2010 @@ -1307,6 +1307,7 @@ class ComprehensionCodeGenerator(AbstractFunctionCodeGenerator): def _compile(self, node): + assert isinstance(node, ast.expr) self.update_position(node.lineno) node.build_container(self) self._comp_generator(node, node.get_generators(), 0) From benjamin at codespeak.net Sun Jul 11 23:40:52 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Jul 2010 23:40:52 +0200 (CEST) Subject: [pypy-svn] r76110 - pypy/branch/fast-forward/pypy/module/posix Message-ID: <20100711214052.8D2AC282B9C@codespeak.net> Author: benjamin Date: Sun Jul 11 23:40:51 2010 New Revision: 76110 Modified: pypy/branch/fast-forward/pypy/module/posix/interp_posix.py Log: don't translate IOError case Modified: pypy/branch/fast-forward/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/fast-forward/pypy/module/posix/interp_posix.py Sun Jul 11 23:40:51 2010 @@ -1,5 +1,5 @@ from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped -from pypy.rlib import rposix +from pypy.rlib import rposix, objectmodel from pypy.rlib.rarithmetic import r_longlong from pypy.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError, wrap_oserror @@ -84,11 +84,13 @@ try: os.ftruncate(fd, length) except IOError, e: - # Python 2.6 raises an IOError here. Let's not repeat that mistake. - w_error = space.call_function(space.w_OSError, space.wrap(e.errno), - space.wrap(e.strerror), - space.wrap(e.filename)) - raise OperationError(space.w_OSError, w_error) + if not objectmodel.we_are_translated(): + # Python 2.6 raises an IOError here. Let's not repeat that mistake. + w_error = space.call_function(space.w_OSError, space.wrap(e.errno), + space.wrap(e.strerror), + space.wrap(e.filename)) + raise OperationError(space.w_OSError, w_error) + raise AssertionError except OSError, e: raise wrap_oserror(space, e) ftruncate.unwrap_spec = [ObjSpace, "c_int", r_longlong] From benjamin at codespeak.net Sun Jul 11 23:42:11 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Jul 2010 23:42:11 +0200 (CEST) Subject: [pypy-svn] r76111 - pypy/branch/fast-forward/pypy/interpreter Message-ID: <20100711214211.06EEF282B9C@codespeak.net> Author: benjamin Date: Sun Jul 11 23:42:10 2010 New Revision: 76111 Modified: pypy/branch/fast-forward/pypy/interpreter/pyopcode.py Log: don't try to translate this Modified: pypy/branch/fast-forward/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/pyopcode.py Sun Jul 11 23:42:10 2010 @@ -1053,7 +1053,6 @@ ofs = self.last_instr c = self.pycode.co_code[ofs] name = self.pycode.co_name - self.pycode.dump() raise BytecodeCorruption("unknown opcode, ofs=%d, code=%d, name=%s" % (ofs, ord(c), name) ) From benjamin at codespeak.net Sun Jul 11 23:43:02 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Jul 2010 23:43:02 +0200 (CEST) Subject: [pypy-svn] r76112 - pypy/branch/fast-forward/pypy/rlib Message-ID: <20100711214302.4DAAA282B9C@codespeak.net> Author: benjamin Date: Sun Jul 11 23:43:01 2010 New Revision: 76112 Modified: pypy/branch/fast-forward/pypy/rlib/rstring.py Log: enforce type strictness untranslated Modified: pypy/branch/fast-forward/pypy/rlib/rstring.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstring.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstring.py Sun Jul 11 23:43:01 2010 @@ -53,22 +53,26 @@ self.l = [] def append(self, s): + assert isinstance(s, self.tp) self.l.append(s) def append_slice(self, s, start, end): + assert isinstance(s, self.tp) assert 0 <= start <= end <= len(s) self.l.append(s[start:end]) def append_multiple_char(self, c, times): + assert isinstance(c, self.tp) self.l.append(c * times) -class StringBuilder(AbstractStringBuilder): def build(self): - return "".join(self.l) + return self.tp("").join(self.l) + +class StringBuilder(AbstractStringBuilder): + tp = str class UnicodeBuilder(AbstractStringBuilder): - def build(self): - return u''.join(self.l) + tp = unicode # XXX: This does log(mul) mallocs, the GCs probably make that efficient, but From benjamin at codespeak.net Sun Jul 11 23:43:45 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Jul 2010 23:43:45 +0200 (CEST) Subject: [pypy-svn] r76113 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100711214345.C10F1282B9C@codespeak.net> Author: benjamin Date: Sun Jul 11 23:43:44 2010 New Revision: 76113 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py Log: lots of mess to sort out annotation Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Sun Jul 11 23:43:44 2010 @@ -300,7 +300,7 @@ def _parse_spec(self, default_type, default_align): space = self.space - self._fill_char = "\0" + self._fill_char = self._lit("\0") self._align = default_align self._alternate = False self._sign = "\0" @@ -329,7 +329,7 @@ self._alternate = True i += 1 if self._fill_char == "\0" and length - i >= 1 and spec[i] == "0": - self._fill_char = "0" + self._fill_char = self._lit("0") if not got_align: self._align = "=" i += 1 @@ -387,11 +387,17 @@ self._right_pad = right return total + def _lit(self, s): + if self.is_unicode: + return s.decode("ascii") + else: + return s + def _pad(self, string): builder = self._builder() - builder.append_multiple_char(self._fill_char, self._left_pad) + builder.append_multiple_char(self._fill_char[0], self._left_pad) builder.append(string) - builder.append_multiple_char(self._fill_char, self._right_pad) + builder.append_multiple_char(self._fill_char[0], self._right_pad) return builder.build() def _builder(self): @@ -403,12 +409,10 @@ def _unknown_presentation(self, tp): msg = "unknown presentation for %s: '%s'" if self.is_unicode: - the_msg = unicode(msg) - the_tp = tp.decode("ascii") + inval_type = self._type.encode("ascii") else: - the_msg = msg - the_tp = tp - w_msg = self.space.wrap(the_msg % (the_tp, self._type)) + inval_type = self._type + w_msg = self.space.wrap(msg % (tp, inval_type)) raise OperationError(self.space.w_ValueError, w_msg) def format_string(self, string): @@ -430,7 +434,7 @@ if self._precision != -1 and length >= self._precision: length = self._precision if self._fill_char == "\0": - self._fill_char = " " + self._fill_char = self._lit(" ") self._calc_padding(string, length) return space.wrap(self._pad(string)) @@ -522,6 +526,7 @@ need_separator = False done = False groupings = len(grouping) + previous = 0 while True: group = ord(grouping[grouping_state]) if group > 0: @@ -547,7 +552,7 @@ group = max(max(left, min_width), 1) n_zeros = max(0, group - left) n_chars = max(0, min(left, group)) - ts = thousands_sep if need_separator else None + ts = self._loc_thousands if need_separator else None self._fill_digits(buf, digits, left, n_chars, n_zeros, ts) buf.reverse() self._grouped_digits = self.empty.join(buf) @@ -562,34 +567,40 @@ return self.empty.join(buf) - def _fill_number(self, spec, num, to_digits, n_digits, to_prefix, fill_char, + def _fill_number(self, spec, num, to_digits, to_prefix, fill_char, to_remainder, upper): out = self._builder() if spec.n_lpadding: - out.append_multiple_char(fill_char, spec.n_lpadding) + out.append_multiple_char(fill_char[0], spec.n_lpadding) if spec.n_sign: - out.append(spec.sign) + if self.is_unicode: + sign = spec.sign.decode("ascii") + else: + sign = spec.sign + out.append(sign) if spec.n_prefix: pref = num[to_prefix:to_prefix + spec.n_prefix] if upper: pref = self._upcase_string(pref) out.append(pref) if spec.n_spadding: - out.append_multiple_char(fill_char, spec.n_spadding) + out.append_multiple_char(fill_char[0], spec.n_spadding) if spec.n_digits != 0: if self._loc_thousands: digits = self._grouped_digits else: - digits = num[to_digits:to_digits + spec.n_digits] + stop = to_digits + spec.n_digits + assert stop >= 0 + digits = num[to_digits:stop] if upper: digits = self._upcase_string(digits) out.append(digits) if spec.n_decimal: - out.append(".") + out.append(self._lit(".")[0]) if spec.n_remainder: out.append(num[to_remainder:]) if spec.n_rpadding: - out.append_multiple_char(fill_char, spec.n_rpadding) + out.append_multiple_char(fill_char[0], spec.n_rpadding) return self.space.wrap(out.build()) def _format_int_or_long(self, w_num, kind): @@ -646,9 +657,9 @@ self._get_locale(tp) spec = self._calc_num_width(n_prefix, sign_char, to_numeric, n_digits, n_remainder, False, result) - fill = " " if self._fill_char == "\0" else self._fill_char + fill = self._lit(" ") if self._fill_char == "\0" else self._fill_char upper = self._type == "X" - return self._fill_number(spec, result, to_numeric, n_digits, to_prefix, + return self._fill_number(spec, result, to_numeric, to_prefix, fill, to_remainder, upper) def _long_to_base(self, base, value): @@ -701,6 +712,7 @@ if negative: i -= 1 buf[i] = "-" + assert i >= 0 return self.empty.join(buf[i:]) def format_int_or_long(self, w_num, kind): @@ -763,7 +775,11 @@ add_pct = False if self._precision == -1: self._precision = default_precision - result, special = rarithmetic.double_to_string(value, tp, + if self.is_unicode: + to_string_tp = tp.encode("ascii") + else: + to_string_tp = tp + result, special = rarithmetic.double_to_string(value, to_string_tp, self._precision, flags) if add_pct: result += "%" @@ -778,10 +794,14 @@ have_dec_point, to_remainder = self._parse_number(result, to_number) n_remainder = len(result) - to_remainder self._get_locale(tp) + if self.is_unicode: + digits = result.decode("ascii") + else: + digits = result spec = self._calc_num_width(0, sign, to_number, n_digits, - n_remainder, have_dec_point, result) - fill = " " if self._fill_char == "\0" else self._fill_char - return self._fill_number(spec, result, to_number, None, 0, fill, + n_remainder, have_dec_point, digits) + fill = self._lit(" ") if self._fill_char == "\0" else self._fill_char + return self._fill_number(spec, digits, to_number, 0, fill, to_remainder, False) def format_float(self, w_float): From benjamin at codespeak.net Sun Jul 11 23:59:28 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Jul 2010 23:59:28 +0200 (CEST) Subject: [pypy-svn] r76114 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20100711215928.CB4CB282B9C@codespeak.net> Author: benjamin Date: Sun Jul 11 23:59:27 2010 New Revision: 76114 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py Log: presentation type should always be a char Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Sun Jul 11 23:59:27 2010 @@ -306,7 +306,7 @@ self._sign = "\0" self._thousands_sep = False self._precision = -1 - self._type = default_type + presentation_type = default_type spec = self.spec if not spec: return True @@ -348,8 +348,17 @@ raise OperationError(space.w_ValueError, space.wrap("invalid format spec")) if length - i == 1: - self._type = spec[i] + presentation_type = spec[i] i += 1 + if self.is_unicode: + try: + the_type = presentation_type.encode("ascii") + except UnicodeEncodeError: + raise OperationError(space.w_ValueError, + space.wrap("invalid type")) + else: + the_type = presentation_type + self._type = the_type if self._thousands_sep: tp = self._type if (tp == "d" or @@ -408,11 +417,7 @@ def _unknown_presentation(self, tp): msg = "unknown presentation for %s: '%s'" - if self.is_unicode: - inval_type = self._type.encode("ascii") - else: - inval_type = self._type - w_msg = self.space.wrap(msg % (tp, inval_type)) + w_msg = self.space.wrap(msg % (tp, self._type)) raise OperationError(self.space.w_ValueError, w_msg) def format_string(self, string): @@ -775,11 +780,7 @@ add_pct = False if self._precision == -1: self._precision = default_precision - if self.is_unicode: - to_string_tp = tp.encode("ascii") - else: - to_string_tp = tp - result, special = rarithmetic.double_to_string(value, to_string_tp, + result, special = rarithmetic.double_to_string(value, tp, self._precision, flags) if add_pct: result += "%" Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_newformat.py Sun Jul 11 23:59:27 2010 @@ -95,6 +95,10 @@ assert format(self.s("h"), "c<3") == self.s("hcc") raises(ValueError, format, self.s("blah"), "=12") + def test_non_ascii_presentation(self): + raises(ValueError, format, self.s(""), "\x234") + + class AppTestUnicodeFormat(BaseStringFormatTests): @@ -116,6 +120,7 @@ raises(KeyError, self.s("{\u1000}").format) + class AppTestStringFormat(BaseStringFormatTests): def setup_class(cls): From benjamin at codespeak.net Mon Jul 12 00:18:29 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 00:18:29 +0200 (CEST) Subject: [pypy-svn] r76115 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100711221829.A3691282B9C@codespeak.net> Author: benjamin Date: Mon Jul 12 00:18:28 2010 New Revision: 76115 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py Log: explicitly convert to list Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Mon Jul 12 00:18:28 2010 @@ -514,7 +514,7 @@ def _fill_digits(self, buf, digits, d_state, n_chars, n_zeros, thousands_sep): if thousands_sep: - buf.extend(thousands_sep) + buf.extend(list(thousands_sep)) for i in range(d_state - 1, d_state - n_chars - 1, -1): buf.append(digits[i]) for i in range(n_zeros): From benjamin at codespeak.net Mon Jul 12 00:18:52 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 00:18:52 +0200 (CEST) Subject: [pypy-svn] r76116 - pypy/branch/fast-forward/pypy/annotation Message-ID: <20100711221852.CE5F6282B9C@codespeak.net> Author: benjamin Date: Mon Jul 12 00:18:51 2010 New Revision: 76116 Modified: pypy/branch/fast-forward/pypy/annotation/model.py Log: be more specific Modified: pypy/branch/fast-forward/pypy/annotation/model.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/model.py (original) +++ pypy/branch/fast-forward/pypy/annotation/model.py Mon Jul 12 00:18:51 2010 @@ -164,7 +164,8 @@ def __eq__(self, other): # NaN unpleasantness. - if (self.is_constant() and other.is_constant() and + if (type(self) is SomeFloat and type(other) is SomeFloat and + self.is_constant() and other.is_constant() and isnan(self.const) and isnan(other.const)): return True return super(SomeFloat, self).__eq__(other) From benjamin at codespeak.net Mon Jul 12 00:44:52 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 00:44:52 +0200 (CEST) Subject: [pypy-svn] r76117 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100711224452.5C6CE282B9C@codespeak.net> Author: benjamin Date: Mon Jul 12 00:44:50 2010 New Revision: 76117 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py Log: fix annotation here Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Mon Jul 12 00:44:50 2010 @@ -306,7 +306,7 @@ self._sign = "\0" self._thousands_sep = False self._precision = -1 - presentation_type = default_type + the_type = default_type spec = self.spec if not spec: return True @@ -349,15 +349,15 @@ space.wrap("invalid format spec")) if length - i == 1: presentation_type = spec[i] + if self.is_unicode: + try: + the_type = spec[i].encode("ascii") + except UnicodeEncodeError: + raise OperationError(space.w_ValueError, + space.wrap("invalid presentation type")) + else: + the_type = presentation_type i += 1 - if self.is_unicode: - try: - the_type = presentation_type.encode("ascii") - except UnicodeEncodeError: - raise OperationError(space.w_ValueError, - space.wrap("invalid type")) - else: - the_type = presentation_type self._type = the_type if self._thousands_sep: tp = self._type From benjamin at codespeak.net Mon Jul 12 01:09:18 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 01:09:18 +0200 (CEST) Subject: [pypy-svn] r76118 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100711230918.726D0282B9C@codespeak.net> Author: benjamin Date: Mon Jul 12 01:09:14 2010 New Revision: 76118 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py Log: no unicode(integer) in rpython Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Mon Jul 12 01:09:14 2010 @@ -682,7 +682,10 @@ def _int_to_base(self, base, value): if base == 10: - return unicode(value) if self.is_unicode else str(value) + s = str(value) + if self.is_unicode: + return s.decode("ascii") + return s # This part is slow. negative = value < 0 value = abs(value) From benjamin at codespeak.net Mon Jul 12 01:11:10 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 01:11:10 +0200 (CEST) Subject: [pypy-svn] r76119 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100711231110.1E8D8282B9C@codespeak.net> Author: benjamin Date: Mon Jul 12 01:11:09 2010 New Revision: 76119 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py Log: desugar max() Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Mon Jul 12 01:11:09 2010 @@ -541,7 +541,7 @@ previous = group else: group = previous - final_grouping = min(group, max(left, min_width, 1)) + final_grouping = min(group, max(left, max(min_width, 1))) n_zeros = max(0, final_grouping - left) n_chars = max(0, min(left, final_grouping)) ts = self._loc_thousands if need_separator else None From benjamin at codespeak.net Mon Jul 12 01:24:13 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 01:24:13 +0200 (CEST) Subject: [pypy-svn] r76120 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100711232413.CD932282B9C@codespeak.net> Author: benjamin Date: Mon Jul 12 01:24:12 2010 New Revision: 76120 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py Log: list doesn't work on strings either Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Mon Jul 12 01:24:12 2010 @@ -514,7 +514,8 @@ def _fill_digits(self, buf, digits, d_state, n_chars, n_zeros, thousands_sep): if thousands_sep: - buf.extend(list(thousands_sep)) + for c in thousands_sep: + buf.append(c) for i in range(d_state - 1, d_state - n_chars - 1, -1): buf.append(digits[i]) for i in range(n_zeros): From benjamin at codespeak.net Mon Jul 12 01:38:31 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 01:38:31 +0200 (CEST) Subject: [pypy-svn] r76121 - pypy/branch/fast-forward/pypy/interpreter/astcompiler Message-ID: <20100711233831.54166282BDE@codespeak.net> Author: benjamin Date: Mon Jul 12 01:38:28 2010 New Revision: 76121 Modified: pypy/branch/fast-forward/pypy/interpreter/astcompiler/symtable.py Log: can't iterate over tuples Modified: pypy/branch/fast-forward/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/astcompiler/symtable.py Mon Jul 12 01:38:28 2010 @@ -441,7 +441,7 @@ if outer.ifs: self.visit_sequence(outer.ifs) self.visit_sequence(comps[1:]) - for item in consider: + for item in list(consider): item.walkabout(self) self.pop_scope() From benjamin at codespeak.net Mon Jul 12 02:12:11 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 02:12:11 +0200 (CEST) Subject: [pypy-svn] r76122 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100712001211.5A394282B9C@codespeak.net> Author: benjamin Date: Mon Jul 12 02:11:56 2010 New Revision: 76122 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py Log: stubs Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Mon Jul 12 02:11:56 2010 @@ -265,7 +265,13 @@ pass class BaseFormatter(object): - pass + + def format_int_or_long(self, w_num, kind): + raise NotImplementedError + + def format_float(self, w_num): + raise NotImplementedError + INT_KIND = 1 LONG_KIND = 2 From benjamin at codespeak.net Mon Jul 12 05:05:31 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 05:05:31 +0200 (CEST) Subject: [pypy-svn] r76123 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100712030531.6670E282B9C@codespeak.net> Author: benjamin Date: Mon Jul 12 05:05:29 2010 New Revision: 76123 Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py pypy/branch/fast-forward/pypy/objspace/std/intobject.py pypy/branch/fast-forward/pypy/objspace/std/longobject.py pypy/branch/fast-forward/pypy/objspace/std/newformat.py Log: finally hack this method lookup into submission Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Mon Jul 12 05:05:29 2010 @@ -158,7 +158,7 @@ return float2string(space, w_float, "%.12g") def format__Float_ANY(space, w_float, w_spec): - return newformat.get_formatter(space, w_spec).format_float(w_float) + return newformat.run_formatter(space, w_spec, "format_float", w_float) # ____________________________________________________________ # A mess to handle all cases of float comparison without relying Modified: pypy/branch/fast-forward/pypy/objspace/std/intobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/intobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/intobject.py Mon Jul 12 05:05:29 2010 @@ -62,8 +62,8 @@ str__Int = repr__Int def format__Int_ANY(space, w_int, w_format_spec): - form = newformat.get_formatter(space, w_format_spec) - return form.format_int_or_long(w_int, newformat.INT_KIND) + return newformat.run_formatter(space, w_format_spec, "format_int_or_long", + w_int, newformat.INT_KIND) def declare_new_int_comparison(opname): import operator Modified: pypy/branch/fast-forward/pypy/objspace/std/longobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/longobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/longobject.py Mon Jul 12 05:05:29 2010 @@ -126,8 +126,8 @@ return space.wrap(w_long.num.str()) def format__Long_ANY(space, w_long, w_format_spec): - formatter = newformat.get_formatter(space, w_format_spec) - return formatter.format_int_or_long(w_long, newformat.LONG_KIND) + return newformat.run_formatter(space, w_format_spec, "format_int_or_long", + w_long, newformat.LONG_KIND) def lt__Long_Long(space, w_long1, w_long2): Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Mon Jul 12 05:05:29 2010 @@ -843,8 +843,11 @@ return Formatter(space, False, spec) -def get_formatter(space, w_format_spec): + at specialize.arg(2) +def run_formatter(space, w_format_spec, meth, *args): if space.isinstance_w(w_format_spec, space.w_unicode): - return unicode_formatter(space, space.unicode_w(w_format_spec)) + formatter = unicode_formatter(space, space.unicode_w(w_format_spec)) + return getattr(formatter, meth)(*args) else: - return str_formatter(space, space.str_w(w_format_spec)) + formatter = str_formatter(space, space.str_w(w_format_spec)) + return getattr(formatter, meth)(*args) From benjamin at codespeak.net Mon Jul 12 05:40:29 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 05:40:29 +0200 (CEST) Subject: [pypy-svn] r76124 - in pypy/branch/fast-forward/pypy/rpython: . lltypesystem ootypesystem test Message-ID: <20100712034029.D3A84282B9C@codespeak.net> Author: benjamin Date: Mon Jul 12 05:40:28 2010 New Revision: 76124 Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/rstr.py pypy/branch/fast-forward/pypy/rpython/ootypesystem/rstr.py pypy/branch/fast-forward/pypy/rpython/rstr.py pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py Log: allow chars to be joined into a different type Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/rstr.py Mon Jul 12 05:40:28 2010 @@ -675,19 +675,21 @@ i += 1 return result - def ll_join_chars(length, chars): + def ll_join_chars(length, chars, RES): # no need to optimize this, will be replaced by string builder # at some point soon num_chars = length - if typeOf(chars).TO.OF == Char: + if RES is StringRepr.lowleveltype: + target = Char malloc = mallocstr else: + target = UniChar malloc = mallocunicode result = malloc(num_chars) res_chars = result.chars i = 0 while i < num_chars: - res_chars[i] = chars[i] + res_chars[i] = cast_primitive(target, chars[i]) i += 1 return result Modified: pypy/branch/fast-forward/pypy/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/fast-forward/pypy/rpython/ootypesystem/rstr.py Mon Jul 12 05:40:28 2010 @@ -169,16 +169,18 @@ buf.ll_append(lastitem) return buf.ll_build() - def ll_join_chars(length_dummy, lst): - if typeOf(lst).ITEM == Char: + def ll_join_chars(length_dummy, lst, RES): + if RES is ootype.String: + target = Char buf = ootype.new(ootype.StringBuilder) else: + target = UniChar buf = ootype.new(ootype.UnicodeBuilder) length = lst.ll_length() buf.ll_allocate(length) i = 0 while i < length: - buf.ll_append_char(lst.ll_getitem_fast(i)) + buf.ll_append_char(cast_primitive(target, lst.ll_getitem_fast(i))) i += 1 return buf.ll_build() Modified: pypy/branch/fast-forward/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/rstr.py (original) +++ pypy/branch/fast-forward/pypy/rpython/rstr.py Mon Jul 12 05:40:28 2010 @@ -190,8 +190,11 @@ if hop.args_s[0].is_constant() and hop.args_s[0].const == '': if r_lst.item_repr == rstr.repr: llfn = self.ll.ll_join_strs - elif r_lst.item_repr == rstr.char_repr: - llfn = self.ll.ll_join_chars + elif (r_lst.item_repr == hop.rtyper.type_system.rstr.char_repr or + r_lst.item_repr == hop.rtyper.type_system.rstr.unichar_repr): + v_tp = hop.inputconst(Void, self.lowleveltype) + return hop.gendirectcall(self.ll.ll_join_chars, v_length, + v_items, v_tp) else: raise TyperError("''.join() of non-string list: %r" % r_lst) return hop.gendirectcall(llfn, v_length, v_items) Modified: pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py (original) +++ pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py Mon Jul 12 05:40:28 2010 @@ -405,6 +405,13 @@ res = self.interpret(fn, [i,j]) assert self.ll_to_string(res) == fn(i, j) + def fn(i): + c = ["a", "b", "c"] + assert i >= 0 + return const('').join(c[i:]) + res = self.interpret(fn, [0]) + assert self.ll_to_string(res) == const("abc") + def test_str_slice(self): const = self.const def fn(n): From hakanardo at codespeak.net Mon Jul 12 08:01:58 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 12 Jul 2010 08:01:58 +0200 (CEST) Subject: [pypy-svn] r76125 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100712060158.468A5282BAD@codespeak.net> Author: hakanardo Date: Mon Jul 12 08:01:56 2010 New Revision: 76125 Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: list protocol Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/app_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/app_array.py Mon Jul 12 08:01:56 2010 @@ -170,3 +170,10 @@ else: return self.tolist() >= other.tolist() + ##### list protocol + + def __getslice__(self, i, j): + return self.__getitem__(slice(i, j)) + + def __setslice__(self, i, j, x): + self.__setitem__(slice(i, j), x) Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Jul 12 08:01:56 2010 @@ -126,6 +126,7 @@ for i in range(min(size,self.len)): new_buffer[i] = self.buffer[i] else: + assert size == 0 new_buffer = lltype.nullptr(mytype.arraytype) if self.buffer != lltype.nullptr(mytype.arraytype): lltype.free(self.buffer, flavor='raw') @@ -138,22 +139,36 @@ descr_len.unwrap_spec = ['self'] - def descr_getslice(self, start, stop, step): - size = (stop - start) / step - if (stop - start) % step > 0: size += 1 - w_a=mytype.w_class(self.space) - w_a.setlen(size) - j=0 - for i in range(start, stop, step): - w_a.buffer[j]=self.buffer[i] - j+=1 + def descr_getslice(self, w_idx): + space = self.space + start, stop, step = space.decode_index(w_idx, self.len) + if step < 0: + w_lst = space.call_function( + space.getattr(self, space.wrap('tolist'))) + w_lst = space.call_function( + space.getattr(w_lst, space.wrap('__getitem__')), + w_idx) + w_a=mytype.w_class(self.space) + w_a.descr_fromsequence(w_lst) + else: + size = (stop - start) / step + if (stop - start) % step > 0: size += 1 + print size + if size < 0: size = 0 + w_a=mytype.w_class(self.space) + w_a.setlen(size) + j=0 + for i in range(start, stop, step): + w_a.buffer[j]=self.buffer[i] + j+=1 return w_a def descr_getitem(self, w_idx): space=self.space start, stop, step = space.decode_index(w_idx, self.len) - if step==0: - item = self.buffer[start] + if step == 0: + idx = start + item = self.buffer[idx] tc=mytype.typecode if tc == 'b' or tc == 'B' or tc == 'h' or tc == 'H' or tc == 'i' or tc == 'l': item = rffi.cast(lltype.Signed, item) @@ -161,7 +176,7 @@ item = float(item) return self.space.wrap(item) else: - return self.descr_getslice(start, stop, step) + return self.descr_getslice(w_idx) descr_getitem.unwrap_spec = ['self', W_Root] @@ -208,30 +223,39 @@ self.descr_append(w_item) descr_extend.unwrap_spec = ['self', W_Root] - def descr_setslice(self, start, stop, step, w_item): + def descr_setslice(self, w_idx, w_item): + space=self.space + start, stop, step = self.space.decode_index(w_idx, self.len) if isinstance(w_item, W_Array): # Implies mytype.typecode == w_item.typecode size = (stop - start) / step if (stop - start) % step > 0: size += 1 - if w_item.len != size: # FIXME: Support for step=1 - msg = ('attempt to assign array of size %d to ' + - 'slice of size %d') % (w_item.len, size) - raise OperationError(self.space.w_ValueError, - self.space.wrap(msg)) - j=0 - for i in range(start, stop, step): - self.buffer[i]=w_item.buffer[j] - j+=1 + if w_item.len != size or step < 0: + w_lst = space.call_function( + space.getattr(self, space.wrap('tolist'))) + w_item = space.call_function( + space.getattr(w_item, space.wrap('tolist'))) + space.call_function( + space.getattr(w_lst, space.wrap('__setitem__')), + w_idx, w_item) + self.setlen(0) + self.descr_fromsequence(w_lst) + else: + j=0 + for i in range(start, stop, step): + self.buffer[i]=w_item.buffer[j] + j+=1 return msg='can only assign array to array slice' raise OperationError(self.space.w_TypeError, self.space.wrap(msg)) def descr_setitem(self, w_idx, w_item): start, stop, step = self.space.decode_index(w_idx, self.len) - if step==0: + if step == 0: + idx = start item = self.item_w(w_item) - self.buffer[start] = item + self.buffer[idx] = item else: - self.descr_setslice(start, stop, step, w_item) + self.descr_setslice(w_idx, w_item) descr_setitem.unwrap_spec = ['self', W_Root, W_Root] def charbuf(self): @@ -310,7 +334,7 @@ def descr_copy(self): - w_a=mytype.w_class(self.space) + w_a = mytype.w_class(self.space) w_a.descr_fromsequence(self) return w_a @@ -333,6 +357,44 @@ for i in range(mytype.bytes): bytes[stop - i] = tmp[i] + def descr_add(self, w_other): + other = self.space.interp_w(W_Array, w_other) + w_a = mytype.w_class(self.space) + w_a.setlen(self.len + other.len) + for i in range(self.len): + w_a.buffer[i] = self.buffer[i] + for i in range(other.len): + w_a.buffer[i + self.len] = other.buffer[i] + return w_a + descr_add.unwrap_spec = ['self', W_Root] + + def descr_iadd(self, w_other): + other = self.space.interp_w(W_Array, w_other) + oldlen = self.len + otherlen = other.len + self.setlen(oldlen + otherlen) + for i in range(otherlen): + self.buffer[oldlen + i] = other.buffer[i] + return self + descr_add.unwrap_spec = ['self', W_Root] + + def descr_mul(self, repeat): + w_a = mytype.w_class(self.space) + w_a.setlen(self.len * repeat) + for r in range(repeat): + for i in range(self.len): + w_a.buffer[r * self.len + i] = self.buffer[i] + return w_a + descr_mul.unwrap_spec = ['self', int] + + def descr_imul(self, repeat): + oldlen=self.len + self.setlen(self.len * repeat) + for r in range(1,repeat): + for i in range(oldlen): + self.buffer[r * oldlen + i] = self.buffer[i] + return self + descr_imul.unwrap_spec = ['self', int] @@ -348,6 +410,8 @@ __len__ = interp2app(W_Array.descr_len), __getitem__ = interp2app(W_Array.descr_getitem), __setitem__ = interp2app(W_Array.descr_setitem), + __getslice__ = appmethod('__getslice__'), + __setslice__ = appmethod('__setslice__'), itemsize = GetSetProperty(descr_itemsize, cls=W_Array), typecode = GetSetProperty(descr_typecode, cls=W_Array), @@ -390,6 +454,12 @@ __reduce__ = interp2app(W_Array.descr_reduce), __copy__ = interp2app(W_Array.descr_copy), + __add__ = interp2app(W_Array.descr_add), + __iadd__ = interp2app(W_Array.descr_iadd), + __mul__ = interp2app(W_Array.descr_mul), + __rmul__ = interp2app(W_Array.descr_mul), + __imul__ = interp2app(W_Array.descr_imul), + buffer_info = interp2app(W_Array.descr_buffer_info), byteswap = interp2app(W_Array.descr_byteswap), Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Mon Jul 12 08:01:56 2010 @@ -289,6 +289,64 @@ except TypeError: pass + a=self.array('i', [1,2,3]) + assert a.__getslice__(1,2) == a[1:2] + a.__setslice__(1,2, self.array('i',(7,))) + assert a[0] == 1 and a[1] == 7 and a[2] == 3 + + def test_resizingslice(self): + a=self.array('i', [1, 2, 3]) + a[1:2] = self.array('i', [7, 8, 9]) + assert repr(a) == "array('i', [1, 7, 8, 9, 3])" + a[1:2] = self.array('i', [10]) + assert repr(a) == "array('i', [1, 10, 8, 9, 3])" + a[1:2] = self.array('i') + assert repr(a) == "array('i', [1, 8, 9, 3])" + + a[1:3] = self.array('i', [11, 12, 13]) + assert repr(a) == "array('i', [1, 11, 12, 13, 3])" + a[1:3] = self.array('i', [14]) + assert repr(a) == "array('i', [1, 14, 13, 3])" + a[1:3] = self.array('i') + assert repr(a) == "array('i', [1, 3])" + + a[1:1] = self.array('i', [15, 16, 17]) + assert repr(a) == "array('i', [1, 15, 16, 17, 3])" + a[1:1] = self.array('i', [18]) + assert repr(a) == "array('i', [1, 18, 15, 16, 17, 3])" + a[1:1] = self.array('i') + assert repr(a) == "array('i', [1, 18, 15, 16, 17, 3])" + + a[:] = self.array('i', [20, 21, 22]) + assert repr(a) == "array('i', [20, 21, 22])" + + def test_reversingslice(self): + a = self.array('i', [22, 21, 20]) + assert repr(a[::-1]) == "array('i', [20, 21, 22])" + assert repr(a[2:1:-1]) == "array('i', [20])" + assert repr(a[2:-1:-1]) == "array('i')" + assert repr(a[-1:0:-1]) == "array('i', [20, 21])" + + for a in range(-4,5): + for b in range(-4,5): + for c in [-4, -3, -2, -1, 1, 2, 3, 4]: + lst = [1, 2, 3] + arr=self.array('i', lst) + assert repr(arr[a:b:c]) == repr(self.array('i', lst[a:b:c])) + for vals in ([4,5], [6], []): + try: + ok = False + lst[a:b:c]=vals + ok = True + arr[a:b:c]=self.array('i', vals) + assert repr(arr) == repr(self.array('i', lst)) + except ValueError: + assert not ok + raises(ValueError, + "arr[a:b:c]=self.array('i', vals)") + + + def test_toxxx(self): a = self.array('i', [1,2,3]) l = a.tolist() @@ -458,7 +516,30 @@ assert a[0] == 1 assert a[1] == 2 assert a[2] == 3 - + + def test_addmul(self): + a = self.array('i', [1, 2, 3]) + assert repr(a + a) == "array('i', [1, 2, 3, 1, 2, 3])" + assert 2 * a == a + a + assert a * 2 == a + a + b = self.array('i', [4, 5, 6, 7]) + assert repr(a + b) == "array('i', [1, 2, 3, 4, 5, 6, 7])" + assert repr(2 * self.array('i')) == "array('i')" + assert repr(self.array('i') + self.array('i')) == "array('i')" + + a = self.array('i', [1, 2]) + b = a + a += a + assert repr(b) == "array('i', [1, 2, 1, 2])" + b *= 3 + assert repr(a) == "array('i', [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2])" + assert a == b + a += self.array('i', (7,)) + assert repr(a) == "array('i', [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 7])" + + raises(TypeError, self.array('i').__add__, (2,)) + raises(TypeError, self.array('i').__add__, self.array('b')) + #FIXME #def test_type(self): From arigo at codespeak.net Mon Jul 12 10:31:04 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 10:31:04 +0200 (CEST) Subject: [pypy-svn] r76126 - in pypy/branch/kill-caninline/pypy/jit/metainterp: . test Message-ID: <20100712083104.6C3A6282B9C@codespeak.net> Author: arigo Date: Mon Jul 12 10:31:02 2010 New Revision: 76126 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py Log: Intermediate check-in: restore 'maybe_enter_from_start'. There is a bug somewhere, as shown by tests... Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py Mon Jul 12 10:31:02 2010 @@ -98,23 +98,18 @@ policy=StopAtXPolicy(opaque)) assert res == 1 - def get_interpreter(self, codes, always_inline=False): + def get_interpreter(self, codes): ADD = "0" JUMP_BACK = "1" CALL = "2" EXIT = "3" - if always_inline: - def can_inline(*args): - return True - else: - def can_inline(i, code): - code = hlstr(code) - return not JUMP_BACK in code + def getloc(i, code): + return 'code="%s", i=%d' % (code, i) jitdriver = JitDriver(greens = ['i', 'code'], reds = ['n'], - can_inline = can_inline) - + get_printable_location = getloc) + def interpret(codenum, n, i): code = codes[codenum] while i < len(code): @@ -162,31 +157,16 @@ assert self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE, inline=True) == 42 - self.check_loops(call_may_force = 1, call = 0) - - def test_inline_faulty_can_inline(self): - code = "021" - subcode = "301" - codes = [code, subcode] - - f = self.get_interpreter(codes, always_inline=True) - - try: - self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE, - inline=True) - except CannotInlineCanEnterJit: - pass - else: - py.test.fail("DID NOT RAISE") + # the call is fully inlined, because we jump to subcode[1], thus + # skipping completely the JUMP_BACK in subcode[0] + self.check_loops(call_may_force = 0, call_assembler = 0, call = 0) def test_guard_failure_in_inlined_function(self): def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 while pc < len(code): @@ -219,10 +199,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n', 'flag'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 flag = False @@ -262,10 +240,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) class Exc(Exception): pass @@ -307,10 +283,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 @@ -523,11 +497,9 @@ def test_trace_from_start(self): def p(pc, code): code = hlstr(code) - return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) + return "'%s' at %d: %s" % (code, pc, code[pc]) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 @@ -537,9 +509,9 @@ op = code[pc] if op == "+": n += 7 - if op == "-": + elif op == "-": n -= 1 - if op == "c": + elif op == "c": n = f('---', n) elif op == "l": if n > 0: @@ -556,7 +528,10 @@ result = 0 for i in range(m): result += f('+-cl--', i) + g(50) self.meta_interp(g, [50], backendopt=True) + py.test.skip("tracing from start is by now only longer enabled " + "if a trace gets too big") self.check_tree_loop_count(3) self.check_history(int_add=1) @@ -564,10 +539,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 @@ -606,8 +579,7 @@ def test_directly_call_assembler(self): driver = JitDriver(greens = ['codeno'], reds = ['i'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno): i = 0 @@ -623,28 +595,29 @@ def test_recursion_cant_call_assembler_directly(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'j'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno, j): i = 0 - while i < 1: - driver.can_enter_jit(codeno=codeno, i=i, j=j) + while 1: driver.jit_merge_point(codeno=codeno, i=i, j=j) - i += 1 - if j == 0: + if i == 1: + if j == 0: + return + portal(2, j - 1) + elif i == 3: return - portal(2, j - 1) + i += 1 + driver.can_enter_jit(codeno=codeno, i=i, j=j) portal(2, 50) self.meta_interp(portal, [2, 20], inline=True) - self.check_history(call_assembler=0, call_may_force=1) - self.check_enter_count_at_most(1) + self.check_loops(call_assembler=0, call_may_force=1, + everywhere=True) def test_directly_call_assembler_return(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno): i = 0 @@ -667,8 +640,7 @@ self.x = x driver = JitDriver(greens = ['codeno'], reds = ['i'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno): i = 0 @@ -689,8 +661,7 @@ def test_directly_call_assembler_fail_guard(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno, k): i = 0 @@ -721,8 +692,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def main(codeno): frame = Frame() @@ -760,8 +730,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) @dont_look_inside def check_frame(subframe): @@ -811,8 +780,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) class SomewhereElse(object): pass @@ -851,8 +819,7 @@ def test_directly_call_assembler_virtualizable_with_array(self): myjitdriver = JitDriver(greens = ['codeno'], reds = ['n', 'x', 'frame'], - virtualizables = ['frame'], - can_inline = lambda codeno : False) + virtualizables = ['frame']) class Frame(object): _virtualizable2_ = ['l[*]', 's'] @@ -898,8 +865,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) class SomewhereElse(object): pass @@ -941,8 +907,7 @@ def test_assembler_call_red_args(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def residual(k): if k > 40: Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py Mon Jul 12 10:31:02 2010 @@ -396,6 +396,14 @@ maybe_enter_jit._always_inline_ = True jd._maybe_enter_jit_fn = maybe_enter_jit + can_inline = state.can_inline_greenargs + num_green_args = jd.num_green_args + def maybe_enter_from_start(*args): + if not can_inline(*args[:num_green_args]): + maybe_compile_and_run(*args) + maybe_enter_from_start._always_inline_ = True + jd._maybe_enter_from_start_fn = maybe_enter_from_start + def make_driverhook_graphs(self): from pypy.rlib.jit import BaseJitCell bk = self.rtyper.annotator.bookkeeper @@ -569,6 +577,7 @@ def ll_portal_runner(*args): while 1: try: + jd._maybe_enter_from_start_fn(*args) return support.maybe_on_top_of_llinterp(rtyper, portal_ptr)(*args) except self.ContinueRunningNormally, e: Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py Mon Jul 12 10:31:02 2010 @@ -478,12 +478,15 @@ jit_getter = self.make_jitcell_getter() jit_getter_maybe = self.jit_getter_maybe - def can_inline_callable(greenkey): - greenargs = unwrap_greenkey(greenkey) + def can_inline_greenargs(*greenargs): cell = jit_getter_maybe(*greenargs) if cell is not None and cell.dont_trace_here: return False return True + def can_inline_callable(greenkey): + greenargs = unwrap_greenkey(greenkey) + return can_inline_greenargs(*greenargs) + self.can_inline_greenargs = can_inline_greenargs self.can_inline_callable = can_inline_callable def get_assembler_token(greenkey): From arigo at codespeak.net Mon Jul 12 10:33:14 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 10:33:14 +0200 (CEST) Subject: [pypy-svn] r76127 - pypy/branch/kill-caninline/pypy/jit/metainterp Message-ID: <20100712083314.A9748282B9C@codespeak.net> Author: arigo Date: Mon Jul 12 10:33:13 2010 New Revision: 76127 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py Log: Add an assert. Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py Mon Jul 12 10:33:13 2010 @@ -149,6 +149,7 @@ assert oldbox not in registers[count:] def make_result_of_lastop(self, resultbox): + assert resultbox is not None target_index = ord(self.bytecode[self.pc-1]) if resultbox.type == history.INT: self.registers_i[target_index] = resultbox From arigo at codespeak.net Mon Jul 12 11:54:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 11:54:21 +0200 (CEST) Subject: [pypy-svn] r76129 - in pypy/branch/kill-caninline/pypy/jit/metainterp: . test Message-ID: <20100712095421.5BD4C282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 11:54:19 2010 New Revision: 76129 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py Log: Testing code, and fix in pyjitpl.py (see comment). Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py Mon Jul 12 11:54:19 2010 @@ -2,7 +2,7 @@ from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import debug_start, debug_stop -from pypy.rlib.debug import make_sure_not_resized +from pypy.rlib.debug import make_sure_not_resized, fatalerror from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLException @@ -756,6 +756,10 @@ assert e reraise(e) + @arguments("r") + def bhimpl_debug_fatalerror(msg): + llop.debug_fatalerror(lltype.Void, msg) + # ---------- # the main hints and recursive calls Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py Mon Jul 12 11:54:19 2010 @@ -825,12 +825,21 @@ else: warmrunnerstate = jitdriver_sd.warmstate token = warmrunnerstate.get_assembler_token(greenboxes) - resbox = self.do_recursive_call(jitdriver_sd, - greenboxes + redboxes, - token) - # in case of exception, do_recursive_call() stops by raising - # the ChangeFrame exception already. - self.metainterp.finishframe(resbox) + # warning! careful here. We have to return from the current + # frame containing the jit_merge_point, and then use + # do_recursive_call() to follow the recursive call. This is + # needed because do_recursive_call() will write its result + # with make_result_of_lastop(), so the lastop must be right: + # it must be the call to 'self', and not the jit_merge_point + # itself, which has no lastop at all. + assert self.metainterp.framestack + try: + self.metainterp.finishframe(None) + except ChangeFrame: + pass + frame = self.metainterp.framestack[-1] + frame.do_recursive_call(jitdriver_sd, greenboxes + redboxes, token) + raise ChangeFrame def debug_merge_point(self, jitdriver_sd, greenkey): # debugging: produce a DEBUG_MERGE_POINT operation @@ -881,6 +890,12 @@ return exc_value_box @arguments("box") + def opimpl_debug_fatalerror(self, box): + from pypy.rpython.lltypesystem import rstr, lloperation + msg = box.getref(lltype.Ptr(rstr.STR)) + lloperation.llop.debug_fatalerror(msg) + + @arguments("box") def opimpl_virtual_ref(self, box): # Details on the content of metainterp.virtualref_boxes: # Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py Mon Jul 12 11:54:19 2010 @@ -2,6 +2,7 @@ from pypy.rlib.jit import JitDriver, we_are_jitted, OPTIMIZER_SIMPLE, hint from pypy.rlib.jit import unroll_safe, dont_look_inside from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import fatalerror from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.rpython.annlowlevel import hlstr @@ -770,7 +771,7 @@ res = self.meta_interp(main, [0], inline=True) assert res == main(0) - def test_directly_call_assembler_virtualizable_force(self): + def test_directly_call_assembler_virtualizable_force1(self): class Thing(object): def __init__(self, val): self.val = val @@ -797,6 +798,7 @@ return frame.thing.val def portal(codeno, frame): + print 'ENTER:', codeno, frame.thing.val i = 0 while i < 10: driver.can_enter_jit(frame=frame, codeno=codeno, i=i) @@ -806,11 +808,15 @@ subframe = Frame() subframe.thing = Thing(nextval) nextval = portal(1, subframe) - elif frame.thing.val > 40: - change(Thing(13)) - nextval = 13 + elif codeno == 1: + if frame.thing.val > 40: + change(Thing(13)) + nextval = 13 + else: + fatalerror("bad codeno = " + str(codeno)) frame.thing = Thing(nextval + 1) i += 1 + print 'LEAVE:', codeno, frame.thing.val return frame.thing.val res = self.meta_interp(main, [0], inline=True, From arigo at codespeak.net Mon Jul 12 12:16:54 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 12:16:54 +0200 (CEST) Subject: [pypy-svn] r76130 - pypy/branch/kill-caninline/pypy/jit/metainterp Message-ID: <20100712101654.016B8282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 12:16:53 2010 New Revision: 76130 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py Log: No-op simplification. Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py Mon Jul 12 12:16:53 2010 @@ -475,7 +475,7 @@ return # unwrap_greenkey = self.make_unwrap_greenkey() - jit_getter = self.make_jitcell_getter() + self.make_jitcell_getter() jit_getter_maybe = self.jit_getter_maybe def can_inline_greenargs(*greenargs): @@ -491,8 +491,8 @@ def get_assembler_token(greenkey): greenargs = unwrap_greenkey(greenkey) - cell = jit_getter(*greenargs) - if cell.counter >= 0: + cell = jit_getter_maybe(*greenargs) + if cell is None or cell.counter >= 0: return None return cell.entry_loop_token self.get_assembler_token = get_assembler_token From arigo at codespeak.net Mon Jul 12 12:30:03 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 12:30:03 +0200 (CEST) Subject: [pypy-svn] r76131 - pypy/branch/kill-caninline/pypy/jit/metainterp Message-ID: <20100712103003.A0824282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 12:30:01 2010 New Revision: 76131 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py Log: Debug asserts that would have caught r76129. Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py Mon Jul 12 12:30:01 2010 @@ -149,15 +149,21 @@ assert oldbox not in registers[count:] def make_result_of_lastop(self, resultbox): - assert resultbox is not None + got_type = resultbox.type + if not we_are_translated(): + assert resultbox is not None + typeof = {'i': history.INT, + 'r': history.REF, + 'f': history.FLOAT} + assert typeof[self.jitcode._resulttypes[self.pc]] == got_type target_index = ord(self.bytecode[self.pc-1]) - if resultbox.type == history.INT: + if got_type == history.INT: self.registers_i[target_index] = resultbox - elif resultbox.type == history.REF: + elif got_type == history.REF: #debug_print(' ->', # llmemory.cast_ptr_to_adr(resultbox.getref_base())) self.registers_r[target_index] = resultbox - elif resultbox.type == history.FLOAT: + elif got_type == history.FLOAT: self.registers_f[target_index] = resultbox else: raise AssertionError("bad result box type") @@ -2260,6 +2266,8 @@ # if resultbox is not None: self.make_result_of_lastop(resultbox) + elif not we_are_translated(): + assert self._result_argcode == 'v' # unboundmethod = getattr(MIFrame, 'opimpl_' + name).im_func argtypes = unrolling_iterable(unboundmethod.argtypes) From hakanardo at codespeak.net Mon Jul 12 13:19:31 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 12 Jul 2010 13:19:31 +0200 (CEST) Subject: [pypy-svn] r76132 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100712111931.EF9A6282BAD@codespeak.net> Author: hakanardo Date: Mon Jul 12 13:19:24 2010 New Revision: 76132 Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: delitem, iter and contains Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/app_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/app_array.py Mon Jul 12 13:19:24 2010 @@ -177,3 +177,13 @@ def __setslice__(self, i, j, x): self.__setitem__(slice(i, j), x) + + def __delslice__(self, i, j): + self.__delitem__(slice(i, j)) + + def __contains__(self, item): + for x in self: + if x == item: + return True + return False + Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Jul 12 13:19:24 2010 @@ -74,6 +74,22 @@ unroll_typecodes = unrolling_iterable(types.keys()) def make_array(mytype): + class W_ArrayIter(Wrappable): + def __init__(self, a): + self.space = a.space + self.a = a + self.pos = 0 + + def iter_w(self): + return self.space.wrap(self) + + def next_w(self): + if self.pos >= self.a.len: + raise OperationError(self.space.w_StopIteration, self.space.w_None) + val = self.a.descr_getitem(self.space.wrap(self.pos)) + self.pos += 1 + return val + class W_Array(W_ArrayBase): itemsize=mytype.bytes typecode=mytype.typecode @@ -396,6 +412,19 @@ return self descr_imul.unwrap_spec = ['self', int] + def descr_delitem(self, w_idx): + space=self.space + w_lst = space.call_function( + space.getattr(self, space.wrap('tolist'))) + space.call_function( + space.getattr(w_lst, space.wrap('__delitem__')), + w_idx) + self.setlen(0) + self.descr_fromsequence(w_lst) + descr_delitem.unwrap_spec = ['self', W_Root] + + def descr_iter(self): + return W_ArrayIter(self) def descr_itemsize(space, self): @@ -403,6 +432,13 @@ def descr_typecode(space, self): return space.wrap(mytype.typecode) + W_ArrayIter.__name__ = 'W_ArrayIterType_'+mytype.typecode + W_ArrayIter.typedef = TypeDef( + 'ArrayIterType_'+mytype.typecode, + __iter__ = interp2app(W_ArrayIter.iter_w), + next = interp2app(W_ArrayIter.next_w), + ) + W_Array.__name__ = 'W_ArrayType_'+mytype.typecode W_Array.typedef = TypeDef( 'ArrayType_'+mytype.typecode, @@ -410,8 +446,10 @@ __len__ = interp2app(W_Array.descr_len), __getitem__ = interp2app(W_Array.descr_getitem), __setitem__ = interp2app(W_Array.descr_setitem), + __delitem__ = interp2app(W_Array.descr_delitem), __getslice__ = appmethod('__getslice__'), __setslice__ = appmethod('__setslice__'), + __delslice__ = appmethod('__delslice__'), itemsize = GetSetProperty(descr_itemsize, cls=W_Array), typecode = GetSetProperty(descr_typecode, cls=W_Array), @@ -461,8 +499,10 @@ __imul__ = interp2app(W_Array.descr_imul), buffer_info = interp2app(W_Array.descr_buffer_info), - byteswap = interp2app(W_Array.descr_byteswap), + + __iter__ = interp2app(W_Array.descr_iter), + __contains__ = appmethod('__contains__'), ) mytype.w_class = W_Array Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Mon Jul 12 13:19:24 2010 @@ -539,6 +539,26 @@ raises(TypeError, self.array('i').__add__, (2,)) raises(TypeError, self.array('i').__add__, self.array('b')) + + def test_delitem(self): + a = self.array('i', [1, 2, 3]) + del a[1] + assert repr(a) == "array('i', [1, 3])" + + a = self.array('i', [1, 2, 3, 4, 5]) + del a[1:3] + assert repr(a) == "array('i', [1, 4, 5])" + + a.__delslice__(0,2) + assert repr(a) == "array('i', [5])" + + def test_iter(self): + a = self.array('i', [1, 2, 3]) + assert 1 in a + b = self.array('i') + for i in a: + b.append(i) + assert repr(b) == "array('i', [1, 2, 3])" #FIXME From arigo at codespeak.net Mon Jul 12 13:21:02 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 13:21:02 +0200 (CEST) Subject: [pypy-svn] r76133 - pypy/branch/kill-caninline/pypy/jit/codewriter Message-ID: <20100712112102.84B00282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 13:20:55 2010 New Revision: 76133 Modified: pypy/branch/kill-caninline/pypy/jit/codewriter/assembler.py pypy/branch/kill-caninline/pypy/jit/codewriter/jitcode.py Log: Oups, this goes together with r76131. Modified: pypy/branch/kill-caninline/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/kill-caninline/pypy/jit/codewriter/assembler.py Mon Jul 12 13:20:55 2010 @@ -53,6 +53,7 @@ self.liveness = {} self.startpoints = set() self.alllabels = set() + self.resulttypes = {} def emit_reg(self, reg): if reg.index >= self.count_regs[reg.kind]: @@ -165,7 +166,9 @@ raise NotImplementedError(x) # opname = insn[0] - assert '>' not in argcodes or argcodes.index('>') == len(argcodes) - 2 + if '>' in argcodes: + assert argcodes.index('>') == len(argcodes) - 2 + self.resulttypes[len(self.code)] = argcodes[-1] key = opname + '/' + ''.join(argcodes) num = self.insns.setdefault(key, len(self.insns)) self.code[startposition] = chr(num) @@ -212,7 +215,8 @@ self.count_regs['float'], liveness=self.liveness, startpoints=self.startpoints, - alllabels=self.alllabels) + alllabels=self.alllabels, + resulttypes=self.resulttypes) def see_raw_object(self, value): if value._obj not in self._seen_raw_objects: Modified: pypy/branch/kill-caninline/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/kill-caninline/pypy/jit/codewriter/jitcode.py Mon Jul 12 13:20:55 2010 @@ -19,7 +19,8 @@ def setup(self, code='', constants_i=[], constants_r=[], constants_f=[], num_regs_i=255, num_regs_r=255, num_regs_f=255, - liveness=None, startpoints=None, alllabels=None): + liveness=None, startpoints=None, alllabels=None, + resulttypes=None): self.code = code # if the following lists are empty, use a single shared empty list self.constants_i = constants_i or self._empty_i @@ -33,6 +34,7 @@ self.liveness = make_liveness_cache(liveness) self._startpoints = startpoints # debugging self._alllabels = alllabels # debugging + self._resulttypes = resulttypes # debugging def get_fnaddr_as_int(self): return heaptracker.adr2int(self.fnaddr) From arigo at codespeak.net Mon Jul 12 13:24:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 13:24:24 +0200 (CEST) Subject: [pypy-svn] r76134 - in pypy/branch/kill-caninline/pypy/jit: codewriter codewriter/test metainterp metainterp/test Message-ID: <20100712112424.44555282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 13:24:17 2010 New Revision: 76134 Modified: pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py pypy/branch/kill-caninline/pypy/jit/codewriter/test/test_flatten.py pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_jitdriver.py pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py Log: Rename: a 'can_enter_jit' in the source graph becomes a 'loop_header' operation in the transformed graph, as its only purpose in the transformed graph is to detect loops. The goal is later to add a jitdriver.loop_header() that has only the latter effect. Modified: pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py Mon Jul 12 13:24:17 2010 @@ -809,10 +809,13 @@ return ops + [op1, op2] def handle_jit_marker__can_enter_jit(self, op, jitdriver): + # a 'can_enter_jit' in the source graph becomes a 'loop_header' + # operation in the transformed graph, as its only purpose in + # the transformed graph is to detect loops. jd = self.callcontrol.jitdriver_sd_from_jitdriver(jitdriver) assert jd is not None c_index = Constant(jd.index, lltype.Signed) - return SpaceOperation('can_enter_jit', [c_index], None) + return SpaceOperation('loop_header', [c_index], None) def rewrite_op_debug_assert(self, op): log.WARNING("found debug_assert in %r; should have be removed" % Modified: pypy/branch/kill-caninline/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/kill-caninline/pypy/jit/codewriter/test/test_flatten.py Mon Jul 12 13:24:17 2010 @@ -593,7 +593,8 @@ -live- %i0, %i1 int_guard_value %i0 jit_merge_point $27, I[%i0], R[], F[], I[%i1], R[], F[] - can_enter_jit $27 + -live- + loop_header $27 void_return """, transform=True, liveness=True, cc=MyFakeCallControl(), jd=jd) Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py Mon Jul 12 13:24:17 2010 @@ -764,7 +764,7 @@ # the main hints and recursive calls @arguments("i") - def bhimpl_can_enter_jit(jdindex): + def bhimpl_loop_header(jdindex): pass @arguments("self", "i", "I", "R", "F", "I", "R", "F") Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py Mon Jul 12 13:24:17 2010 @@ -795,8 +795,8 @@ return clsbox @arguments("int") - def opimpl_can_enter_jit(self, jdindex): - self.metainterp.seen_can_enter_jit_for_jdindex = jdindex + def opimpl_loop_header(self, jdindex): + self.metainterp.seen_loop_header_for_jdindex = jdindex def verify_green_args(self, jitdriver_sd, varargs): num_green_args = jitdriver_sd.num_green_args @@ -810,23 +810,23 @@ self.verify_green_args(jitdriver_sd, greenboxes) # xxx we may disable the following line in some context later self.debug_merge_point(jitdriver_sd, greenboxes) - if self.metainterp.seen_can_enter_jit_for_jdindex < 0: + if self.metainterp.seen_loop_header_for_jdindex < 0: return # - assert self.metainterp.seen_can_enter_jit_for_jdindex == jdindex, ( - "found a can_enter_jit for a JitDriver that does not match " + assert self.metainterp.seen_loop_header_for_jdindex == jdindex, ( + "found a loop_header for a JitDriver that does not match " "the following jit_merge_point's") - self.metainterp.seen_can_enter_jit_for_jdindex = -1 + self.metainterp.seen_loop_header_for_jdindex = -1 # if not self.metainterp.in_recursion: assert jitdriver_sd is self.metainterp.jitdriver_sd # Set self.pc to point to jit_merge_point instead of just after: - # if reached_can_enter_jit() raises SwitchToBlackhole, then the + # if reached_loop_header() raises SwitchToBlackhole, then the # pc is still at the jit_merge_point, which is a point that is # much less expensive to blackhole out of. saved_pc = self.pc self.pc = orgpc - self.metainterp.reached_can_enter_jit(greenboxes, redboxes) + self.metainterp.reached_loop_header(greenboxes, redboxes) self.pc = saved_pc else: warmrunnerstate = jitdriver_sd.warmstate @@ -1584,7 +1584,7 @@ redkey = original_boxes[num_green_args:] self.resumekey = compile.ResumeFromInterpDescr(original_greenkey, redkey) - self.seen_can_enter_jit_for_jdindex = -1 + self.seen_loop_header_for_jdindex = -1 try: self.interpret() except GenerateMergePoint, gmp: @@ -1611,7 +1611,7 @@ # because we cannot reconstruct the beginning of the proper loop self.current_merge_points = [(original_greenkey, -1)] self.resumekey = key - self.seen_can_enter_jit_for_jdindex = -1 + self.seen_loop_header_for_jdindex = -1 try: self.prepare_resume_from_failure(key.guard_opnum) self.interpret() @@ -1641,7 +1641,7 @@ else: duplicates[box] = None - def reached_can_enter_jit(self, greenboxes, redboxes): + def reached_loop_header(self, greenboxes, redboxes): duplicates = {} self.remove_consts_and_duplicates(redboxes, len(redboxes), duplicates) @@ -1655,7 +1655,7 @@ live_arg_boxes += self.virtualizable_boxes live_arg_boxes.pop() assert len(self.virtualref_boxes) == 0, "missing virtual_ref_finish()?" - # Called whenever we reach the 'can_enter_jit' hint. + # Called whenever we reach the 'loop_header' hint. # First, attempt to make a bridge: # - if self.resumekey is a ResumeGuardDescr, it starts from a guard # that failed; Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_jitdriver.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_jitdriver.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_jitdriver.py Mon Jul 12 13:24:17 2010 @@ -29,11 +29,14 @@ while r > 0: myjitdriver2.can_enter_jit(g=g, r=r) myjitdriver2.jit_merge_point(g=g, r=r) - r += loop1(r, g) - 1 + r += loop1(r, g) + (-1) return r # res = self.meta_interp(loop2, [4, 40], repeat=7, inline=True) assert res == loop2(4, 40) + # we expect only one int_sub, corresponding to the single + # compiled instance of loop1() + self.check_loops(int_sub=1) # the following numbers are not really expectations of the test # itself, but just the numbers that we got after looking carefully # at the generated machine code @@ -41,7 +44,7 @@ self.check_tree_loop_count(4) # 2 x loop, 2 x enter bridge self.check_enter_count(7) - def test_simple_inline(self): + def test_inline(self): # this is not an example of reasonable code: loop1() is unrolled # 'n/m' times, where n and m are given as red arguments. myjitdriver1 = JitDriver(greens=[], reds=['n', 'm'], Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py Mon Jul 12 13:24:17 2010 @@ -473,7 +473,12 @@ jitdriver = op.args[1].value assert jitdriver in sublists, \ "can_enter_jit with no matching jit_merge_point" - sublists[jitdriver].append((graph, block, index)) + origportalgraph = jd._jit_merge_point_pos[0] + if graph is not origportalgraph: + sublists[jitdriver].append((graph, block, index)) + else: + pass # a 'can_enter_jit' before the 'jit-merge_point', but + # originally in the same function: we ignore it here for jd in self.jitdrivers_sd: sublist = sublists[jd.jitdriver] assert len(sublist) > 0, \ @@ -682,17 +687,6 @@ origblock.exitswitch = None origblock.recloseblock(Link([v_result], origportalgraph.returnblock)) # - # Also kill any can_enter_jit left behind (example: see - # test_jitdriver.test_simple, which has a can_enter_jit in - # loop1's origportalgraph) - can_enter_jits = _find_jit_marker([origportalgraph], 'can_enter_jit') - for _, block, i in can_enter_jits: - op = block.operations[i] - assert op.opname == 'jit_marker' - block.operations[i] = SpaceOperation('same_as', - [Constant(None, lltype.Void)], - op.result) - # checkgraph(origportalgraph) def add_finish(self): From hakanardo at codespeak.net Mon Jul 12 13:38:18 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 12 Jul 2010 13:38:18 +0200 (CEST) Subject: [pypy-svn] r76135 - in pypy/branch/interplevel-array: lib_pypy pypy/module/array/test pypy/module/test_lib_pypy Message-ID: <20100712113818.DB1BF282BAD@codespeak.net> Author: hakanardo Date: Mon Jul 12 13:38:17 2010 New Revision: 76135 Added: pypy/branch/interplevel-array/pypy/module/array/test/test_array_old.py (contents, props changed) - copied, changed from r76055, pypy/branch/interplevel-array/pypy/module/test_lib_pypy/test_array.py Removed: pypy/branch/interplevel-array/lib_pypy/oldarray.py pypy/branch/interplevel-array/pypy/module/test_lib_pypy/test_array.py Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: reusing previous tests Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Mon Jul 12 13:38:17 2010 @@ -16,18 +16,7 @@ assert a[5] == 7.42 -class AppTestArray: - def setup_class(cls): - cls.space = gettestobjspace(usemodules=('array','struct')) - cls.w_array = cls.space.appexec([], """(): - import array - return array.array - """) - cls.w_unpack = cls.space.appexec([], """(): - import struct - return struct.unpack - """) - +class BaseArrayTests: def test_ctor(self): raises(TypeError, self.array, 'hi') raises(TypeError, self.array, 1) @@ -566,6 +555,26 @@ # for t in 'bBhHiIlLfdcu': # assert type(self.array(t)) is self.array +class TestCPythonsOwnArray(BaseArrayTests): + + def setup_class(cls): + import array + cls.array = array.array + import struct + cls.unpack = struct.unpack + + +class AppTestArray(BaseArrayTests): + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array', 'struct')) + cls.w_array = cls.space.appexec([], """(): + import array + return array.array + """) + cls.w_unpack = cls.space.appexec([], """(): + import struct + return struct.unpack + """) ## class AppTestAppArray(AppTestArray): ## def setup_class(cls): Copied: pypy/branch/interplevel-array/pypy/module/array/test/test_array_old.py (from r76055, pypy/branch/interplevel-array/pypy/module/test_lib_pypy/test_array.py) ============================================================================== --- pypy/branch/interplevel-array/pypy/module/test_lib_pypy/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array_old.py Mon Jul 12 13:38:17 2010 @@ -70,27 +70,27 @@ cls.array = array -class TestArrayOnTopOfCPython(BaseArrayTests): +## class TestArrayOnTopOfCPython(BaseArrayTests): - def setup_class(cls): - from pypy.tool.lib_pypy import LIB_PYPY - if not hasattr(struct, 'pack_into'): - py.test.skip("requires CPython >= 2.5") - import new - path = LIB_PYPY.join('array.py') - myarraymodule = new.module('array') - execfile(str(path), myarraymodule.__dict__) - cls.array = myarraymodule +## def setup_class(cls): +## from pypy.tool.lib_pypy import LIB_PYPY +## if not hasattr(struct, 'pack_into'): +## py.test.skip("requires CPython >= 2.5") +## import new +## path = LIB_PYPY.join('array.py') +## myarraymodule = new.module('array') +## execfile(str(path), myarraymodule.__dict__) +## cls.array = myarraymodule - def test_unicode(self): - py.test.skip("no 'u' type code in CPython's struct module") +## def test_unicode(self): +## py.test.skip("no 'u' type code in CPython's struct module") - def test_pickle(self): - py.test.skip("pickle getting confused by the hack in setup_class()") +## def test_pickle(self): +## py.test.skip("pickle getting confused by the hack in setup_class()") class AppTestArray(BaseArrayTests): - usemodules = ['struct'] + usemodules = ['struct', 'array'] def setup_class(cls): """ @@ -105,18 +105,18 @@ cls.w_native_sizes = cls.space.wrap(cls.native_sizes) -class AppTestArrayWithRawFFI(AppTestArray): - """ - The same as the base class, but with a space that also includes the - _rawffi module. The array module internally uses it in this case. - """ - usemodules = ['struct', '_rawffi'] - - def test_buffer_info(self): - a = self.array.array('l', [123, 456]) - assert a.itemsize == self.native_sizes['l'] - address, length = a.buffer_info() - assert length == 2 # and not 2 * self.native_sizes['l'] - assert address != 0 - # should check the address via some unsafe peeking, but it's - # not easy on top of py.py +## class AppTestArrayWithRawFFI(AppTestArray): +## """ +## The same as the base class, but with a space that also includes the +## _rawffi module. The array module internally uses it in this case. +## """ +## usemodules = ['struct', '_rawffi'] + +## def test_buffer_info(self): +## a = self.array.array('l', [123, 456]) +## assert a.itemsize == self.native_sizes['l'] +## address, length = a.buffer_info() +## assert length == 2 # and not 2 * self.native_sizes['l'] +## assert address != 0 +## # should check the address via some unsafe peeking, but it's +## # not easy on top of py.py From arigo at codespeak.net Mon Jul 12 14:12:39 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 14:12:39 +0200 (CEST) Subject: [pypy-svn] r76136 - pypy/branch/kill-caninline/pypy/jit/metainterp/test Message-ID: <20100712121239.20C87282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 14:12:38 2010 New Revision: 76136 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py Log: Fix the test by increasing some values. The exact reason is yet to be understood... Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py Mon Jul 12 14:12:38 2010 @@ -916,13 +916,13 @@ get_printable_location = lambda codeno : str(codeno)) def residual(k): - if k > 40: + if k > 150: return 0 return 1 def portal(codeno, k): i = 0 - while i < 10: + while i < 15: driver.can_enter_jit(codeno=codeno, i=i, k=k) driver.jit_merge_point(codeno=codeno, i=i, k=k) if codeno == 2: From arigo at codespeak.net Mon Jul 12 14:31:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 14:31:21 +0200 (CEST) Subject: [pypy-svn] r76138 - in pypy/branch/kill-caninline/pypy: jit/codewriter jit/metainterp jit/metainterp/test rlib Message-ID: <20100712123121.879A1282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 14:31:11 2010 New Revision: 76138 Modified: pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py pypy/branch/kill-caninline/pypy/rlib/jit.py Log: Write jitdriver.loop_header(), with a simple test. Works like can_enter_jit(), except it only has effect in the jitcode. In other words, it is used by pyjitpl to find closing loops or recursive loops, but it is not used by the non-jitted interpreter to perform statistics and start JITting. Modified: pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py Mon Jul 12 14:31:11 2010 @@ -808,15 +808,17 @@ # ^^^ we need a -live- for the case of do_recursive_call() return ops + [op1, op2] - def handle_jit_marker__can_enter_jit(self, op, jitdriver): - # a 'can_enter_jit' in the source graph becomes a 'loop_header' - # operation in the transformed graph, as its only purpose in - # the transformed graph is to detect loops. + def handle_jit_marker__loop_header(self, op, jitdriver): jd = self.callcontrol.jitdriver_sd_from_jitdriver(jitdriver) assert jd is not None c_index = Constant(jd.index, lltype.Signed) return SpaceOperation('loop_header', [c_index], None) + # a 'can_enter_jit' in the source graph becomes a 'loop_header' + # operation in the transformed graph, as its only purpose in + # the transformed graph is to detect loops. + handle_jit_marker__can_enter_jit = handle_jit_marker__loop_header + def rewrite_op_debug_assert(self, op): log.WARNING("found debug_assert in %r; should have be removed" % (self.graph,)) Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py Mon Jul 12 14:31:11 2010 @@ -837,8 +837,8 @@ # needed because do_recursive_call() will write its result # with make_result_of_lastop(), so the lastop must be right: # it must be the call to 'self', and not the jit_merge_point - # itself, which has no lastop at all. - assert self.metainterp.framestack + # itself, which has no result at all. + assert len(self.metainterp.framestack) >= 2 try: self.metainterp.finishframe(None) except ChangeFrame: @@ -2267,7 +2267,7 @@ if resultbox is not None: self.make_result_of_lastop(resultbox) elif not we_are_translated(): - assert self._result_argcode == 'v' + assert self._result_argcode in 'v?' # unboundmethod = getattr(MIFrame, 'opimpl_' + name).im_func argtypes = unrolling_iterable(unboundmethod.argtypes) Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_warmspot.py Mon Jul 12 14:31:11 2010 @@ -272,6 +272,30 @@ self.check_enter_count_at_most(2) self.check_loops(call=0) + def test_loop_header(self): + # artificial test: we enter into the JIT only when can_enter_jit() + # is seen, but we close a loop in the JIT much more quickly + # because of loop_header(). + mydriver = JitDriver(reds = ['n', 'm'], greens = []) + + def f(m): + n = 0 + while True: + mydriver.jit_merge_point(n=n, m=m) + if n > m: + m -= 1 + if m < 0: + return n + n = 0 + mydriver.can_enter_jit(n=n, m=m) + else: + n += 1 + mydriver.loop_header() + assert f(15) == 1 + res = self.meta_interp(f, [15], backendopt=True) + assert res == 1 + self.check_loops(int_add=1) # I get 13 without the loop_header() + class TestLLWarmspot(WarmspotTests, LLJitMixin): CPUClass = runner.LLtypeCPU Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py Mon Jul 12 14:31:11 2010 @@ -479,6 +479,7 @@ else: pass # a 'can_enter_jit' before the 'jit-merge_point', but # originally in the same function: we ignore it here + # see e.g. test_jitdriver.test_simple for jd in self.jitdrivers_sd: sublist = sublists[jd.jitdriver] assert len(sublist) > 0, \ Modified: pypy/branch/kill-caninline/pypy/rlib/jit.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rlib/jit.py (original) +++ pypy/branch/kill-caninline/pypy/rlib/jit.py Mon Jul 12 14:31:11 2010 @@ -282,6 +282,10 @@ # special-cased by ExtRegistryEntry assert dict.fromkeys(livevars) == _self._alllivevars + def loop_header(self): + # special-cased by ExtRegistryEntry + pass + def _set_param(self, name, value): # special-cased by ExtRegistryEntry # (internal, must receive a constant 'name') @@ -321,11 +325,15 @@ # specifically for them. self.jit_merge_point = self.jit_merge_point self.can_enter_jit = self.can_enter_jit + self.loop_header = self.loop_header self._set_param = self._set_param class Entry(ExtEnterLeaveMarker): _about_ = (self.jit_merge_point, self.can_enter_jit) + class Entry(ExtLoopHeader): + _about_ = self.loop_header + class Entry(ExtSetParam): _about_ = self._set_param @@ -422,6 +430,23 @@ return hop.genop('jit_marker', vlist, resulttype=lltype.Void) +class ExtLoopHeader(ExtRegistryEntry): + # Replace a call to myjitdriver.loop_header() + # with an operation jit_marker('loop_header', myjitdriver). + + def compute_result_annotation(self, **kwds_s): + from pypy.annotation import model as annmodel + return annmodel.s_None + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + driver = self.instance.im_self + hop.exception_cannot_occur() + vlist = [hop.inputconst(lltype.Void, 'loop_header'), + hop.inputconst(lltype.Void, driver)] + return hop.genop('jit_marker', vlist, + resulttype=lltype.Void) + class ExtSetParam(ExtRegistryEntry): def compute_result_annotation(self, s_name, s_value): From arigo at codespeak.net Mon Jul 12 14:59:41 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 14:59:41 +0200 (CEST) Subject: [pypy-svn] r76139 - in pypy/branch/kill-caninline/pypy/jit/metainterp: . test Message-ID: <20100712125941.B383E282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 14:59:40 2010 New Revision: 76139 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_ztranslation.py pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py Log: Fix tests. Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_virtualizable.py Mon Jul 12 14:59:40 2010 @@ -1330,11 +1330,9 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['frame'], virtualizables=["frame"], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, frame): pc = 0 while pc < len(code): Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_warmstate.py Mon Jul 12 14:59:40 2010 @@ -61,12 +61,14 @@ _green_args_spec = [lltype.Signed, lltype.Float] state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state._make_jitcell_getter_default() - cell1 = get_jitcell(42, 42.5) + cell1 = get_jitcell(True, 42, 42.5) assert isinstance(cell1, JitCell) - cell2 = get_jitcell(42, 42.5) + cell2 = get_jitcell(True, 42, 42.5) assert cell1 is cell2 - cell3 = get_jitcell(41, 42.5) - cell4 = get_jitcell(42, 0.25) + cell3 = get_jitcell(True, 41, 42.5) + assert get_jitcell(False, 42, 0.25) is None + cell4 = get_jitcell(True, 42, 0.25) + assert get_jitcell(False, 42, 0.25) is cell4 assert cell1 is not cell3 is not cell4 is not cell1 def test_make_jitcell_getter(): @@ -75,8 +77,8 @@ _get_jitcell_at_ptr = None state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state.make_jitcell_getter() - cell1 = get_jitcell(1.75) - cell2 = get_jitcell(1.75) + cell1 = get_jitcell(True, 1.75) + cell2 = get_jitcell(True, 1.75) assert cell1 is cell2 assert get_jitcell is state.make_jitcell_getter() @@ -103,14 +105,16 @@ # state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) get_jitcell = state._make_jitcell_getter_custom() - cell1 = get_jitcell(5, 42.5) + cell1 = get_jitcell(True, 5, 42.5) assert isinstance(cell1, JitCell) assert cell1.x == 5 assert cell1.y == 42.5 - cell2 = get_jitcell(5, 42.5) + cell2 = get_jitcell(True, 5, 42.5) assert cell2 is cell1 - cell3 = get_jitcell(41, 42.5) - cell4 = get_jitcell(42, 0.25) + cell3 = get_jitcell(True, 41, 42.5) + assert get_jitcell(False, 42, 0.25) is None + cell4 = get_jitcell(True, 42, 0.25) + assert get_jitcell(False, 42, 0.25) is cell4 assert cell1 is not cell3 is not cell4 is not cell1 def test_make_set_future_values(): @@ -153,52 +157,25 @@ state.attach_unoptimized_bridge_from_interp([ConstInt(5), ConstFloat(2.25)], "entry loop token") - cell1 = get_jitcell(5, 2.25) + cell1 = get_jitcell(True, 5, 2.25) assert cell1.counter < 0 assert cell1.entry_loop_token == "entry loop token" def test_make_jitdriver_callbacks_1(): class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = None _get_printable_location_ptr = None _confirm_enter_jit_ptr = None class FakeCell: dont_trace_here = False state = WarmEnterState(None, FakeJitDriverSD()) - def jit_getter(*args): + def jit_getter(build, *args): return FakeCell() state.jit_getter = jit_getter state.make_jitdriver_callbacks() - res = state.can_inline_callable([ConstInt(5), ConstFloat(42.5)]) - assert res is True res = state.get_location_str([ConstInt(5), ConstFloat(42.5)]) assert res == '(no jitdriver.get_printable_location!)' -def test_make_jitdriver_callbacks_2(): - def can_inline(x, y): - assert x == 5 - assert y == 42.5 - return False - CAN_INLINE = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Float], - lltype.Bool)) - class FakeCell: - dont_trace_here = False - class FakeWarmRunnerDesc: - rtyper = None - class FakeJitDriverSD: - _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = llhelper(CAN_INLINE, can_inline) - _get_printable_location_ptr = None - _confirm_enter_jit_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) - def jit_getter(*args): - return FakeCell() - state.jit_getter = jit_getter - state.make_jitdriver_callbacks() - res = state.can_inline_callable([ConstInt(5), ConstFloat(42.5)]) - assert res is False - def test_make_jitdriver_callbacks_3(): def get_location(x, y): assert x == 5 @@ -210,7 +187,6 @@ rtyper = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = None _get_printable_location_ptr = llhelper(GET_LOCATION, get_location) _confirm_enter_jit_ptr = None _get_jitcell_at_ptr = None @@ -231,7 +207,6 @@ rtyper = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = None _get_printable_location_ptr = None _confirm_enter_jit_ptr = llhelper(ENTER_JIT, confirm_enter_jit) _get_jitcell_at_ptr = None Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_ztranslation.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_ztranslation.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_ztranslation.py Mon Jul 12 14:59:40 2010 @@ -37,15 +37,12 @@ return jitcellcache.entry def get_printable_location(): return '(hello world)' - def can_inline(): - return False jitdriver = JitDriver(greens = [], reds = ['total', 'frame'], virtualizables = ['frame'], get_jitcell_at=get_jitcell_at, set_jitcell_at=set_jitcell_at, - get_printable_location=get_printable_location, - can_inline=can_inline) + get_printable_location=get_printable_location) def f(i): for param in unroll_parameters: defl = PARAMETERS[param] @@ -63,8 +60,7 @@ frame.i -= 1 return total * 10 # - myjitdriver2 = JitDriver(greens = ['g'], reds = ['m', 'x'], - can_inline = lambda *args: False) + myjitdriver2 = JitDriver(greens = ['g'], reds = ['m', 'x']) def f2(g, m, x): while m > 0: myjitdriver2.can_enter_jit(g=g, m=m, x=x) Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py Mon Jul 12 14:59:40 2010 @@ -232,7 +232,7 @@ # look for the cell corresponding to the current greenargs greenargs = args[:num_green_args] - cell = get_jitcell(*greenargs) + cell = get_jitcell(True, *greenargs) if cell.counter >= 0: # update the profiling counter @@ -316,17 +316,16 @@ return self.jit_getter # if self.jitdriver_sd._get_jitcell_at_ptr is None: - jit_getter_maybe, jit_getter = self._make_jitcell_getter_default() + jit_getter = self._make_jitcell_getter_default() else: - jit_getter_maybe, jit_getter = self._make_jitcell_getter_custom() + jit_getter = self._make_jitcell_getter_custom() # unwrap_greenkey = self.make_unwrap_greenkey() # def jit_cell_at_key(greenkey): greenargs = unwrap_greenkey(greenkey) - return jit_getter(*greenargs) + return jit_getter(True, *greenargs) self.jit_cell_at_key = jit_cell_at_key - self.jit_getter_maybe = jit_getter_maybe self.jit_getter = jit_getter # return jit_getter @@ -356,16 +355,16 @@ # jitcell_dict = r_dict(comparekey, hashkey) # - def get_jitcell(*greenargs): - return jitcell_dict.get(greenargs, None) - def get_or_build_jitcell(*greenargs): + def get_jitcell(build, *greenargs): try: cell = jitcell_dict[greenargs] except KeyError: + if not build: + return None cell = JitCell() jitcell_dict[greenargs] = cell return cell - return get_jitcell, get_or_build_jitcell + return get_jitcell def _make_jitcell_getter_custom(self): "NOT_RPYTHON" @@ -374,47 +373,47 @@ set_jitcell_at_ptr = self.jitdriver_sd._set_jitcell_at_ptr lltohlhack = {} # - def get_jitcell(*greenargs): + def get_jitcell(build, *greenargs): fn = support.maybe_on_top_of_llinterp(rtyper, get_jitcell_at_ptr) cellref = fn(*greenargs) # if we_are_translated(): BASEJITCELL = lltype.typeOf(cellref) cell = cast_base_ptr_to_instance(JitCell, cellref) - elif isinstance(cellref, (BaseJitCell, type(None))): - BASEJITCELL = None - cell = cellref else: - BASEJITCELL = lltype.typeOf(cellref) - if cellref: - cell = lltohlhack[rtyper.type_system.deref(cellref)] + if isinstance(cellref, (BaseJitCell, type(None))): + BASEJITCELL = None + cell = cellref else: - cell = None - # - return cell - def get_or_build_jitcell(*greenargs): - cell = get_jitcell_or_none(greenargs) + BASEJITCELL = lltype.typeOf(cellref) + if cellref: + cell = lltohlhack[rtyper.type_system.deref(cellref)] + else: + cell = None + if not build: + return cell if cell is None: cell = JitCell() # if we_are_translated(): cellref = cast_object_to_ptr(BASEJITCELL, cell) - elif BASEJITCELL is None: - cellref = cell else: - if isinstance(BASEJITCELL, lltype.Ptr): - cellref = lltype.malloc(BASEJITCELL.TO) - elif isinstance(BASEJITCELL, ootype.Instance): - cellref = ootype.new(BASEJITCELL) + if BASEJITCELL is None: + cellref = cell else: - assert False, "no clue" - lltohlhack[rtyper.type_system.deref(cellref)] = cell + if isinstance(BASEJITCELL, lltype.Ptr): + cellref = lltype.malloc(BASEJITCELL.TO) + elif isinstance(BASEJITCELL, ootype.Instance): + cellref = ootype.new(BASEJITCELL) + else: + assert False, "no clue" + lltohlhack[rtyper.type_system.deref(cellref)] = cell # fn = support.maybe_on_top_of_llinterp(rtyper, set_jitcell_at_ptr) fn(cellref, *greenargs) return cell - return get_jitcell, get_or_build_jitcell + return get_jitcell # ---------- @@ -475,11 +474,10 @@ return # unwrap_greenkey = self.make_unwrap_greenkey() - self.make_jitcell_getter() - jit_getter_maybe = self.jit_getter_maybe + jit_getter = self.make_jitcell_getter() def can_inline_greenargs(*greenargs): - cell = jit_getter_maybe(*greenargs) + cell = jit_getter(False, *greenargs) if cell is not None and cell.dont_trace_here: return False return True @@ -491,7 +489,7 @@ def get_assembler_token(greenkey): greenargs = unwrap_greenkey(greenkey) - cell = jit_getter_maybe(*greenargs) + cell = jit_getter(False, *greenargs) if cell is None or cell.counter >= 0: return None return cell.entry_loop_token From hakanardo at codespeak.net Mon Jul 12 15:14:32 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 12 Jul 2010 15:14:32 +0200 (CEST) Subject: [pypy-svn] r76140 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100712131432.E3C56282BAD@codespeak.net> Author: hakanardo Date: Mon Jul 12 15:14:31 2010 New Revision: 76140 Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: to/fromfile Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/app_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/app_array.py Mon Jul 12 15:14:31 2010 @@ -36,12 +36,11 @@ the array. Also called as read.""" if not isinstance(f, file): raise TypeError("arg1 must be open file") - self._fromfile(f, n) - - def _fromfile(self, f, n): size = self.itemsize * n + print "read" item = f.read(size) + print item if len(item) < size: n = len(item) % self.itemsize if n != 0: item = item[0:-(len(item) % self.itemsize)] Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Jul 12 15:14:31 2010 @@ -460,7 +460,6 @@ fromunicode = appmethod('fromunicode'), fromfile = appmethod('fromfile'), read = appmethod('fromfile'), - _fromfile = appmethod('_fromfile'), fromlist = appmethod('fromlist'), tolist = interp2app(W_Array.descr_tolist), Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Mon Jul 12 15:14:31 2010 @@ -153,12 +153,17 @@ def test_fromfile(self): - class myfile(object): - def __init__(self, c, s): - self.c = c - self.s = s - def read(self,n): - return self.c*min(n,self.s) + ## class myfile(object): + ## def __init__(self, c, s): + ## self.c = c + ## self.s = s + ## def read(self,n): + ## return self.c*min(n,self.s) + def myfile(c, s): + f=open('/tmp/deleteme', 'w') + f.write(c*s) + f.close() + return open('/tmp/deleteme', 'r') f=open('/dev/zero','r') for t in 'bBhHiIlLfd': @@ -167,16 +172,16 @@ assert len(a)==2 and a[0]==0 and a[1]==0 a = self.array('b') - a._fromfile(myfile('\x01', 20),2) + a.fromfile(myfile('\x01', 20),2) assert len(a)==2 and a[0]==1 and a[1]==1 a = self.array('h') - a._fromfile(myfile('\x01', 20),2) + a.fromfile(myfile('\x01', 20),2) assert len(a)==2 and a[0]==257 and a[1]==257 for i in (0,1): a = self.array('h') - raises(EOFError, a._fromfile, myfile('\x01', 2+i),2) + raises(EOFError, a.fromfile, myfile('\x01', 2+i),2) assert len(a)==1 and a[0]==257 @@ -357,10 +362,11 @@ a=self.array('i', s) assert a[0]==1 and a[1]==2 and a[2]==3 - unpack=self.unpack + unpack=self.struct.unpack values = (-129, 128, -128, 127, 0, 255, -1, 256, -32760, 32760) s = self.array('i', values).tostring() - a=unpack('i'*len(values), s) + fmt = 'i'*len(values) + a=unpack(fmt, s) assert a==values for tcodes, values in (('bhilfd', (-128, 127, 0, 1, 7, -10)), @@ -371,14 +377,16 @@ s = self.array(tc, values).tostring() a=unpack(tc*len(values), s) assert a==values - - - #FIXME: How to test? - #from cStringIO import StringIO - #f=StringIO() - #self.array('c', ('h', 'i')).tofile(f) - #assert f.getvalue() == 'hi' + f=open('/tmp/deleteme', 'w') + self.array('c', ('h', 'i')).tofile(f) + f.close() + assert open('/tmp/deleteme', 'r').readline() == 'hi' + + a=self.array('c') + a.fromfile(open('/tmp/deleteme', 'r'),2) + assert repr(a) == "array('c', 'hi')" + raises(ValueError, self.array('i').tounicode) assert self.array('u', unicode('hello')).tounicode() == unicode('hello') @@ -561,7 +569,7 @@ import array cls.array = array.array import struct - cls.unpack = struct.unpack + cls.struct = struct class AppTestArray(BaseArrayTests): @@ -571,15 +579,8 @@ import array return array.array """) - cls.w_unpack = cls.space.appexec([], """(): + cls.w_struct = cls.space.appexec([], """(): import struct - return struct.unpack + return struct """) -## class AppTestAppArray(AppTestArray): -## def setup_class(cls): -## cls.space = gettestobjspace(usemodules=('array',)) -## cls.w_array = cls.space.appexec([], """(): -## import apparray -## return apparray.array -## """) From hakanardo at codespeak.net Mon Jul 12 15:28:32 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 12 Jul 2010 15:28:32 +0200 (CEST) Subject: [pypy-svn] r76141 - pypy/branch/interplevel-array/pypy/module/array/test Message-ID: <20100712132832.4B824282BAD@codespeak.net> Author: hakanardo Date: Mon Jul 12 15:28:30 2010 New Revision: 76141 Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: buffer_info test Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Mon Jul 12 15:28:30 2010 @@ -490,17 +490,13 @@ b = pickle.loads(s) assert len(b) == 0 and b.typecode == 'l' - def test_misc(self): + def test_copy_swap(self): a = self.array('i', [1, 2, 3]) from copy import copy b = copy(a) a[1] = 7 assert repr(b) == "array('i', [1, 2, 3])" - bi=b.buffer_info() - assert bi[0] != 0 # FIXME: How can the address be tested? - assert bi[1] == 3 - for tc in 'bhilBHIL': a=self.array(tc, [1, 2, 3]) a.byteswap() @@ -574,7 +570,7 @@ class AppTestArray(BaseArrayTests): def setup_class(cls): - cls.space = gettestobjspace(usemodules=('array', 'struct')) + cls.space = gettestobjspace(usemodules=('array', 'struct', '_rawffi')) cls.w_array = cls.space.appexec([], """(): import array return array.array @@ -583,4 +579,17 @@ import struct return struct """) + cls.w_rffi = cls.space.appexec([], """(): + import _rawffi + return _rawffi + """) + + def test_buffer_info(self): + a = self.array('c', 'Hi!') + bi=a.buffer_info() + assert bi[0] != 0 + assert bi[1] == 3 + data = self.rffi.charp2string(bi[0]) + assert data == 'Hi!' + From arigo at codespeak.net Mon Jul 12 15:32:26 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 15:32:26 +0200 (CEST) Subject: [pypy-svn] r76142 - in pypy/trunk/pypy/rpython: . test Message-ID: <20100712133226.333B9282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 15:32:24 2010 New Revision: 76142 Modified: pypy/trunk/pypy/rpython/rstr.py pypy/trunk/pypy/rpython/test/test_rstr.py Log: Fix for str(u) where u is a UniChar. Modified: pypy/trunk/pypy/rpython/rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/rstr.py (original) +++ pypy/trunk/pypy/rpython/rstr.py Mon Jul 12 15:32:24 2010 @@ -415,7 +415,17 @@ sourcevars.append((v_item, r_arg)) return r_str.ll.do_stringformat(hop, sourcevars) - + + +class __extend__(AbstractCharRepr): + def ll_str(self, ch): + return self.ll.ll_chr2str(ch) + +class __extend__(AbstractUniCharRepr): + def ll_str(self, ch): + # xxx suboptimal, maybe + return str(unicode(ch)) + class __extend__(AbstractCharRepr, AbstractUniCharRepr): @@ -433,9 +443,6 @@ get_ll_fasthash_function = get_ll_hash_function - def ll_str(self, ch): - return self.ll.ll_chr2str(ch) - def rtype_len(_, hop): return hop.inputconst(Signed, 1) Modified: pypy/trunk/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rstr.py (original) +++ pypy/trunk/pypy/rpython/test/test_rstr.py Mon Jul 12 15:32:24 2010 @@ -863,6 +863,12 @@ res = self.interpret(f, [1]) assert self.ll_to_string(res) == "hello" + def test_str_unichar(self): + def f(i): + c = u"abc" + return str(c[i])[0] + assert self.interpret(f, [1]) == "b" + def FIXME_test_str_to_pystringobj(): def f(n): if n >= 0: From arigo at codespeak.net Mon Jul 12 15:34:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 15:34:21 +0200 (CEST) Subject: [pypy-svn] r76143 - in pypy/trunk/pypy/rpython: . test Message-ID: <20100712133421.4BD0D282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 15:34:19 2010 New Revision: 76143 Modified: pypy/trunk/pypy/rpython/rstr.py pypy/trunk/pypy/rpython/test/test_rstr.py Log: Fix for encode("ascii") called on a unichar. Modified: pypy/trunk/pypy/rpython/rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/rstr.py (original) +++ pypy/trunk/pypy/rpython/rstr.py Mon Jul 12 15:34:19 2010 @@ -288,7 +288,7 @@ if not hop.args_s[1].is_constant(): raise TyperError("encoding must be constant") encoding = hop.args_s[1].const - v_self = hop.inputarg(self.repr, 0) + v_self = hop.inputarg(self.lowleveltype, 0) hop.exception_is_here() if encoding == "ascii": return hop.gendirectcall(self.ll_str, v_self) Modified: pypy/trunk/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rstr.py (original) +++ pypy/trunk/pypy/rpython/test/test_rstr.py Mon Jul 12 15:34:19 2010 @@ -869,6 +869,12 @@ return str(c[i])[0] assert self.interpret(f, [1]) == "b" + def test_encode_char(self): + def f(i): + c = u"abc" + return c[i].encode("ascii") + assert self.ll_to_string(self.interpret(f, [0])) == "a" + def FIXME_test_str_to_pystringobj(): def f(n): if n >= 0: From arigo at codespeak.net Mon Jul 12 15:37:22 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 15:37:22 +0200 (CEST) Subject: [pypy-svn] r76144 - in pypy/trunk/pypy/rpython: . test Message-ID: <20100712133722.361F5282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 15:37:20 2010 New Revision: 76144 Modified: pypy/trunk/pypy/rpython/rstr.py pypy/trunk/pypy/rpython/test/test_rstr.py Log: The previous fix broke unichar.encode("latin-1"). Test it and re-fix it. Modified: pypy/trunk/pypy/rpython/rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/rstr.py (original) +++ pypy/trunk/pypy/rpython/rstr.py Mon Jul 12 15:37:20 2010 @@ -288,7 +288,11 @@ if not hop.args_s[1].is_constant(): raise TyperError("encoding must be constant") encoding = hop.args_s[1].const - v_self = hop.inputarg(self.lowleveltype, 0) + if encoding == "ascii": + expect = self.lowleveltype # can be a UniChar + else: + expect = self.repr # must be a regular unicode string + v_self = hop.inputarg(expect, 0) hop.exception_is_here() if encoding == "ascii": return hop.gendirectcall(self.ll_str, v_self) Modified: pypy/trunk/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rstr.py (original) +++ pypy/trunk/pypy/rpython/test/test_rstr.py Mon Jul 12 15:37:20 2010 @@ -875,6 +875,12 @@ return c[i].encode("ascii") assert self.ll_to_string(self.interpret(f, [0])) == "a" + def test_encode_char_latin1(self): + def f(i): + c = u"abc" + return c[i].encode("latin-1") + assert self.ll_to_string(self.interpret(f, [0])) == "a" + def FIXME_test_str_to_pystringobj(): def f(n): if n >= 0: From benjamin at codespeak.net Mon Jul 12 15:52:48 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 15:52:48 +0200 (CEST) Subject: [pypy-svn] r76145 - in pypy/branch/fast-forward: . pypy/jit/backend/llgraph pypy/jit/backend/llsupport pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/test pypy/rlib pypy/rlib/test pypy/rpython pypy/rpython/test pypy/translator/c/test Message-ID: <20100712135248.E6330282BAD@codespeak.net> Author: benjamin Date: Mon Jul 12 15:52:46 2010 New Revision: 76145 Modified: pypy/branch/fast-forward/ (props changed) pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/descr.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/support.py pypy/branch/fast-forward/pypy/jit/codewriter/codewriter.py pypy/branch/fast-forward/pypy/jit/codewriter/jtransform.py pypy/branch/fast-forward/pypy/jit/codewriter/test/test_flatten.py pypy/branch/fast-forward/pypy/jit/metainterp/executor.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_recursive.py pypy/branch/fast-forward/pypy/rlib/rarithmetic.py pypy/branch/fast-forward/pypy/rlib/rdynload.py pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py pypy/branch/fast-forward/pypy/rpython/rstr.py pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py pypy/branch/fast-forward/pypy/translator/c/test/test_typed.py Log: merge from trunk Modified: pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py Mon Jul 12 15:52:46 2010 @@ -421,11 +421,9 @@ global _last_exception assert _last_exception is None, "exception left behind" verbose = True - operations = self.loop.operations - opindex = 0 + self.opindex = 0 while True: - self.opindex = opindex - op = operations[opindex] + op = self.loop.operations[self.opindex] args = [self.getenv(v) for v in op.args] if not op.is_final(): try: @@ -439,8 +437,8 @@ args = [self.getenv(v) for v in op.fail_args if v] assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) - operations = op.jump_target.operations - opindex = 0 + self.loop = op.jump_target + self.opindex = 0 continue else: self._populate_fail_args(op) @@ -465,14 +463,13 @@ raise Exception("op.result.concretetype is %r" % (RESTYPE,)) self.env[op.result] = x - opindex += 1 + self.opindex += 1 continue if op.opnum == rop.JUMP: assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) self.loop = op.jump_target - operations = self.loop.operations - opindex = 0 + self.opindex = 0 _stats.exec_jumps += 1 elif op.opnum == rop.FINISH: if self.verbose: @@ -1549,6 +1546,8 @@ setannotation(do_getarrayitem_gc_int, annmodel.SomeInteger()) setannotation(do_getarrayitem_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) setannotation(do_getarrayitem_gc_float, annmodel.SomeFloat()) +setannotation(do_getarrayitem_raw_int, annmodel.SomeInteger()) +setannotation(do_getarrayitem_raw_float, annmodel.SomeFloat()) setannotation(do_getfield_gc_int, annmodel.SomeInteger()) setannotation(do_getfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) setannotation(do_getfield_gc_float, annmodel.SomeFloat()) @@ -1560,6 +1559,8 @@ setannotation(do_setarrayitem_gc_int, annmodel.s_None) setannotation(do_setarrayitem_gc_ptr, annmodel.s_None) setannotation(do_setarrayitem_gc_float, annmodel.s_None) +setannotation(do_setarrayitem_raw_int, annmodel.s_None) +setannotation(do_setarrayitem_raw_float, annmodel.s_None) setannotation(do_setfield_gc_int, annmodel.s_None) setannotation(do_setfield_gc_ptr, annmodel.s_None) setannotation(do_setfield_gc_float, annmodel.s_None) Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/descr.py Mon Jul 12 15:52:46 2010 @@ -23,10 +23,10 @@ self._cache_call = {} def init_size_descr(self, STRUCT, sizedescr): - pass + assert isinstance(STRUCT, lltype.GcStruct) def init_array_descr(self, ARRAY, arraydescr): - pass + assert isinstance(ARRAY, lltype.GcArray) # ____________________________________________________________ @@ -205,7 +205,8 @@ assert basesize == arraydescr.get_base_size(False) assert itemsize == arraydescr.get_item_size(False) assert ofslength == arraydescr.get_ofs_length(False) - gccache.init_array_descr(ARRAY, arraydescr) + if isinstance(ARRAY, lltype.GcArray): + gccache.init_array_descr(ARRAY, arraydescr) cache[ARRAY] = arraydescr return arraydescr Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/support.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/support.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/support.py Mon Jul 12 15:52:46 2010 @@ -30,7 +30,13 @@ funcobj = get_funcobj(fnptr) if hasattr(funcobj, 'graph'): - llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) + # cache the llinterp; otherwise the remember_malloc/remember_free + # done on the LLInterpreter don't match + try: + llinterp = rtyper._on_top_of_llinterp_llinterp + except AttributeError: + llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) + rtyper._on_top_of_llinterp_llinterp = llinterp def on_top_of_llinterp(*args): real_args = process_args(args) return llinterp.eval_graph(funcobj.graph, real_args) Modified: pypy/branch/fast-forward/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/codewriter.py Mon Jul 12 15:52:46 2010 @@ -14,11 +14,12 @@ class CodeWriter(object): callcontrol = None # for tests - def __init__(self, cpu=None, jitdrivers_sd=[]): + def __init__(self, cpu=None, jitdrivers_sd=[], debug=False): self.cpu = cpu self.assembler = Assembler() self.callcontrol = CallControl(cpu, jitdrivers_sd) self._seen_files = set() + self.debug = debug def transform_func_to_jitcode(self, func, values, type_system='lltype'): """For testing.""" @@ -60,7 +61,8 @@ self.assembler.assemble(ssarepr, jitcode) # # print the resulting assembler - self.print_ssa_repr(ssarepr, portal_jd, verbose) + if self.debug: + self.print_ssa_repr(ssarepr, portal_jd, verbose) def make_jitcodes(self, verbose=False): log.info("making JitCodes...") Modified: pypy/branch/fast-forward/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/jtransform.py Mon Jul 12 15:52:46 2010 @@ -667,7 +667,7 @@ return self._rewrite_symmetric(op) def _is_gc(self, v): - return v.concretetype.TO._gckind == 'gc' + return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc' def _rewrite_cmp_ptrs(self, op): if self._is_gc(op.args[0]): @@ -701,6 +701,42 @@ #return op raise NotImplementedError("cast_ptr_to_int") + def rewrite_op_force_cast(self, op): + from pypy.rpython.lltypesystem.rffi import size_and_sign, sizeof + from pypy.rlib.rarithmetic import intmask + assert not self._is_gc(op.args[0]) + size1, unsigned1 = size_and_sign(op.args[0].concretetype) + size2, unsigned2 = size_and_sign(op.result.concretetype) + if size2 >= sizeof(lltype.Signed): + return # the target type is LONG or ULONG + # + def bounds(size, unsigned): + if unsigned: + return 0, 1<<(8*size) + else: + return -(1<<(8*size-1)), 1<<(8*size-1) + min1, max1 = bounds(size1, unsigned1) + min2, max2 = bounds(size2, unsigned2) + if min2 <= min1 <= max1 <= max2: + return # the target type includes the source range + # + result = [] + v1 = op.args[0] + if min2: + c_min2 = Constant(min2, lltype.Signed) + v2 = Variable(); v2.concretetype = lltype.Signed + result.append(SpaceOperation('int_sub', [v1, c_min2], v2)) + else: + v2 = v1 + c_mask = Constant(int((1<<(8*size2))-1), lltype.Signed) + v3 = Variable(); v3.concretetype = lltype.Signed + result.append(SpaceOperation('int_and', [v2, c_mask], v3)) + if min2: + result.append(SpaceOperation('int_add', [v3, c_min2], op.result)) + else: + result[-1].result = op.result + return result + # ---------- # Renames, from the _old opname to the _new one. # The new operation is optionally further processed by rewrite_operation(). Modified: pypy/branch/fast-forward/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/test/test_flatten.py Mon Jul 12 15:52:46 2010 @@ -729,3 +729,108 @@ int_between %i0, %i1, %i2 -> %i3 int_return %i3 """, transform=True) + + def test_force_cast(self): + from pypy.rpython.lltypesystem import rffi + + for FROM, TO, expected in [ + (rffi.SIGNEDCHAR, rffi.SIGNEDCHAR, ""), + (rffi.SIGNEDCHAR, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.SIGNEDCHAR, rffi.SHORT, ""), + (rffi.SIGNEDCHAR, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.SIGNEDCHAR, rffi.LONG, ""), + (rffi.SIGNEDCHAR, rffi.ULONG, ""), + + (rffi.UCHAR, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.UCHAR, rffi.UCHAR, ""), + (rffi.UCHAR, rffi.SHORT, ""), + (rffi.UCHAR, rffi.USHORT, ""), + (rffi.UCHAR, rffi.LONG, ""), + (rffi.UCHAR, rffi.ULONG, ""), + + (rffi.SHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.SHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.SHORT, rffi.SHORT, ""), + (rffi.SHORT, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.SHORT, rffi.LONG, ""), + (rffi.SHORT, rffi.ULONG, ""), + + (rffi.USHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.USHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.USHORT, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.USHORT, rffi.USHORT, ""), + (rffi.USHORT, rffi.LONG, ""), + (rffi.USHORT, rffi.ULONG, ""), + + (rffi.LONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.LONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.LONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.LONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.LONG, rffi.LONG, ""), + (rffi.LONG, rffi.ULONG, ""), + + (rffi.ULONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.ULONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.ULONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.ULONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.ULONG, rffi.LONG, ""), + (rffi.ULONG, rffi.ULONG, ""), + ]: + expected = [s.strip() for s in expected.splitlines()] + check_force_cast(FROM, TO, expected, 42) + check_force_cast(FROM, TO, expected, -42) + expected.append('int_return %i' + str(len(expected))) + expected = '\n'.join(expected) + # + def f(n): + return rffi.cast(TO, n) + self.encoding_test(f, [rffi.cast(FROM, 42)], expected, + transform=True) + + def test_force_cast_pointer(self): + from pypy.rpython.lltypesystem import rffi + def h(p): + return rffi.cast(rffi.VOIDP, p) + self.encoding_test(h, [lltype.nullptr(rffi.CCHARP.TO)], """ + int_return %i0 + """, transform=True) + + +def check_force_cast(FROM, TO, operations, value): + """Check that the test is correctly written...""" + from pypy.rpython.lltypesystem import rffi + import re + r = re.compile('(\w+) \%i\d, \$(-?\d+)') + # + value = rffi.cast(FROM, value) + value = rffi.cast(lltype.Signed, value) + # + expected_value = rffi.cast(TO, value) + expected_value = rffi.cast(lltype.Signed, expected_value) + # + for op in operations: + match = r.match(op) + assert match, "line %r does not match regexp" % (op,) + opname = match.group(1) + if opname == 'int_add': value += int(match.group(2)) + elif opname == 'int_sub': value -= int(match.group(2)) + elif opname == 'int_and': value &= int(match.group(2)) + else: assert 0, opname + # + assert rffi.cast(lltype.Signed, value) == expected_value Modified: pypy/branch/fast-forward/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/executor.py Mon Jul 12 15:52:46 2010 @@ -91,7 +91,7 @@ return BoxInt(cpu.bh_getarrayitem_gc_i(arraydescr, array, index)) def do_getarrayitem_raw(cpu, _, arraybox, indexbox, arraydescr): - array = arraybox.getint() + array = arraybox.getref_base() index = indexbox.getint() assert not arraydescr.is_array_of_pointers() if arraydescr.is_array_of_floats(): Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_recursive.py Mon Jul 12 15:52:46 2010 @@ -523,7 +523,7 @@ def test_trace_from_start(self): def p(pc, code): code = hlstr(code) - return "%s %d %s" % (code, pc, code[pc]) + return "'%s' at %d: %s" % (code, pc, code[pc]) def c(pc, code): return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], @@ -537,9 +537,9 @@ op = code[pc] if op == "+": n += 7 - if op == "-": + elif op == "-": n -= 1 - if op == "c": + elif op == "c": n = f('---', n) elif op == "l": if n > 0: @@ -556,6 +556,7 @@ result = 0 for i in range(m): result += f('+-cl--', i) + g(50) self.meta_interp(g, [50], backendopt=True) self.check_tree_loop_count(3) self.check_history(int_add=1) Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Mon Jul 12 15:52:46 2010 @@ -33,7 +33,7 @@ """ -import math +import sys, math from pypy.rpython import extregistry from pypy.rlib import objectmodel @@ -206,14 +206,23 @@ "NOT_RPYTHON" return _local_ovfcheck(int(long(a) << b)) -FL_MAXINT = float(LONG_TEST-1) -FL_MININT = float(-LONG_TEST) - -def ovfcheck_float_to_int(x): - _, intp = math.modf(x) - if FL_MININT < intp < FL_MAXINT: - return int(intp) - raise OverflowError +# Strange things happening for float to int on 64 bit: +# int(float(i)) != i because of rounding issues. +# These are the minimum and maximum float value that can +# successfully be casted to an int. +if sys.maxint == 2147483647: + def ovfcheck_float_to_int(x): + if -2147483649.0 < x < 2147483648.0: + return int(x) + raise OverflowError +else: + # The following values are not quite +/-sys.maxint. + # Note the "<= x <" here, as opposed to "< x <" above. + # This is justified by test_typed in translator/c/test. + def ovfcheck_float_to_int(x): + if -9223372036854776832.0 <= x < 9223372036854775296.0: + return int(x) + raise OverflowError def compute_restype(self_type, other_type): if self_type is other_type: Modified: pypy/branch/fast-forward/pypy/rlib/rdynload.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rdynload.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rdynload.py Mon Jul 12 15:52:46 2010 @@ -18,8 +18,6 @@ if _WIN32: from pypy.rlib import rwin32 - -if _WIN32: includes = ['windows.h'] else: includes = ['dlfcn.h'] Modified: pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py Mon Jul 12 15:52:46 2010 @@ -271,26 +271,29 @@ assert ovfcheck_float_to_int(13.0) == 13 assert ovfcheck_float_to_int(-1.0) == -1 assert ovfcheck_float_to_int(-13.0) == -13 - # strange things happening for float to int on 64 bit - maxint32 = 2 ** 31 - 1 - assert ovfcheck_float_to_int(float(maxint32-1)) == maxint32-1 - #assert ovfcheck_float_to_int(float(maxint32)) == maxint32 - assert ovfcheck_float_to_int(float(-maxint32)) == -maxint32 - #assert ovfcheck_float_to_int(float(-maxint32-1)) == -maxint32-1 - - try: - ovfcheck_float_to_int(float(-sys.maxint-1)-1) - except OverflowError: - pass - else: - assert False - - try: - ovfcheck_float_to_int(float(sys.maxint)+1) - except OverflowError: - pass - else: - assert False + + # strange things happening for float to int on 64 bit: + # int(float(i)) != i because of rounding issues + x = sys.maxint + while int(float(x)) > sys.maxint: + x -= 1 + assert ovfcheck_float_to_int(float(x)) == int(float(x)) + + x = sys.maxint + 1 + while int(float(x)) <= sys.maxint: + x += 1 + py.test.raises(OverflowError, ovfcheck_float_to_int, x) + + x = -sys.maxint-1 + while int(float(x)) < -sys.maxint-1: + x += 1 + assert ovfcheck_float_to_int(float(x)) == int(float(x)) + + x = -sys.maxint-1 + while int(float(x)) >= -sys.maxint-1: + x -= 1 + py.test.raises(OverflowError, ovfcheck_float_to_int, x) + def test_abs(): assert type(abs(r_longlong(1))) is r_longlong Modified: pypy/branch/fast-forward/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/rstr.py (original) +++ pypy/branch/fast-forward/pypy/rpython/rstr.py Mon Jul 12 15:52:46 2010 @@ -291,7 +291,11 @@ if not hop.args_s[1].is_constant(): raise TyperError("encoding must be constant") encoding = hop.args_s[1].const - v_self = hop.inputarg(self.repr, 0) + if encoding == "ascii": + expect = self.lowleveltype # can be a UniChar + else: + expect = self.repr # must be a regular unicode string + v_self = hop.inputarg(expect, 0) hop.exception_is_here() if encoding == "ascii": return hop.gendirectcall(self.ll_str, v_self) @@ -418,7 +422,17 @@ sourcevars.append((v_item, r_arg)) return r_str.ll.do_stringformat(hop, sourcevars) - + + +class __extend__(AbstractCharRepr): + def ll_str(self, ch): + return self.ll.ll_chr2str(ch) + +class __extend__(AbstractUniCharRepr): + def ll_str(self, ch): + # xxx suboptimal, maybe + return str(unicode(ch)) + class __extend__(AbstractCharRepr, AbstractUniCharRepr): @@ -436,9 +450,6 @@ get_ll_fasthash_function = get_ll_hash_function - def ll_str(self, ch): - return self.ll.ll_chr2str(ch) - def rtype_len(_, hop): return hop.inputconst(Signed, 1) Modified: pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py (original) +++ pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py Mon Jul 12 15:52:46 2010 @@ -870,6 +870,24 @@ res = self.interpret(f, [1]) assert self.ll_to_string(res) == "hello" + def test_str_unichar(self): + def f(i): + c = u"abc" + return str(c[i])[0] + assert self.interpret(f, [1]) == "b" + + def test_encode_char(self): + def f(i): + c = u"abc" + return c[i].encode("ascii") + assert self.ll_to_string(self.interpret(f, [0])) == "a" + + def test_encode_char_latin1(self): + def f(i): + c = u"abc" + return c[i].encode("latin-1") + assert self.ll_to_string(self.interpret(f, [0])) == "a" + def FIXME_test_str_to_pystringobj(): def f(n): if n >= 0: Modified: pypy/branch/fast-forward/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/test/test_typed.py (original) +++ pypy/branch/fast-forward/pypy/translator/c/test/test_typed.py Mon Jul 12 15:52:46 2010 @@ -789,3 +789,37 @@ return u'hello' + unichr(i) f = self.getcompiled(func, [int]) assert f(0x1234) == u'hello\u1234' + + def test_ovfcheck_float_to_int(self): + from pypy.rlib.rarithmetic import ovfcheck_float_to_int + + def func(fl): + try: + return ovfcheck_float_to_int(fl) + except OverflowError: + return -666 + f = self.getcompiled(func, [float]) + assert f(-123.0) == -123 + + for frac in [0.0, 0.01, 0.99]: + # strange things happening for float to int on 64 bit: + # int(float(i)) != i because of rounding issues + x = sys.maxint + while int(x + frac) > sys.maxint: + x -= 1 + assert f(x + frac) == int(x + frac) + + x = sys.maxint + while int(x - frac) <= sys.maxint: + x += 1 + assert f(x - frac) == -666 + + x = -sys.maxint-1 + while int(x - frac) < -sys.maxint-1: + x += 1 + assert f(x - frac) == int(x - frac) + + x = -sys.maxint-1 + while int(x + frac) >= -sys.maxint-1: + x -= 1 + assert f(x + frac) == -666 From arigo at codespeak.net Mon Jul 12 15:56:41 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 15:56:41 +0200 (CEST) Subject: [pypy-svn] r76146 - in pypy/branch/kill-caninline/pypy: interpreter interpreter/astcompiler interpreter/test jit/backend/x86/test module/__builtin__ module/pypyjit module/pypyjit/test rlib/test Message-ID: <20100712135641.731FF282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 15:56:39 2010 New Revision: 76146 Removed: pypy/branch/kill-caninline/pypy/module/pypyjit/test/test_can_inline.py Modified: pypy/branch/kill-caninline/pypy/interpreter/astcompiler/consts.py pypy/branch/kill-caninline/pypy/interpreter/pycode.py pypy/branch/kill-caninline/pypy/interpreter/test/test_code.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_ztranslation.py pypy/branch/kill-caninline/pypy/module/__builtin__/functional.py pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py pypy/branch/kill-caninline/pypy/rlib/test/test_jit.py Log: Kill "can_inline" in the PyPy interpreter. Modified: pypy/branch/kill-caninline/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/astcompiler/consts.py Mon Jul 12 15:56:39 2010 @@ -9,7 +9,6 @@ CO_NESTED = 0x0010 CO_GENERATOR = 0x0020 CO_NOFREE = 0x0040 -CO_CONTAINSLOOP = 0x0080 CO_CONTAINSGLOBALS = 0x0800 CO_GENERATOR_ALLOWED = 0x1000 CO_FUTURE_DIVISION = 0x2000 Modified: pypy/branch/kill-caninline/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/pycode.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/pycode.py Mon Jul 12 15:56:39 2010 @@ -13,7 +13,7 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.astcompiler.consts import (CO_OPTIMIZED, CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, - CO_GENERATOR, CO_CONTAINSLOOP, CO_CONTAINSGLOBALS) + CO_GENERATOR, CO_CONTAINSGLOBALS) from pypy.rlib.rarithmetic import intmask from pypy.rlib.debug import make_sure_not_resized, make_sure_not_modified from pypy.rlib import jit @@ -133,9 +133,7 @@ while opcode == opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) next_instr += 3 - if opcode == opcodedesc.JUMP_ABSOLUTE.index: - self.co_flags |= CO_CONTAINSLOOP - elif opcode == opcodedesc.LOAD_GLOBAL.index: + if opcode == opcodedesc.LOAD_GLOBAL.index: self.co_flags |= CO_CONTAINSGLOBALS elif opcode == opcodedesc.LOAD_NAME.index: self.co_flags |= CO_CONTAINSGLOBALS Modified: pypy/branch/kill-caninline/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/test/test_code.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/test/test_code.py Mon Jul 12 15:56:39 2010 @@ -184,8 +184,6 @@ # CO_NESTED assert f(4).func_code.co_flags & 0x10 assert f.func_code.co_flags & 0x10 == 0 - # check for CO_CONTAINSLOOP - assert not f.func_code.co_flags & 0x0080 # check for CO_CONTAINSGLOBALS assert not f.func_code.co_flags & 0x0800 @@ -198,9 +196,6 @@ return [l for l in [1, 2, 3, 4]] """ - # check for CO_CONTAINSLOOP - assert f.func_code.co_flags & 0x0080 - assert g.func_code.co_flags & 0x0080 # check for CO_CONTAINSGLOBALS assert f.func_code.co_flags & 0x0800 assert not g.func_code.co_flags & 0x0800 Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_ztranslation.py Mon Jul 12 15:56:39 2010 @@ -74,8 +74,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno: str(codeno)) class SomewhereElse(object): pass Modified: pypy/branch/kill-caninline/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/__builtin__/functional.py (original) +++ pypy/branch/kill-caninline/pypy/module/__builtin__/functional.py Mon Jul 12 15:56:39 2010 @@ -221,8 +221,7 @@ from pypy.rlib.jit import JitDriver mapjitdriver = JitDriver(greens = ['code'], - reds = ['w_func', 'w_iter', 'result_w'], - can_inline = lambda *args: False) + reds = ['w_func', 'w_iter', 'result_w']) def map_single_user_function(code, w_func, w_iter): result_w = [] while True: Modified: pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py Mon Jul 12 15:56:39 2010 @@ -9,7 +9,7 @@ import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, Arguments -from pypy.interpreter.pycode import PyCode, CO_CONTAINSLOOP +from pypy.interpreter.pycode import PyCode from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.pyopcode import ExitFrame from opcode import opmap @@ -24,9 +24,6 @@ JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE'] -def can_inline(next_instr, bytecode): - return not bool(bytecode.co_flags & CO_CONTAINSLOOP) - def get_printable_location(next_instr, bytecode): from pypy.tool.stdlib_opcode import opcode_method_names name = opcode_method_names[ord(bytecode.co_code[next_instr])] @@ -57,8 +54,7 @@ ## blockstack = frame.blockstack ## return (valuestackdepth, blockstack) -pypyjitdriver = PyPyJitDriver(can_inline = can_inline, - get_printable_location = get_printable_location, +pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location, get_jitcell_at = get_jitcell_at, set_jitcell_at = set_jitcell_at, confirm_enter_jit = confirm_enter_jit) Modified: pypy/branch/kill-caninline/pypy/rlib/test/test_jit.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rlib/test/test_jit.py (original) +++ pypy/branch/kill-caninline/pypy/rlib/test/test_jit.py Mon Jul 12 15:56:39 2010 @@ -59,11 +59,9 @@ def test_annotate_hooks(self): - def can_inline(m): pass def get_printable_location(m): pass myjitdriver = JitDriver(greens=['m'], reds=['n'], - can_inline=can_inline, get_printable_location=get_printable_location) def fn(n): m = 42.5 @@ -81,9 +79,8 @@ return [v.concretetype for v in graph.getargs()] raise Exception, 'function %r has not been annotated' % func - can_inline_args = getargs(can_inline) get_printable_location_args = getargs(get_printable_location) - assert can_inline_args == get_printable_location_args == [lltype.Float] + assert get_printable_location_args == [lltype.Float] def test_annotate_argumenterror(self): myjitdriver = JitDriver(greens=['m'], reds=['n']) From benjamin at codespeak.net Mon Jul 12 15:59:06 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 15:59:06 +0200 (CEST) Subject: [pypy-svn] r76147 - pypy/branch/fast-forward/pypy/rlib Message-ID: <20100712135906.3CB19282BAD@codespeak.net> Author: benjamin Date: Mon Jul 12 15:59:04 2010 New Revision: 76147 Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Log: I still need these constants Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Mon Jul 12 15:59:04 2010 @@ -50,6 +50,9 @@ LONG_MASK = _Ltest*2-1 LONG_TEST = _Ltest +FL_MAXINT = float(LONG_TEST - 1) +FL_MININT = float(-LONG_TEST) + INFINITY = 1e200 * 1e200 NAN = INFINITY / INFINITY From benjamin at codespeak.net Mon Jul 12 16:10:41 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 16:10:41 +0200 (CEST) Subject: [pypy-svn] r76148 - in pypy/branch/fast-forward/pypy: objspace/std rlib Message-ID: <20100712141041.ADB7D282BAD@codespeak.net> Author: benjamin Date: Mon Jul 12 16:10:38 2010 New Revision: 76148 Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Log: use ovfcheck_float_to_int in trunc__Float Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Mon Jul 12 16:10:38 2010 @@ -9,8 +9,7 @@ from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.longobject import W_LongObject from pypy.rlib.rarithmetic import ovfcheck_float_to_int, intmask, isinf, isnan -from pypy.rlib.rarithmetic import (formatd, LONG_BIT, FL_MAXINT, FL_MININT, - INFINITY) +from pypy.rlib.rarithmetic import formatd, LONG_BIT, INFINITY from pypy.rlib.rbigint import rbigint from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rfloat @@ -79,8 +78,12 @@ space.wrap("cannot convert float infinity to long")) def trunc__Float(space, w_floatobj): whole = math.modf(w_floatobj.floatval)[1] - if FL_MININT < whole < FL_MAXINT: - return space.newint(int(whole)) + try: + i = ovfcheck_float_to_int(whole) + except OverflowError: + pass + else: + return space.wrap(i) try: return W_LongObject.fromfloat(w_floatobj.floatval) except OverflowError: Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Mon Jul 12 16:10:38 2010 @@ -50,9 +50,6 @@ LONG_MASK = _Ltest*2-1 LONG_TEST = _Ltest -FL_MAXINT = float(LONG_TEST - 1) -FL_MININT = float(-LONG_TEST) - INFINITY = 1e200 * 1e200 NAN = INFINITY / INFINITY From hakanardo at codespeak.net Mon Jul 12 16:31:55 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 12 Jul 2010 16:31:55 +0200 (CEST) Subject: [pypy-svn] r76149 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100712143155.85FCB282BAD@codespeak.net> Author: hakanardo Date: Mon Jul 12 16:31:54 2010 New Revision: 76149 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: rpythonish Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Jul 12 16:31:54 2010 @@ -364,8 +364,9 @@ if mytype.bytes not in [1, 2, 4, 8]: msg="byteswap not supported for this array" raise OperationError(self.space.w_RuntimeError, self.space.wrap(msg)) + if self.len == 0: return bytes = self.charbuf() - tmp = [0] * mytype.bytes + tmp = [bytes[0]] * mytype.bytes for start in range(0, self.len * mytype.bytes, mytype.bytes): stop = start + mytype.bytes - 1 for i in range(mytype.bytes): From benjamin at codespeak.net Mon Jul 12 16:49:55 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 16:49:55 +0200 (CEST) Subject: [pypy-svn] r76150 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100712144955.02079282BAD@codespeak.net> Author: benjamin Date: Mon Jul 12 16:49:53 2010 New Revision: 76150 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py Log: make _fill_char always a char Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Mon Jul 12 16:49:53 2010 @@ -306,7 +306,7 @@ def _parse_spec(self, default_type, default_align): space = self.space - self._fill_char = self._lit("\0") + self._fill_char = self._lit("\0")[0] self._align = default_align self._alternate = False self._sign = "\0" @@ -335,7 +335,7 @@ self._alternate = True i += 1 if self._fill_char == "\0" and length - i >= 1 and spec[i] == "0": - self._fill_char = self._lit("0") + self._fill_char = self._lit("0")[0] if not got_align: self._align = "=" i += 1 @@ -410,9 +410,9 @@ def _pad(self, string): builder = self._builder() - builder.append_multiple_char(self._fill_char[0], self._left_pad) + builder.append_multiple_char(self._fill_char, self._left_pad) builder.append(string) - builder.append_multiple_char(self._fill_char[0], self._right_pad) + builder.append_multiple_char(self._fill_char, self._right_pad) return builder.build() def _builder(self): @@ -445,7 +445,7 @@ if self._precision != -1 and length >= self._precision: length = self._precision if self._fill_char == "\0": - self._fill_char = self._lit(" ") + self._fill_char = self._lit(" ")[0] self._calc_padding(string, length) return space.wrap(self._pad(string)) From arigo at codespeak.net Mon Jul 12 17:01:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 17:01:13 +0200 (CEST) Subject: [pypy-svn] r76151 - pypy/trunk/pypy/interpreter Message-ID: <20100712150113.2D2B9282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 17:01:11 2010 New Revision: 76151 Modified: pypy/trunk/pypy/interpreter/pyopcode.py Log: No-op change: remove special-casing of YIELD_VALUE in the interpreter. Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Mon Jul 12 17:01:11 2010 @@ -211,10 +211,6 @@ next_instr = block.handle(self, unroller) return next_instr # now inside a 'finally' block - if opcode == self.opcodedesc.YIELD_VALUE.index: - #self.last_instr = intmask(next_instr - 1) XXX clean up! - raise Yield - if opcode == self.opcodedesc.END_FINALLY.index: unroller = self.end_finally() if isinstance(unroller, SuspendedUnroller): @@ -239,7 +235,7 @@ if not opdesc.is_enabled(space): continue if opdesc.methodname in ( - 'EXTENDED_ARG', 'RETURN_VALUE', 'YIELD_VALUE', + 'EXTENDED_ARG', 'RETURN_VALUE', 'END_FINALLY', 'JUMP_ABSOLUTE'): continue # opcodes implemented above @@ -814,6 +810,9 @@ self.space.str_w(w_name)) self.pushvalue(w_obj) + def YIELD_VALUE(self, oparg, next_instr): + raise Yield + def jump_absolute(self, jumpto, next_instr, ec): return jumpto From benjamin at codespeak.net Mon Jul 12 17:02:55 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 17:02:55 +0200 (CEST) Subject: [pypy-svn] r76152 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20100712150255.779B9282BAD@codespeak.net> Author: benjamin Date: Mon Jul 12 17:02:53 2010 New Revision: 76152 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py Log: must compare only chars Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Mon Jul 12 17:02:53 2010 @@ -127,7 +127,7 @@ raise OperationError(self.space.w_ValueError, w_msg) conversion = s[i] else: - conversion = None + conversion = self.empty i += 1 return s[start:end_name], conversion, i i += 1 @@ -228,9 +228,10 @@ def _convert(self, w_obj, conversion): space = self.space - if conversion == "r": + conv = conversion[0] + if conv == "r": return space.repr(w_obj) - elif conversion == "s": + elif conv == "s": if self.is_unicode: return space.call_function(space.w_unicode, w_obj) return space.str(w_obj) From arigo at codespeak.net Mon Jul 12 17:03:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 17:03:45 +0200 (CEST) Subject: [pypy-svn] r76153 - pypy/branch/kill-caninline/pypy/module/pypyjit Message-ID: <20100712150345.A617D282BAD@codespeak.net> Author: arigo Date: Mon Jul 12 17:03:44 2010 New Revision: 76153 Modified: pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py Log: Attempt to better support generators: a JUMP_ABSOLUTE should not be considered as ending a loop if a YIELD_VALUE was just executed in the same frame. This should support the most common case of doing one yield (or more) per iteration of the loop in the generator. Modified: pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py Mon Jul 12 17:03:44 2010 @@ -59,7 +59,14 @@ set_jitcell_at = set_jitcell_at, confirm_enter_jit = confirm_enter_jit) +PyFrame_execute_generator_frame = PyFrame.execute_generator_frame + class __extend__(PyFrame): + just_resuming_generator = False + + def execute_generator_frame(self, w_inputvalue, ex=False): + self.just_resuming_generator = True + return PyFrame_execute_generator_frame(self, w_inputvalue, ex) def dispatch(self, pycode, next_instr, ec): self = hint(self, access_directly=True) @@ -75,6 +82,9 @@ return self.popvalue() def jump_absolute(self, jumpto, _, ec=None): + if self.just_resuming_generator: + self.just_resuming_generator = False + return jumpto if we_are_jitted(): self.last_instr = intmask(jumpto) ec.bytecode_trace(self) From arigo at codespeak.net Mon Jul 12 18:29:38 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 18:29:38 +0200 (CEST) Subject: [pypy-svn] r76154 - pypy/branch/reflex-support/pypy/jit/codewriter/test Message-ID: <20100712162938.00816282BD6@codespeak.net> Author: arigo Date: Mon Jul 12 18:29:37 2010 New Revision: 76154 Modified: pypy/branch/reflex-support/pypy/jit/codewriter/test/test_effectinfo.py Log: A failing test. Modified: pypy/branch/reflex-support/pypy/jit/codewriter/test/test_effectinfo.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/codewriter/test/test_effectinfo.py (original) +++ pypy/branch/reflex-support/pypy/jit/codewriter/test/test_effectinfo.py Mon Jul 12 18:29:37 2010 @@ -1,7 +1,9 @@ from pypy.rpython.lltypesystem.rclass import OBJECT -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.ootypesystem import ootype +from pypy.rpython.test.test_llinterp import gengraph from pypy.jit.codewriter.effectinfo import effectinfo_from_writeanalyze +from pypy.jit.codewriter.effectinfo import VirtualizableAnalyzer class FakeCPU: def fielddescrof(self, T, fieldname): @@ -77,3 +79,19 @@ assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays + + +def test_external_calls(): + C_METHPTRGETTER = lltype.FuncType([lltype.Signed], lltype.Signed) + c_get_methptr_getter = rffi.llexternal( + "cppyy_get_methptr_getter", + [lltype.Signed], lltype.Ptr(C_METHPTRGETTER)) + # + def f(handle): + methgetter = c_get_methptr_getter(handle) + return methgetter(123) + t, _, graph = gengraph(f, [int]) + # + va = VirtualizableAnalyzer(t) + result = va.analyze_direct_call(graph) + assert result is False From arigo at codespeak.net Mon Jul 12 18:43:31 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Jul 2010 18:43:31 +0200 (CEST) Subject: [pypy-svn] r76155 - in pypy/branch/reflex-support/pypy: jit/codewriter translator/backendopt Message-ID: <20100712164331.64D4C282BD6@codespeak.net> Author: arigo Date: Mon Jul 12 18:43:30 2010 New Revision: 76155 Modified: pypy/branch/reflex-support/pypy/jit/codewriter/effectinfo.py pypy/branch/reflex-support/pypy/translator/backendopt/graphanalyze.py Log: Fix the bug in a not-completely-satisfying way... Modified: pypy/branch/reflex-support/pypy/jit/codewriter/effectinfo.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/codewriter/effectinfo.py (original) +++ pypy/branch/reflex-support/pypy/jit/codewriter/effectinfo.py Mon Jul 12 18:43:30 2010 @@ -104,3 +104,10 @@ def analyze_simple_operation(self, op): return op.opname in ('jit_force_virtualizable', 'jit_force_virtual') + + def analyze_unknown_indirect_call(self, op, seen=None): + # hack hack hack. The idea is that no indirect_call(..., None) + # should force the virtuals or virtualizables, which is kind of + # roughly reasonable: such an operation may call either a C function + # or an 'instantiate' method from a class vtable. + return False Modified: pypy/branch/reflex-support/pypy/translator/backendopt/graphanalyze.py ============================================================================== --- pypy/branch/reflex-support/pypy/translator/backendopt/graphanalyze.py (original) +++ pypy/branch/reflex-support/pypy/translator/backendopt/graphanalyze.py Mon Jul 12 18:43:30 2010 @@ -72,7 +72,7 @@ return self.analyze_direct_call(graph, seen) elif op.opname == "indirect_call": if op.args[-1].value is None: - return self.top_result() + return self.analyze_unknown_indirect_call(op, seen) return self.analyze_indirect_call(op.args[-1].value, seen) elif op.opname == "oosend": name = op.args[0].value @@ -124,6 +124,9 @@ results.append(self.analyze_direct_call(graph, seen)) return self.join_results(results) + def analyze_unknown_indirect_call(self, op, seen=None): + return self.top_result() + def analyze_oosend(self, TYPE, name, seen=None): graphs = TYPE._lookup_graphs(name) return self.analyze_indirect_call(graphs, seen) From benjamin at codespeak.net Mon Jul 12 18:54:12 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 18:54:12 +0200 (CEST) Subject: [pypy-svn] r76156 - pypy/branch/fast-forward/pypy/interpreter Message-ID: <20100712165412.32339282BD6@codespeak.net> Author: benjamin Date: Mon Jul 12 18:54:10 2010 New Revision: 76156 Modified: pypy/branch/fast-forward/pypy/interpreter/pycode.py Log: bump bytecode version Modified: pypy/branch/fast-forward/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/pycode.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/pycode.py Mon Jul 12 18:54:10 2010 @@ -29,8 +29,8 @@ # Magic numbers for the bytecode version in code objects. # See comments in pypy/module/imp/importing. cpython_magic, = struct.unpack(" Author: arigo Date: Mon Jul 12 19:08:30 2010 New Revision: 76157 Modified: pypy/branch/reflex-support/pypy/jit/codewriter/effectinfo.py pypy/branch/reflex-support/pypy/jit/codewriter/test/test_effectinfo.py pypy/branch/reflex-support/pypy/translator/backendopt/graphanalyze.py Log: Revert r76155, which does not help, and the test in r76154, which is a bit bogus. Modified: pypy/branch/reflex-support/pypy/jit/codewriter/effectinfo.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/codewriter/effectinfo.py (original) +++ pypy/branch/reflex-support/pypy/jit/codewriter/effectinfo.py Mon Jul 12 19:08:30 2010 @@ -104,10 +104,3 @@ def analyze_simple_operation(self, op): return op.opname in ('jit_force_virtualizable', 'jit_force_virtual') - - def analyze_unknown_indirect_call(self, op, seen=None): - # hack hack hack. The idea is that no indirect_call(..., None) - # should force the virtuals or virtualizables, which is kind of - # roughly reasonable: such an operation may call either a C function - # or an 'instantiate' method from a class vtable. - return False Modified: pypy/branch/reflex-support/pypy/jit/codewriter/test/test_effectinfo.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/codewriter/test/test_effectinfo.py (original) +++ pypy/branch/reflex-support/pypy/jit/codewriter/test/test_effectinfo.py Mon Jul 12 19:08:30 2010 @@ -1,9 +1,7 @@ from pypy.rpython.lltypesystem.rclass import OBJECT -from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype -from pypy.rpython.test.test_llinterp import gengraph from pypy.jit.codewriter.effectinfo import effectinfo_from_writeanalyze -from pypy.jit.codewriter.effectinfo import VirtualizableAnalyzer class FakeCPU: def fielddescrof(self, T, fieldname): @@ -79,19 +77,3 @@ assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays - - -def test_external_calls(): - C_METHPTRGETTER = lltype.FuncType([lltype.Signed], lltype.Signed) - c_get_methptr_getter = rffi.llexternal( - "cppyy_get_methptr_getter", - [lltype.Signed], lltype.Ptr(C_METHPTRGETTER)) - # - def f(handle): - methgetter = c_get_methptr_getter(handle) - return methgetter(123) - t, _, graph = gengraph(f, [int]) - # - va = VirtualizableAnalyzer(t) - result = va.analyze_direct_call(graph) - assert result is False Modified: pypy/branch/reflex-support/pypy/translator/backendopt/graphanalyze.py ============================================================================== --- pypy/branch/reflex-support/pypy/translator/backendopt/graphanalyze.py (original) +++ pypy/branch/reflex-support/pypy/translator/backendopt/graphanalyze.py Mon Jul 12 19:08:30 2010 @@ -72,7 +72,7 @@ return self.analyze_direct_call(graph, seen) elif op.opname == "indirect_call": if op.args[-1].value is None: - return self.analyze_unknown_indirect_call(op, seen) + return self.top_result() return self.analyze_indirect_call(op.args[-1].value, seen) elif op.opname == "oosend": name = op.args[0].value @@ -124,9 +124,6 @@ results.append(self.analyze_direct_call(graph, seen)) return self.join_results(results) - def analyze_unknown_indirect_call(self, op, seen=None): - return self.top_result() - def analyze_oosend(self, TYPE, name, seen=None): graphs = TYPE._lookup_graphs(name) return self.analyze_indirect_call(graphs, seen) From hakanardo at codespeak.net Mon Jul 12 20:11:08 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 12 Jul 2010 20:11:08 +0200 (CEST) Subject: [pypy-svn] r76159 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100712181108.F015F282BFE@codespeak.net> Author: hakanardo Date: Mon Jul 12 20:11:06 2010 New Revision: 76159 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: Speedups for lists and tuples Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Jul 12 20:11:06 2010 @@ -3,6 +3,8 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.rpython.lltypesystem import lltype, rffi from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root, ApplevelClass +from pypy.objspace.std.listobject import W_ListObject +from pypy.objspace.std.tupleobject import W_TupleObject from pypy.rlib.jit import dont_look_inside from pypy.rlib import rgc from pypy.rlib.unroll import unrolling_iterable @@ -43,11 +45,12 @@ self.canoverflow = canoverflow self.w_class = None - assert self.bytes <= rffi.sizeof(rffi.ULONG) - if self.bytes == rffi.sizeof(rffi.ULONG) and not signed and self.unwrap == 'int_w': - # Treat this type as a ULONG - self.unwrap = 'bigint_w' - self.canoverflow = False + if self.canoverflow: + assert self.bytes <= rffi.sizeof(rffi.ULONG) + if self.bytes == rffi.sizeof(rffi.ULONG) and not signed and self.unwrap == 'int_w': + # Treat this type as a ULONG + self.unwrap = 'bigint_w' + self.canoverflow = False def _freeze_(self): @@ -209,16 +212,23 @@ new = space.int_w(w_new) oldlen = self.len self.setlen(self.len + new) - for i in range(new): - w_item = space.call_function( - space.getattr(w_seq, space.wrap('__getitem__')), - space.wrap(i)) - try: - item=self.item_w(w_item) - except OperationError: - self.setlen(oldlen + i) - raise - self.buffer[oldlen + i ] = item + + try: + if (isinstance(w_seq, W_ListObject) or + isinstance(w_seq, W_TupleObject)): + for i in range(new): + item = self.item_w(w_seq.wrappeditems[i]) + self.buffer[oldlen + i ] = item + else: + getitem = space.getattr(w_seq, space.wrap('__getitem__')) + for i in range(new): + w_item = space.call_function(getitem, space.wrap(i)) + item=self.item_w(w_item) + self.buffer[oldlen + i ] = item + except OperationError: + self.setlen(oldlen + i) + raise + descr_fromsequence.unwrap_spec = ['self', W_Root] @@ -340,12 +350,12 @@ if i < 0 or i >= self.len: msg = 'pop index out of range' raise OperationError(self.space.w_IndexError, self.space.wrap(msg)) - val = self.buffer[i] + val = self.descr_getitem(self.space.wrap(i)) while i < self.len - 1: self.buffer[i] = self.buffer[i + 1] i += 1 self.setlen(self.len-1) - return self.space.wrap(val) + return val descr_pop.unwrap_spec = ['self', int] @@ -467,7 +477,6 @@ tounicode = appmethod('tounicode'), tofile = appmethod('tofile'), write = appmethod('tofile'), - #tostring = appmethod('tostring'), tostring = interp2app(W_Array.descr_tostring), _setlen = interp2app(W_Array.setlen), From benjamin at codespeak.net Mon Jul 12 21:16:57 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Jul 2010 21:16:57 +0200 (CEST) Subject: [pypy-svn] r76160 - pypy/branch/fast-forward/pypy/rlib/rstruct Message-ID: <20100712191657.B76CC282B9E@codespeak.net> Author: benjamin Date: Mon Jul 12 21:16:46 2010 New Revision: 76160 Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Log: pack and unpack floats in unsigned longs Modified: pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstruct/ieee.py Mon Jul 12 21:16:46 2010 @@ -4,7 +4,8 @@ import math -from pypy.rlib import rarithmetic +from pypy.rlib import rarithmetic, objectmodel +from pypy.rlib.rarithmetic import r_ulonglong def round_to_nearest(x): @@ -18,7 +19,7 @@ if you use this for negative x. """ - int_part = int(x) + int_part = r_ulonglong(x) frac_part = x - int_part if frac_part > 0.5 or frac_part == 0.5 and int_part & 1 == 1: int_part += 1 @@ -46,9 +47,10 @@ raise ValueError("input out of range") # extract pieces - sign = Q >> BITS - 1 - exp = (Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1 - mant = Q & ((1 << MANT_DIG - 1) - 1) + one = r_ulonglong(1) + sign = rarithmetic.intmask(Q >> BITS - 1) + exp = rarithmetic.intmask((Q & ((one << BITS - 1) - (one << MANT_DIG - 1))) >> MANT_DIG - 1) + mant = Q & ((one << MANT_DIG - 1) - 1) if exp == MAX_EXP - MIN_EXP + 2: # nan or infinity @@ -58,7 +60,7 @@ result = math.ldexp(float(mant), MIN_EXP - MANT_DIG) else: # normal - mant += 1 << MANT_DIG - 1 + mant += r_ulonglong(1) << MANT_DIG - 1 result = math.ldexp(float(mant), exp + MIN_EXP - MANT_DIG - 1) return -result if sign else result @@ -82,33 +84,34 @@ sign = rarithmetic.copysign(1.0, x) < 0.0 if rarithmetic.isinf(x): - mant = 0 + mant = r_ulonglong(0) exp = MAX_EXP - MIN_EXP + 2 elif rarithmetic.isnan(x): - mant = 1 << (MANT_DIG-2) # other values possible + mant = r_ulonglong(1) << (MANT_DIG-2) # other values possible exp = MAX_EXP - MIN_EXP + 2 elif x == 0.0: - mant = 0 + mant = r_ulonglong(0) exp = 0 else: m, e = math.frexp(abs(x)) # abs(x) == m * 2**e exp = e - (MIN_EXP - 1) if exp > 0: # Normal case. - mant = round_to_nearest(m * (1 << MANT_DIG)) - mant -= 1 << MANT_DIG - 1 + mant = round_to_nearest(m * (r_ulonglong(1) << MANT_DIG)) + mant -= r_ulonglong(1) << MANT_DIG - 1 else: # Subnormal case. if exp + MANT_DIG - 1 >= 0: - mant = round_to_nearest(m * (1 << exp + MANT_DIG - 1)) + mant = round_to_nearest(m * (r_ulonglong(1) << exp + MANT_DIG - 1)) else: - mant = 0 + mant = r_ulonglong(0) exp = 0 # Special case: rounding produced a MANT_DIG-bit mantissa. - assert 0 <= mant <= 1 << MANT_DIG - 1 - if mant == 1 << MANT_DIG - 1: - mant = 0 + if not objectmodel.we_are_translated(): + assert 0 <= mant <= 1 << MANT_DIG - 1 + if mant == r_ulonglong(1) << MANT_DIG - 1: + mant = r_ulonglong(0) exp += 1 # Raise on overflow (in some circumstances, may want to return @@ -117,9 +120,10 @@ raise OverflowError("float too large to pack in this format") # check constraints - assert 0 <= mant < 1 << MANT_DIG - 1 - assert 0 <= exp <= MAX_EXP - MIN_EXP + 2 - assert 0 <= sign <= 1 + if not objectmodel.we_are_translated(): + assert 0 <= mant < 1 << MANT_DIG - 1 + assert 0 <= exp <= MAX_EXP - MIN_EXP + 2 + assert 0 <= sign <= 1 return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant @@ -134,8 +138,8 @@ def unpack_float(s, be): - unsigned = 0 + unsigned = r_ulonglong(0) for i in range(len(s)): c = ord(s[len(s) - 1 - i if be else i]) - unsigned |= c << (i * 8) + unsigned |= r_ulonglong(c) << (i * 8) return float_unpack(unsigned, len(s)) From afa at codespeak.net Mon Jul 12 22:01:57 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Jul 2010 22:01:57 +0200 (CEST) Subject: [pypy-svn] r76161 - in pypy/branch/unicode_filename-2/pypy: rlib rlib/test rpython rpython/lltypesystem rpython/module Message-ID: <20100712200157.238E4282B9E@codespeak.net> Author: afa Date: Mon Jul 12 22:01:55 2010 New Revision: 76161 Added: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py pypy/branch/unicode_filename-2/pypy/rpython/extfunc.py pypy/branch/unicode_filename-2/pypy/rpython/lltypesystem/rffi.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Add rlib.rposix.open(), which accept strings, and unicode "with default encoding". The default implementation simply calls the encode() method and passes the result to os.open(); but ll_os can provide a function that directly accepts unicode. Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Mon Jul 12 22:01:55 2010 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rlib.rarithmetic import intmask +from pypy.rlib.objectmodel import specialize class CConstantErrno(CConstant): # these accessors are used when calling get_errno() or set_errno() @@ -42,3 +43,13 @@ os.close(fd) except OSError: pass + + +# pypy.rpython.module.ll_os.py may force the annotator to flow a different +# function that directly handle unicode strings. + at specialize.argtype(0) +def open(path, flags, mode): + if isinstance(path, str): + return os.open(path, flags, mode) + else: + return os.open(path.encode(), flags, mode) Added: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- (empty file) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Mon Jul 12 22:01:55 2010 @@ -0,0 +1,39 @@ +from pypy.rpython.test.test_llinterp import interpret +from pypy.tool.udir import udir +from pypy.rlib import rposix +import os + +def ll_to_string(s): + return ''.join(s.chars) + +class TestPosixUnicode: + def test_access(self): + ufilename = (unicode(udir.join('test_open')) + + u'\u65e5\u672c.txt') # "Japan" + f = file(ufilename, 'w') + f.write("test") + f.close() + filename = str(udir.join('test_open')) + + class UnicodeWithEncoding: + def __init__(self, unistr): + self.unistr = unistr + + def encode(self): + from pypy.rlib.runicode import unicode_encode_utf_8 + return unicode_encode_utf_8(self.unistr, len(self.unistr), + "strict") + path = UnicodeWithEncoding(ufilename) + + def f(): + try: + fd = rposix.open(path, os.O_RDONLY, 0777) + try: + text = os.read(fd, 50) + return text + finally: + os.close(fd) + except OSError: + return '' + + assert ll_to_string(interpret(f, [])) == "test" Modified: pypy/branch/unicode_filename-2/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/extfunc.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/extfunc.py Mon Jul 12 22:01:55 2010 @@ -52,7 +52,10 @@ return super(ExtRegistryEntry, self).__getattr__(attr) raise exc, exc_inst, tb -def registering(func): +def registering(func, condition=True): + if not condition: + return lambda method: None + def decorator(method): method._registering_func = func return method @@ -63,11 +66,9 @@ func = getattr(ns, name) except AttributeError: condition = False + func = None - if condition: - return registering(func) - else: - return lambda method: None + return registering(func, condition=condition) class LazyRegisteringMeta(type): def __new__(self, _name, _type, _vars): Modified: pypy/branch/unicode_filename-2/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/lltypesystem/rffi.py Mon Jul 12 22:01:55 2010 @@ -176,6 +176,15 @@ # XXX leaks if a str2charp() fails with MemoryError # and was not the first in this function freeme = arg + elif TARGET == CWCHARP: + if arg is None: + arg = lltype.nullptr(CWCHARP.TO) # None => (wchar_t*)NULL + freeme = arg + elif isinstance(arg, unicode): + arg = unicode2wcharp(arg) + # XXX leaks if a unicode2wcharp() fails with MemoryError + # and was not the first in this function + freeme = arg elif _isfunctype(TARGET) and not _isllptr(arg): # XXX pass additional arguments if invoke_around_handlers: Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Mon Jul 12 22:01:55 2010 @@ -7,7 +7,7 @@ import os, sys, errno from pypy.rpython.module.support import ll_strcpy, OOSupport -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rlib.rarithmetic import r_longlong from pypy.rpython.extfunc import BaseLazyRegistering from pypy.rpython.extfunc import registering, registering_if, extdef @@ -26,7 +26,41 @@ from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.annlowlevel import llstr from pypy.rlib import rgc -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import keepalive_until_here, specialize +from pypy.rlib.unroll import unrolling_iterable + +def registering_unicode_version(func, nbargs, argnums, condition=True): + """ + Registers an implementation of func() which directly accepts unicode + strings. Replaces the corresponding function in pypy.rlib.rposix. + """ + def unicodefunc(*args): + raise NotImplementedError + + argnum = argnums[0] + unrolling_args = unrolling_iterable(enumerate([i in argnums + for i in range(nbargs)])) + + func_name = func.__name__ + rposix_func = getattr(rposix, func_name) + + @specialize.argtype(*argnums) + @func_renamer(func.__name__) + def new_func(*args): + if isinstance(args[argnum], str): + return func(*args) + else: + real_args = () + for i, isunicode in unrolling_args: + if isunicode: + real_args += (args[i].unistr,) + else: + real_args += (args[i],) + return unicodefunc(*real_args) + if condition: + rposix_func._flowspace_rewrite_directly_as_ = new_func + + return registering(unicodefunc, condition=condition) posix = __import__(os.name) @@ -705,6 +739,20 @@ return extdef([str, int, int], int, "ll_os.ll_os_open", llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) + @registering_unicode_version(os.open, 3, [0], sys.platform=='win32') + def register_os_open_unicode(self): + os_wopen = self.llexternal(underscore_on_windows+'wopen', + [rffi.CWCHARP, rffi.INT, rffi.MODE_T], + rffi.INT) + def os_wopen_llimpl(path, flags, mode): + result = rffi.cast(rffi.LONG, os_wopen(path, flags, mode)) + if result == -1: + raise OSError(rposix.get_errno(), "os_open failed") + return result + + return extdef([unicode, int, int], int, "ll_os.ll_os_wopen", + llimpl=os_wopen_llimpl) + # ------------------------------- os.read ------------------------------- @registering(os.read) From afa at codespeak.net Mon Jul 12 22:05:45 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Jul 2010 22:05:45 +0200 (CEST) Subject: [pypy-svn] r76162 - in pypy/branch/unicode_filename-2/pypy: interpreter module/posix Message-ID: <20100712200545.610BF282B9E@codespeak.net> Author: afa Date: Mon Jul 12 22:05:38 2010 New Revision: 76162 Modified: pypy/branch/unicode_filename-2/pypy/interpreter/error.py pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Log: Try to use it in the posix module Modified: pypy/branch/unicode_filename-2/pypy/interpreter/error.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/interpreter/error.py (original) +++ pypy/branch/unicode_filename-2/pypy/interpreter/error.py Mon Jul 12 22:05:38 2010 @@ -344,7 +344,7 @@ else: _WINDOWS = True - def wrap_windowserror(space, e, filename=None): + def wrap_windowserror(space, e, w_filename=None): from pypy.rlib import rwin32 winerror = e.winerror @@ -353,19 +353,19 @@ except ValueError: msg = 'Windows Error %d' % winerror exc = space.w_WindowsError - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(winerror), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: w_error = space.call_function(exc, space.wrap(winerror), space.wrap(msg)) return OperationError(exc, w_error) -def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): +def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError'): assert isinstance(e, OSError) if _WINDOWS and isinstance(e, WindowsError): - return wrap_windowserror(space, e, filename) + return wrap_windowserror(space, e, w_filename) errno = e.errno try: @@ -373,10 +373,21 @@ except ValueError: msg = 'error %d' % errno exc = getattr(space, exception_name) - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(errno), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: - w_error = space.call_function(exc, space.wrap(errno), space.wrap(msg)) + w_error = space.call_function(exc, space.wrap(errno), + space.wrap(msg)) return OperationError(exc, w_error) +wrap_oserror2._annspecialcase_ = 'specialize:arg(3)' + +def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): + if filename is not None: + return wrap_oserror2(space, e, space.wrap(filename), + exception_name=exception_name) + else: + return wrap_oserror2(space, e, None, + exception_name=exception_name) wrap_oserror._annspecialcase_ = 'specialize:arg(3)' + Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Mon Jul 12 22:05:38 2010 @@ -2,7 +2,7 @@ from pypy.rlib import rposix from pypy.rlib.rarithmetic import r_longlong from pypy.rlib.unroll import unrolling_iterable -from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 from pypy.rpython.module.ll_os import RegisterOs from pypy.rpython.module import ll_os_stat from pypy.rpython.lltypesystem import rffi, lltype @@ -12,15 +12,28 @@ import os, sys _WIN = sys.platform == 'win32' -def open(space, fname, flag, mode=0777): +class FileEncoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def encode(self): + return self.space.path_w(self.w_obj) + +def open(space, w_fname, flag, mode=0777): """Open a file (for low level IO). Return a file descriptor (a small integer).""" - try: - fd = os.open(fname, flag, mode) + try: + if space.isinstance_w(w_fname, space.w_unicode): + fname = FileEncoder(space, w_fname) + fd = rposix.open(fname, flag, mode) + else: + fname = space.str_w(w_fname) + fd = rposix.open(fname, flag, mode) except OSError, e: - raise wrap_oserror(space, e, fname) + raise wrap_oserror2(space, e, w_fname) return space.wrap(fd) -open.unwrap_spec = [ObjSpace, 'path', "c_int", "c_int"] +open.unwrap_spec = [ObjSpace, W_Root, "c_int", "c_int"] def lseek(space, fd, pos, how): """Set the current position of a file descriptor. Return the new position. From wlav at codespeak.net Mon Jul 12 22:26:28 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Mon, 12 Jul 2010 22:26:28 +0200 (CEST) Subject: [pypy-svn] r76163 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100712202628.9A221282B9E@codespeak.net> Author: wlav Date: Mon Jul 12 22:26:26 2010 New Revision: 76163 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/example01.h pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Log: Allow executor calls returning objects to succeed (the resulting return object is not yet usuable, though). Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Mon Jul 12 22:26:26 2010 @@ -39,7 +39,7 @@ compilation_info=eci) c_deallocate = rffi.llexternal( "cppyy_deallocate", - [C_TYPEHANDLE, C_OBJECT], rffi.VOID, + [C_TYPEHANDLE, C_OBJECT], lltype.Void, compilation_info=eci) c_call_v = rffi.llexternal( "cppyy_call_v", Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Mon Jul 12 22:26:26 2010 @@ -68,13 +68,12 @@ return _converters[name] except KeyError: pass + compound = helper.compound(name) cpptype = interp_cppyy.type_byname(space, helper.clean_type(name)) if compound == "*": return InstancePtrConverter(space, cpptype) - print name - raise OperationError(space.w_TypeError, space.wrap("no clue what %s is" % name)) _converters["int"] = IntConverter() Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Mon Jul 12 22:26:26 2010 @@ -1,9 +1,9 @@ -import pypy.module.cppyy.capi as capi - from pypy.rpython.lltypesystem import rffi, lltype -_executors = {} +from pypy.module.cppyy import helper, capi + +_executors = {} class FunctionExecutor(object): def execute(self, space, func, cppthis, num_args, args): @@ -32,12 +32,30 @@ result = capi.charp2str_free(ccpresult) return space.wrap(result) +class InstancePtrExecutor(FunctionExecutor): + _immutable_ = True + def __init__(self, space, cpptype): + self.cpptype = cpptype + + def execute(self, space, func, cppthis, num_args, args): + from pypy.module.cppyy import interp_cppyy + result = capi.c_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) + return interp_cppyy.W_CCPInstance(self.cpptype, result) + + def get_executor(space, name): + from pypy.module.cppyy import interp_cppyy + try: return _executors[name] except KeyError: pass + compound = helper.compound(name) + cpptype = interp_cppyy.type_byname(space, helper.clean_type(name)) + if compound == "*": + return InstancePtrExecutor(space, cpptype) + return None # currently used until proper lazy instantiation available in interp_cppyy # raise TypeError("no clue what %s is" % name) Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Mon Jul 12 22:26:26 2010 @@ -35,6 +35,7 @@ if handle: cpptype = W_CPPType(space, name, handle) state.cpptype_cache[name] = cpptype + cpptype._find_func_members() return cpptype raise OperationError(space.w_TypeError, space.wrap("no such C++ class %s" % name)) @@ -211,7 +212,10 @@ self.name = name self.handle = handle self.function_members = {} - self._find_func_members() + # Do not call "self._find_func_members()" here, so that a distinction can be + # made between testing for existence (i.e. existence in the cache of classes) + # and actual use. Point being that a class can use itself, e.g. as a return + # type or an argument to one of its methods. def _find_func_members(self): num_func_members = capi.c_num_methods(self.handle) Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx Mon Jul 12 22:26:26 2010 @@ -54,7 +54,7 @@ ::strcpy(strout, strin); return strout; } -void example01::setPayload( payload* p, double d ) { +void example01::staticSetPayload(payload* p, double d) { p->setData(d); } @@ -86,4 +86,13 @@ return cresult; } +void example01::setPayload(payload* p) { + p->setData(somedata); +} + +payload* example01::cyclePayload(payload* p) { + setPayload(p); + return p; +} + int example01::count = 0; Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/example01.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.h Mon Jul 12 22:26:26 2010 @@ -28,7 +28,7 @@ static double staticAddToDouble(double a); static int staticAtoi(const char* str); static char* staticStrcpy(const char* strin); - static void setPayload( payload* p, double d ); + static void staticSetPayload(payload* p, double d); static int getCount(); // instance methods @@ -36,4 +36,7 @@ double addDataToDouble(double a); int addDataToAtoi(const char* str); char* addToStringValue(const char* str); + + void setPayload(payload* p); + payload* cyclePayload(payload* p); }; Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Mon Jul 12 22:26:26 2010 @@ -141,8 +141,33 @@ pl = self.payload.construct(3.14) assert round(pl.invoke("getData")-3.14, 8) == 0 - t.invoke("setPayload", pl, 41.) # now pl is a CPPInstance + t.invoke("staticSetPayload", pl, 41.) # now pl is a CPPInstance assert pl.invoke("getData") == 41. + + e = t.construct(50) + e.invoke("setPayload", pl); + assert round(pl.invoke("getData")-50., 8) == 0 + + e.destruct() + pl.destruct() + assert t.invoke("getCount") == 0 + + def testReturningOfAnObjectByPointer(self): + """Test passing of an instance as an argument.""" + + t = self.example01 + pl = self.payload.construct(3.14) + assert round(pl.invoke("getData")-3.14, 8) == 0 + + e = t.construct(50) + + e.invoke("setPayload", pl); + assert round(pl.invoke("getData")-50., 8) == 0 + e.invoke("cyclePayload", pl); + assert round(pl.invoke("getData")-50., 8) == 0 + + e.destruct() pl.destruct() assert t.invoke("getCount") == 0 + Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Mon Jul 12 22:26:26 2010 @@ -130,13 +130,18 @@ pl = payload_class(3.14) assert round(pl.getData()-3.14, 8) == 0 - example01_class.setPayload(pl._cppinstance, 41.) + example01_class.staticSetPayload(pl._cppinstance, 41.) assert pl.getData() == 41. - example01_class.setPayload(pl, 43.) + example01_class.staticSetPayload(pl, 43.) assert pl.getData() == 43. - e.setPayload(pl, 45.) + e.staticSetPayload(pl, 45.) assert pl.getData() == 45. + e.setPayload(pl); + assert round(pl.getData()-14., 8) == 0 + e.cyclePayload(pl); + assert round(pl.getData()-14., 8) == 0 + pl.destruct() e.destruct() assert example01_class.getCount() == 0 From afa at codespeak.net Mon Jul 12 22:28:03 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Jul 2010 22:28:03 +0200 (CEST) Subject: [pypy-svn] r76164 - in pypy/branch/unicode_filename-2/pypy: module/posix module/posix/test rlib/test rpython/module Message-ID: <20100712202803.E3F62282B9E@codespeak.net> Author: afa Date: Mon Jul 12 22:28:02 2010 New Revision: 76164 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Add an interp-level test Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Mon Jul 12 22:28:02 2010 @@ -20,6 +20,9 @@ def encode(self): return self.space.path_w(self.w_obj) + def gettext(self): + return self.space.unicode_w(self.w_obj) + def open(space, w_fname, flag, mode=0777): """Open a file (for low level IO). Return a file descriptor (a small integer).""" Modified: pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py Mon Jul 12 22:28:02 2010 @@ -706,6 +706,25 @@ except OSError: pass +class AppTestUnicodeFilename: + def setup_class(cls): + ufilename = (unicode(udir.join('test_unicode_filename_')) + + u'\u65e5\u672c.txt') # "Japan" + f = file(ufilename, 'w') + f.write("test") + f.close() + cls.space = space + cls.w_filename = space.wrap(ufilename) + cls.w_posix = space.appexec([], GET_POSIX) + + def test_open(self): + fd = self.posix.open(self.filename, self.posix.O_RDONLY) + try: + content = self.posix.read(fd, 50) + finally: + self.posix.close(fd) + assert content == "test" + class TestPexpect(object): # XXX replace with AppExpectTest class as soon as possible Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Mon Jul 12 22:28:02 2010 @@ -23,6 +23,10 @@ from pypy.rlib.runicode import unicode_encode_utf_8 return unicode_encode_utf_8(self.unistr, len(self.unistr), "strict") + + def gettext(self): + return self.unistr + path = UnicodeWithEncoding(ufilename) def f(): Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Mon Jul 12 22:28:02 2010 @@ -35,7 +35,7 @@ strings. Replaces the corresponding function in pypy.rlib.rposix. """ def unicodefunc(*args): - raise NotImplementedError + return func(*args) argnum = argnums[0] unrolling_args = unrolling_iterable(enumerate([i in argnums @@ -53,12 +53,12 @@ real_args = () for i, isunicode in unrolling_args: if isunicode: - real_args += (args[i].unistr,) + real_args += (args[i].gettext(),) else: real_args += (args[i],) return unicodefunc(*real_args) if condition: - rposix_func._flowspace_rewrite_directly_as_ = new_func + setattr(rposix, func_name, new_func) return registering(unicodefunc, condition=condition) From afa at codespeak.net Mon Jul 12 22:37:46 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Jul 2010 22:37:46 +0200 (CEST) Subject: [pypy-svn] r76165 - pypy/branch/unicode_filename-2/pypy/module/posix/test Message-ID: <20100712203746.525FD282B9E@codespeak.net> Author: afa Date: Mon Jul 12 22:37:44 2010 New Revision: 76165 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py Log: Fix test on Linux Modified: pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py Mon Jul 12 22:37:44 2010 @@ -32,6 +32,9 @@ # even when running on top of CPython 2.4. os.stat_float_times(True) + # Initialize sys.filesystemencoding + space.call_function(space.sys.get('getfilesystemencoding')) + def need_sparse_files(): if sys.platform == 'darwin': py.test.skip("no sparse files on default Mac OS X file system") From afa at codespeak.net Mon Jul 12 22:58:56 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Jul 2010 22:58:56 +0200 (CEST) Subject: [pypy-svn] r76166 - in pypy/branch/unicode_filename-2/pypy: module/_file module/_file/test module/posix rlib Message-ID: <20100712205856.481C3282B9E@codespeak.net> Author: afa Date: Mon Jul 12 22:58:54 2010 New Revision: 76166 Modified: pypy/branch/unicode_filename-2/pypy/module/_file/interp_file.py pypy/branch/unicode_filename-2/pypy/module/_file/test/test_file.py pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/rlib/streamio.py Log: Unicode support for __builtin__.open() Modified: pypy/branch/unicode_filename-2/pypy/module/_file/interp_file.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/_file/interp_file.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/_file/interp_file.py Mon Jul 12 22:58:54 2010 @@ -4,6 +4,7 @@ from pypy.rlib.rarithmetic import r_longlong from pypy.module._file.interp_stream import W_AbstractStream from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror +from pypy.module.posix.interp_posix import dispatch_filename from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.typedef import TypeDef, GetSetProperty @@ -81,11 +82,12 @@ # file lock. They don't convert StreamErrors to OperationErrors, too. def direct___init__(self, w_name, mode='r', buffering=-1): - name = self.space.str_w(w_name) self.direct_close() self.w_name = w_name self.check_mode_ok(mode) - stream = streamio.open_file_as_stream(name, mode, buffering) + stream = dispatch_filename( + self.space, w_name, + lambda name: streamio.open_file_as_stream(name, mode, buffering)) fd = stream.try_to_find_file_descriptor() self.fdopenstream(stream, fd, mode) Modified: pypy/branch/unicode_filename-2/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/_file/test/test_file.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/_file/test/test_file.py Mon Jul 12 22:58:54 2010 @@ -125,6 +125,10 @@ assert type(res) is str f.close() + def test_unicode_filename(self): + f = self.file(self.temppath + u'\xe9', "w") + f.close() + def test_oserror_has_filename(self): try: f = self.file("file that is clearly not there") Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Mon Jul 12 22:58:54 2010 @@ -1,5 +1,6 @@ from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong from pypy.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 @@ -23,16 +24,22 @@ def gettext(self): return self.space.unicode_w(self.w_obj) + at specialize.arg(1) +def dispatch_filename(space, w_fname, callable): + if space.isinstance_w(w_fname, space.w_unicode): + fname = FileEncoder(space, w_fname) + return callable(fname) + else: + fname = space.str_w(w_fname) + return callable(fname) + def open(space, w_fname, flag, mode=0777): """Open a file (for low level IO). Return a file descriptor (a small integer).""" try: - if space.isinstance_w(w_fname, space.w_unicode): - fname = FileEncoder(space, w_fname) - fd = rposix.open(fname, flag, mode) - else: - fname = space.str_w(w_fname) - fd = rposix.open(fname, flag, mode) + fd = dispatch_filename( + space, w_fname, + lambda fname: rposix.open(fname, flag, mode)) except OSError, e: raise wrap_oserror2(space, e, w_fname) return space.wrap(fd) Modified: pypy/branch/unicode_filename-2/pypy/rlib/streamio.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/streamio.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/streamio.py Mon Jul 12 22:58:54 2010 @@ -38,7 +38,9 @@ # import os, sys +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong, intmask +from pypy.rlib import rposix from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC O_BINARY = getattr(os, "O_BINARY", 0) @@ -71,6 +73,7 @@ return s.join(string.split(c)) + at specialize.argtype(0) def open_file_as_stream(path, mode="r", buffering=-1): os_flags, universal, reading, writing, basemode, binary = decode_mode(mode) stream = open_path_helper(path, os_flags, basemode == "a") @@ -89,9 +92,10 @@ return construct_stream_tower(stream, buffering, universal, reading, writing, binary) + at specialize.argtype(0) def open_path_helper(path, os_flags, append): # XXX for now always return DiskFile - fd = os.open(path, os_flags, 0666) + fd = rposix.open(path, os_flags, 0666) if append: try: os.lseek(fd, 0, 2) From afa at codespeak.net Mon Jul 12 23:48:33 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Jul 2010 23:48:33 +0200 (CEST) Subject: [pypy-svn] r76167 - pypy/branch/unicode_filename-2/pypy/module/posix Message-ID: <20100712214833.93059282BF4@codespeak.net> Author: afa Date: Mon Jul 12 23:48:30 2010 New Revision: 76167 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Log: One translation fix Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Mon Jul 12 23:48:30 2010 @@ -24,7 +24,7 @@ def gettext(self): return self.space.unicode_w(self.w_obj) - at specialize.arg(1) + at specialize.arg(2) def dispatch_filename(space, w_fname, callable): if space.isinstance_w(w_fname, space.w_unicode): fname = FileEncoder(space, w_fname) From afa at codespeak.net Mon Jul 12 23:56:49 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Jul 2010 23:56:49 +0200 (CEST) Subject: [pypy-svn] r76168 - in pypy/branch/unicode_filename-2/pypy/module: _file posix Message-ID: <20100712215649.680AB282BF4@codespeak.net> Author: afa Date: Mon Jul 12 23:56:33 2010 New Revision: 76168 Modified: pypy/branch/unicode_filename-2/pypy/module/_file/interp_file.py pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Log: Another attempt, lambda is probably not RPython at all. Modified: pypy/branch/unicode_filename-2/pypy/module/_file/interp_file.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/_file/interp_file.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/_file/interp_file.py Mon Jul 12 23:56:33 2010 @@ -85,9 +85,8 @@ self.direct_close() self.w_name = w_name self.check_mode_ok(mode) - stream = dispatch_filename( - self.space, w_name, - lambda name: streamio.open_file_as_stream(name, mode, buffering)) + stream = dispatch_filename(streamio.open_file_as_stream)( + self.space, w_name, mode, buffering) fd = stream.try_to_find_file_descriptor() self.fdopenstream(stream, fd, mode) Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Mon Jul 12 23:56:33 2010 @@ -24,22 +24,23 @@ def gettext(self): return self.space.unicode_w(self.w_obj) - at specialize.arg(2) -def dispatch_filename(space, w_fname, callable): - if space.isinstance_w(w_fname, space.w_unicode): - fname = FileEncoder(space, w_fname) - return callable(fname) - else: - fname = space.str_w(w_fname) - return callable(fname) + at specialize.memo() +def dispatch_filename(func): + def dispatch(space, w_fname, *args): + if space.isinstance_w(w_fname, space.w_unicode): + fname = FileEncoder(space, w_fname) + return func(fname, *args) + else: + fname = space.str_w(w_fname) + return func(fname, *args) + return dispatch def open(space, w_fname, flag, mode=0777): """Open a file (for low level IO). Return a file descriptor (a small integer).""" try: - fd = dispatch_filename( - space, w_fname, - lambda fname: rposix.open(fname, flag, mode)) + fd = dispatch_filename(rposix.open)( + space, w_fname, flag, mode) except OSError, e: raise wrap_oserror2(space, e, w_fname) return space.wrap(fd) From benjamin at codespeak.net Tue Jul 13 01:01:38 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 13 Jul 2010 01:01:38 +0200 (CEST) Subject: [pypy-svn] r76169 - in pypy/branch/fast-forward/pypy/module/_ssl: . test Message-ID: <20100712230138.30A5D282BFC@codespeak.net> Author: benjamin Date: Tue Jul 13 01:01:35 2010 New Revision: 76169 Modified: pypy/branch/fast-forward/pypy/module/_ssl/__init__.py pypy/branch/fast-forward/pypy/module/_ssl/app_ssl.py pypy/branch/fast-forward/pypy/module/_ssl/interp_ssl.py pypy/branch/fast-forward/pypy/module/_ssl/test/test_ssl.py Log: sslerror is now nicely named Modified: pypy/branch/fast-forward/pypy/module/_ssl/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_ssl/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/_ssl/__init__.py Tue Jul 13 01:01:35 2010 @@ -7,7 +7,7 @@ appleveldefs = { '__doc__': 'app_ssl.__doc__', - 'sslerror': 'app_ssl.sslerror', + 'SSLError': 'app_ssl.SSLError', } @classmethod Modified: pypy/branch/fast-forward/pypy/module/_ssl/app_ssl.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_ssl/app_ssl.py (original) +++ pypy/branch/fast-forward/pypy/module/_ssl/app_ssl.py Tue Jul 13 01:01:35 2010 @@ -1,4 +1,4 @@ -class sslerror(Exception): +class SSLError(Exception): pass __doc__ = """Implementation module for SSL socket operations. Modified: pypy/branch/fast-forward/pypy/module/_ssl/interp_ssl.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_ssl/interp_ssl.py (original) +++ pypy/branch/fast-forward/pypy/module/_ssl/interp_ssl.py Tue Jul 13 01:01:35 2010 @@ -155,7 +155,7 @@ def ssl_error(space, msg): w_module = space.getbuiltinmodule('_ssl') - w_exception = space.getattr(w_module, space.wrap('sslerror')) + w_exception = space.getattr(w_module, space.wrap('SSLError')) return OperationError(w_exception, space.wrap(msg)) def _init_ssl(): Modified: pypy/branch/fast-forward/pypy/module/_ssl/test/test_ssl.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_ssl/test/test_ssl.py (original) +++ pypy/branch/fast-forward/pypy/module/_ssl/test/test_ssl.py Tue Jul 13 01:01:35 2010 @@ -12,7 +12,7 @@ def test_sslerror(self): import _ssl - assert issubclass(_ssl.sslerror, Exception) + assert issubclass(_ssl.SSLError, Exception) def test_constants(self): import _ssl From hakanardo at codespeak.net Tue Jul 13 08:01:35 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 13 Jul 2010 08:01:35 +0200 (CEST) Subject: [pypy-svn] r76170 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100713060135.1475E282B9E@codespeak.net> Author: hakanardo Date: Tue Jul 13 08:01:33 2010 New Revision: 76170 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py Log: compiling Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Jul 13 08:01:33 2010 @@ -212,19 +212,26 @@ new = space.int_w(w_new) oldlen = self.len self.setlen(self.len + new) - + + i = 0 try: - if (isinstance(w_seq, W_ListObject) or - isinstance(w_seq, W_TupleObject)): - for i in range(new): + if isinstance(w_seq, W_ListObject): + while i < new: + item = self.item_w(w_seq.wrappeditems[i]) + self.buffer[oldlen + i ] = item + i += 1 + elif isinstance(w_seq, W_TupleObject): + while i < new: item = self.item_w(w_seq.wrappeditems[i]) self.buffer[oldlen + i ] = item + i += 1 else: getitem = space.getattr(w_seq, space.wrap('__getitem__')) - for i in range(new): + while i < new: w_item = space.call_function(getitem, space.wrap(i)) item=self.item_w(w_item) self.buffer[oldlen + i ] = item + i += 1 except OperationError: self.setlen(oldlen + i) raise Modified: pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py Tue Jul 13 08:01:33 2010 @@ -11,7 +11,9 @@ return l if True: - img=array('d', '\x00'*640*480*8) + img=array('d', '\x00'*640*480) + #img=array('d', [0]*640*480) + #img=array('d', (0,))*(640*480) else: img=simple_array(640*480) From hakanardo at codespeak.net Tue Jul 13 08:28:55 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 13 Jul 2010 08:28:55 +0200 (CEST) Subject: [pypy-svn] r76171 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100713062855.490E4282B9E@codespeak.net> Author: hakanardo Date: Tue Jul 13 08:28:53 2010 New Revision: 76171 Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: speedups and bugfix Modified: pypy/branch/interplevel-array/pypy/module/array/app_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/app_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/app_array.py Tue Jul 13 08:28:53 2010 @@ -125,7 +125,7 @@ if not self._isarray(other): return NotImplemented if self.typecode == 'c': - return buffer(self._data) == buffer(other._data) + return buffer(self) == buffer(other) else: return self.tolist() == other.tolist() @@ -133,7 +133,7 @@ if not self._isarray(other): return NotImplemented if self.typecode == 'c': - return buffer(self._data) != buffer(other._data) + return buffer(self) != buffer(other) else: return self.tolist() != other.tolist() @@ -141,7 +141,7 @@ if not self._isarray(other): return NotImplemented if self.typecode == 'c': - return buffer(self._data) < buffer(other._data) + return buffer(self) < buffer(other) else: return self.tolist() < other.tolist() @@ -149,7 +149,7 @@ if not self._isarray(other): return NotImplemented if self.typecode == 'c': - return buffer(self._data) > buffer(other._data) + return buffer(self) > buffer(other) else: return self.tolist() > other.tolist() @@ -157,7 +157,7 @@ if not self._isarray(other): return NotImplemented if self.typecode == 'c': - return buffer(self._data) <= buffer(other._data) + return buffer(self) <= buffer(other) else: return self.tolist() <= other.tolist() @@ -165,7 +165,7 @@ if not self._isarray(other): return NotImplemented if self.typecode == 'c': - return buffer(self._data) >= buffer(other._data) + return buffer(self) >= buffer(other) else: return self.tolist() >= other.tolist() Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Jul 13 08:28:53 2010 @@ -245,15 +245,27 @@ if mytype.typecode != w_iterable.typecode: msg = "can only extend with array of same kind" raise OperationError(space.w_TypeError, space.wrap(msg)) - w_iterator = space.iter(w_iterable) - while True: - try: - w_item = space.next(w_iterator) - except OperationError, e: - if not e.match(space, space.w_StopIteration): - raise - break - self.descr_append(w_item) + if isinstance(w_iterable, W_Array): + oldlen = self.len + new = w_iterable.len + self.setlen(self.len + new) + for i in range(new): + self.buffer[oldlen + i] = w_iterable.buffer[i] + else: + assert False + elif (isinstance(w_iterable, W_ListObject) or + isinstance(w_iterable, W_TupleObject)): + self.descr_fromsequence(w_iterable) + else: + w_iterator = space.iter(w_iterable) + while True: + try: + w_item = space.next(w_iterator) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break + self.descr_append(w_item) descr_extend.unwrap_spec = ['self', W_Root] def descr_setslice(self, w_idx, w_item): Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Tue Jul 13 08:28:53 2010 @@ -44,6 +44,10 @@ assert a[2] == 'c' assert len(a) == 3 + b = self.array('c', a) + assert a == b + raises(TypeError, self.array, 'i', a) + def test_value_range(self): values = (-129, 128, -128, 127, 0, 255, -1, 256, -32768, 32767, -32769, 32768, 65535, 65536, @@ -218,6 +222,9 @@ a = self.array('b', (1, 2)) assert len(a) == 2 and a[0] == 1 and a[1] == 2 + a.extend(a) + assert repr(a) == "array('b', [1, 2, 1, 2])" + def test_fromunicode(self): raises(ValueError, self.array('i').fromunicode, unicode('hi')) a = self.array('u') @@ -432,51 +439,55 @@ assert repr(a) == "array('i', [8, 2, 9, 7])" def test_compare(self): - a = self.array('i', [1, 2, 3]) - b = self.array('i', [1, 2, 3]) - c = self.array('i', [1, 3, 2]) - - assert (a == a) is True - assert (a == b) is True - assert (b == a) is True - assert (a == c) is False - assert (c == a) is False - - assert (a != a) is False - assert (a != b) is False - assert (b != a) is False - assert (a != c) is True - assert (c != a) is True - - assert (a < a) is False - assert (a < b) is False - assert (b < a) is False - assert (a < c) is True - assert (c < a) is False - - assert (a > a) is False - assert (a > b) is False - assert (b > a) is False - assert (a > c) is False - assert (c > a) is True - - assert (a <= a) is True - assert (a <= b) is True - assert (b <= a) is True - assert (a <= c) is True - assert (c <= a) is False - - assert (a >= a) is True - assert (a >= b) is True - assert (b >= a) is True - assert (a >= c) is False - assert (c >= a) is True - - assert cmp(a, a) == 0 - assert cmp(a, b) == 0 - assert cmp(a, c) < 0 - assert cmp(b, a) == 0 - assert cmp(c, a) > 0 + for v1,v2,tt in (([1, 2, 3], [1, 3, 2], 'bhilBHIL'), + ('abc', 'acb', 'c'), + (unicode('abc'), unicode('acb'), 'u')): + for t in tt: + a = self.array(t, v1) + b = self.array(t, v1) + c = self.array(t, v2) + + assert (a == a) is True + assert (a == b) is True + assert (b == a) is True + assert (a == c) is False + assert (c == a) is False + + assert (a != a) is False + assert (a != b) is False + assert (b != a) is False + assert (a != c) is True + assert (c != a) is True + + assert (a < a) is False + assert (a < b) is False + assert (b < a) is False + assert (a < c) is True + assert (c < a) is False + + assert (a > a) is False + assert (a > b) is False + assert (b > a) is False + assert (a > c) is False + assert (c > a) is True + + assert (a <= a) is True + assert (a <= b) is True + assert (b <= a) is True + assert (a <= c) is True + assert (c <= a) is False + + assert (a >= a) is True + assert (a >= b) is True + assert (b >= a) is True + assert (a >= c) is False + assert (c >= a) is True + + assert cmp(a, a) == 0 + assert cmp(a, b) == 0 + assert cmp(a, c) < 0 + assert cmp(b, a) == 0 + assert cmp(c, a) > 0 def test_reduce(self): import pickle From afa at codespeak.net Tue Jul 13 08:56:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Jul 2010 08:56:43 +0200 (CEST) Subject: [pypy-svn] r76172 - pypy/branch/unicode_filename-2/pypy/rpython/module Message-ID: <20100713065643.D594F282B9E@codespeak.net> Author: afa Date: Tue Jul 13 08:56:42 2010 New Revision: 76172 Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Try to fix translation on Windows Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Tue Jul 13 08:56:42 2010 @@ -6,6 +6,7 @@ # might be found in doc/rffi.txt import os, sys, errno +import py from pypy.rpython.module.support import ll_strcpy, OOSupport from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rlib.rarithmetic import r_longlong @@ -27,38 +28,49 @@ from pypy.rpython.annlowlevel import llstr from pypy.rlib import rgc from pypy.rlib.objectmodel import keepalive_until_here, specialize -from pypy.rlib.unroll import unrolling_iterable def registering_unicode_version(func, nbargs, argnums, condition=True): """ Registers an implementation of func() which directly accepts unicode strings. Replaces the corresponding function in pypy.rlib.rposix. + @nbargs is the arity of the function + @argnums is the list of positions of unicode strings """ + if not condition: + registering(None, condition=False) + def unicodefunc(*args): return func(*args) argnum = argnums[0] - unrolling_args = unrolling_iterable(enumerate([i in argnums - for i in range(nbargs)])) - func_name = func.__name__ - rposix_func = getattr(rposix, func_name) + arglist = ['arg%d' % (i,) for i in range(nbargs)] + transformed_arglist = arglist[:] + for i in argnums: + transformed_arglist[i] = transformed_arglist[i] + '.gettext()' + + args = ', '.join(arglist) + transformed_args = ', '.join(transformed_arglist) + main_arg = 'arg%d' % (argnum,) - @specialize.argtype(*argnums) - @func_renamer(func.__name__) - def new_func(*args): - if isinstance(args[argnum], str): - return func(*args) + func_name = func.__name__ + source = py.code.Source(""" + def %(func_name)s(%(args)s): + if isinstance(%(main_arg)s, str): + return func(%(args)s) else: - real_args = () - for i, isunicode in unrolling_args: - if isunicode: - real_args += (args[i].gettext(),) - else: - real_args += (args[i],) - return unicodefunc(*real_args) - if condition: - setattr(rposix, func_name, new_func) + return unicodefunc(%(transformed_args)s) + """ % locals()) + miniglobals = {'func' : func, + 'unicodefunc': unicodefunc, + '__name__': __name__, # for module name propagation + } + exec source.compile() in miniglobals + new_func = miniglobals[func_name] + new_func = specialize.argtype(*argnums)(new_func) + + # Monkeypatch the function in pypy.rlib.rposix + setattr(rposix, func_name, new_func) return registering(unicodefunc, condition=condition) From fijall at gmail.com Tue Jul 13 10:10:29 2010 From: fijall at gmail.com (Maciej Fijalkowski) Date: Tue, 13 Jul 2010 10:10:29 +0200 Subject: [pypy-svn] r76170 - in pypy/branch/interplevel-array/pypy/module/array: . test In-Reply-To: <20100713060135.1475E282B9E@codespeak.net> References: <20100713060135.1475E282B9E@codespeak.net> Message-ID: Hey. There is space.fixedview, which is fast for tuple and space.listview, which is fast for list. On Tue, Jul 13, 2010 at 8:01 AM, wrote: > Author: hakanardo > Date: Tue Jul 13 08:01:33 2010 > New Revision: 76170 > > Modified: > ? pypy/branch/interplevel-array/pypy/module/array/interp_array.py > ? pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py > Log: > compiling > > Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py > ============================================================================== > --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py ? ? (original) > +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py ? ? Tue Jul 13 08:01:33 2010 > @@ -212,19 +212,26 @@ > ? ? ? ? ? ? new = space.int_w(w_new) > ? ? ? ? ? ? oldlen = self.len > ? ? ? ? ? ? self.setlen(self.len + new) > - > + > + ? ? ? ? ? ?i = 0 > ? ? ? ? ? ? try: > - ? ? ? ? ? ? ? ?if (isinstance(w_seq, W_ListObject) or > - ? ? ? ? ? ? ? ? ? ?isinstance(w_seq, W_TupleObject)): > - ? ? ? ? ? ? ? ? ? ?for i in range(new): > + ? ? ? ? ? ? ? ?if isinstance(w_seq, W_ListObject): > + ? ? ? ? ? ? ? ? ? ?while i < new: > + ? ? ? ? ? ? ? ? ? ? ? ?item = self.item_w(w_seq.wrappeditems[i]) > + ? ? ? ? ? ? ? ? ? ? ? ?self.buffer[oldlen + i ] = item > + ? ? ? ? ? ? ? ? ? ? ? ?i += 1 > + ? ? ? ? ? ? ? ?elif isinstance(w_seq, W_TupleObject): > + ? ? ? ? ? ? ? ? ? ?while i < new: > ? ? ? ? ? ? ? ? ? ? ? ? item = self.item_w(w_seq.wrappeditems[i]) > ? ? ? ? ? ? ? ? ? ? ? ? self.buffer[oldlen + i ] = item > + ? ? ? ? ? ? ? ? ? ? ? ?i += 1 > ? ? ? ? ? ? ? ? else: > ? ? ? ? ? ? ? ? ? ? getitem = space.getattr(w_seq, space.wrap('__getitem__')) > - ? ? ? ? ? ? ? ? ? ?for i in range(new): > + ? ? ? ? ? ? ? ? ? ?while i < new: > ? ? ? ? ? ? ? ? ? ? ? ? w_item = space.call_function(getitem, space.wrap(i)) > ? ? ? ? ? ? ? ? ? ? ? ? item=self.item_w(w_item) > ? ? ? ? ? ? ? ? ? ? ? ? self.buffer[oldlen + i ] = item > + ? ? ? ? ? ? ? ? ? ? ? ?i += 1 > ? ? ? ? ? ? except OperationError: > ? ? ? ? ? ? ? ? self.setlen(oldlen + i) > ? ? ? ? ? ? ? ? raise > > Modified: pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py > ============================================================================== > --- pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py ? ? ?(original) > +++ pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py ? ? ?Tue Jul 13 08:01:33 2010 > @@ -11,7 +11,9 @@ > ? ? return l > > ?if True: > - ? ?img=array('d', '\x00'*640*480*8) > + ? ?img=array('d', '\x00'*640*480) > + ? ?#img=array('d', [0]*640*480) > + ? ?#img=array('d', (0,))*(640*480) > ?else: > ? ? img=simple_array(640*480) > > _______________________________________________ > pypy-svn mailing list > pypy-svn at codespeak.net > http://codespeak.net/mailman/listinfo/pypy-svn > From afa at codespeak.net Tue Jul 13 10:42:57 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Jul 2010 10:42:57 +0200 (CEST) Subject: [pypy-svn] r76173 - pypy/branch/unicode_filename-2/pypy/module/posix/test Message-ID: <20100713084257.1176A282B9E@codespeak.net> Author: afa Date: Tue Jul 13 10:42:56 2010 New Revision: 76173 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py Log: Allow the test to run on top of the TinyObjSpace (used by the -A option) Modified: pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py Tue Jul 13 10:42:56 2010 @@ -33,7 +33,7 @@ os.stat_float_times(True) # Initialize sys.filesystemencoding - space.call_function(space.sys.get('getfilesystemencoding')) + space.call_method(space.getbuiltinmodule('sys'), 'getfilesystemencoding') def need_sparse_files(): if sys.platform == 'darwin': From fijal at codespeak.net Tue Jul 13 11:43:34 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Jul 2010 11:43:34 +0200 (CEST) Subject: [pypy-svn] r76174 - pypy/trunk/lib-python/modified-2.5.2 Message-ID: <20100713094334.CF897282B9E@codespeak.net> Author: fijal Date: Tue Jul 13 11:43:33 2010 New Revision: 76174 Modified: pypy/trunk/lib-python/modified-2.5.2/site.py Log: sys.prefix is there only if the path was succesfully found. Don't crash at 'import site' in the case we are using the builtin path Modified: pypy/trunk/lib-python/modified-2.5.2/site.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/site.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/site.py Tue Jul 13 11:43:33 2010 @@ -175,7 +175,7 @@ def addsitepackages(known_paths): """Add site-packages to sys.path, in a PyPy-specific way.""" - if hasattr(sys, 'pypy_version_info'): + if hasattr(sys, 'pypy_version_info') and hasattr(sys, 'prefix'): from distutils.sysconfig import get_python_lib sitedir = get_python_lib(standard_lib=False) if os.path.isdir(sitedir): From arigo at codespeak.net Tue Jul 13 12:16:40 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 13 Jul 2010 12:16:40 +0200 (CEST) Subject: [pypy-svn] r76175 - in pypy/branch/reflex-support/pypy/jit: codewriter codewriter/test metainterp Message-ID: <20100713101640.5146B282B9E@codespeak.net> Author: arigo Date: Tue Jul 13 12:16:38 2010 New Revision: 76175 Modified: pypy/branch/reflex-support/pypy/jit/codewriter/effectinfo.py pypy/branch/reflex-support/pypy/jit/codewriter/test/test_effectinfo.py pypy/branch/reflex-support/pypy/jit/metainterp/optimizeopt.py pypy/branch/reflex-support/pypy/jit/metainterp/pyjitpl.py Log: Extend the EffectInfo to distinguish between the cases EF_PURE etc., even for cases of general escaping where we don't have any list of fields. Modified: pypy/branch/reflex-support/pypy/jit/codewriter/effectinfo.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/codewriter/effectinfo.py (original) +++ pypy/branch/reflex-support/pypy/jit/codewriter/effectinfo.py Tue Jul 13 12:16:38 2010 @@ -18,14 +18,14 @@ def __new__(cls, readonly_descrs_fields, write_descrs_fields, write_descrs_arrays, extraeffect=EF_CAN_RAISE): - key = (frozenset(readonly_descrs_fields), - frozenset(write_descrs_fields), - frozenset(write_descrs_arrays), + key = (_frozenset_or_none(readonly_descrs_fields), + _frozenset_or_none(write_descrs_fields), + _frozenset_or_none(write_descrs_arrays), extraeffect) if key in cls._cache: return cls._cache[key] result = object.__new__(cls) - result.readonly_descrs_fields = readonly_descrs_fields + result.readonly_descrs_fields = readonly_descrs_fields # or None result.write_descrs_fields = write_descrs_fields result.write_descrs_arrays = write_descrs_arrays result.extraeffect = extraeffect @@ -35,11 +35,16 @@ def check_forces_virtual_or_virtualizable(self): return self.extraeffect >= self.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE +def _frozenset_or_none(x): + if x is None: return None + return frozenset(x) + + def effectinfo_from_writeanalyze(effects, cpu, extraeffect=EffectInfo.EF_CAN_RAISE): from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set: - return None + return EffectInfo(None, None, None, extraeffect) readonly_descrs_fields = [] # readonly_descrs_arrays = [] --- not enabled for now write_descrs_fields = [] Modified: pypy/branch/reflex-support/pypy/jit/codewriter/test/test_effectinfo.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/codewriter/test/test_effectinfo.py (original) +++ pypy/branch/reflex-support/pypy/jit/codewriter/test/test_effectinfo.py Tue Jul 13 12:16:38 2010 @@ -2,6 +2,8 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype from pypy.jit.codewriter.effectinfo import effectinfo_from_writeanalyze +from pypy.jit.codewriter.effectinfo import EffectInfo +from pypy.translator.backendopt.writeanalyze import top_set class FakeCPU: def fielddescrof(self, T, fieldname): @@ -77,3 +79,19 @@ assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays + +def test_no_effectinfo(): + effectinfo = effectinfo_from_writeanalyze(top_set, None, + EffectInfo.EF_CANNOT_RAISE) + assert effectinfo.readonly_descrs_fields is None + assert effectinfo.write_descrs_fields is None + assert effectinfo.write_descrs_arrays is None + assert effectinfo.extraeffect == EffectInfo.EF_CANNOT_RAISE + # + effectinfo2 = effectinfo_from_writeanalyze(top_set, None, + EffectInfo.EF_CANNOT_RAISE) + assert effectinfo2 is effectinfo + # + effectinfo3 = effectinfo_from_writeanalyze(top_set, None, + EffectInfo.EF_PURE) + assert effectinfo3.extraeffect == EffectInfo.EF_PURE Modified: pypy/branch/reflex-support/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/reflex-support/pypy/jit/metainterp/optimizeopt.py Tue Jul 13 12:16:38 2010 @@ -1098,14 +1098,10 @@ opnum == rop.DEBUG_MERGE_POINT): return assert opnum != rop.CALL_PURE - if (opnum == rop.CALL or - opnum == rop.CALL_MAY_FORCE or - opnum == rop.CALL_ASSEMBLER): - if opnum == rop.CALL_ASSEMBLER: - effectinfo = None - else: - effectinfo = op.descr.get_extra_info() - if effectinfo is not None: + if opnum == rop.CALL or opnum == rop.CALL_MAY_FORCE: + effectinfo = op.descr.get_extra_info() + if (effectinfo is not None and + effectinfo.readonly_descrs_fields is not None): # XXX we can get the wrong complexity here, if the lists # XXX stored on effectinfo are large for fielddescr in effectinfo.readonly_descrs_fields: @@ -1128,8 +1124,9 @@ # of virtualref_info and virtualizable_info are not gcptrs. return self.force_all_lazy_setfields() - elif op.is_final() or (not we_are_translated() and - op.opnum < 0): # escape() operations + elif (opnum == rop.CALL_ASSEMBLER or + op.is_final() or + (not we_are_translated() and op.opnum < 0)): # escape() ops self.force_all_lazy_setfields() self.clean_caches() Modified: pypy/branch/reflex-support/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/reflex-support/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/reflex-support/pypy/jit/metainterp/pyjitpl.py Tue Jul 13 12:16:38 2010 @@ -1063,10 +1063,13 @@ assert i == len(allboxes) # effectinfo = descr.get_extra_info() - if (effectinfo is None or - effectinfo.extraeffect == - effectinfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE or - assembler_call_token is not None): + if effectinfo is None: + effect = -1 + else: + effect = effectinfo.extraeffect + if (assembler_call_token is not None or + effectinfo is None or + effect == effectinfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE): # residual calls require attention to keep virtualizables in-sync self.metainterp.clear_exception() self.metainterp.vable_and_vrefs_before_residual_call() @@ -1083,7 +1086,6 @@ self.metainterp.handle_possible_exception() return resbox else: - effect = effectinfo.extraeffect if effect == effectinfo.EF_CANNOT_RAISE: return self.execute_varargs(rop.CALL, allboxes, descr, False) elif effect == effectinfo.EF_PURE: From afa at codespeak.net Tue Jul 13 13:23:37 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Jul 2010 13:23:37 +0200 (CEST) Subject: [pypy-svn] r76176 - pypy/branch/unicode_filename-2/pypy/translator/platform Message-ID: <20100713112337.F06F1282B9E@codespeak.net> Author: afa Date: Tue Jul 13 13:23:36 2010 New Revision: 76176 Modified: pypy/branch/unicode_filename-2/pypy/translator/platform/__init__.py pypy/branch/unicode_filename-2/pypy/translator/platform/darwin.py pypy/branch/unicode_filename-2/pypy/translator/platform/posix.py pypy/branch/unicode_filename-2/pypy/translator/platform/windows.py Log: In Makefiles, use relative path for the response file. Modified: pypy/branch/unicode_filename-2/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/translator/platform/__init__.py (original) +++ pypy/branch/unicode_filename-2/pypy/translator/platform/__init__.py Tue Jul 13 13:23:36 2010 @@ -161,7 +161,7 @@ return (library_dirs + self.link_flags + export_flags + link_files + list(eci.link_extra) + libraries) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if eci.export_symbols: raise ValueError("This platform does not support export symbols") return [] Modified: pypy/branch/unicode_filename-2/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/translator/platform/darwin.py (original) +++ pypy/branch/unicode_filename-2/pypy/translator/platform/darwin.py Tue Jul 13 13:23:36 2010 @@ -56,7 +56,7 @@ include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -65,6 +65,9 @@ for sym in eci.export_symbols: f.write("_%s\n" % (sym,)) f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["-Wl,-exported_symbols_list,%s" % (response_file,)] class Darwin_i386(Darwin): Modified: pypy/branch/unicode_filename-2/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/translator/platform/posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/translator/platform/posix.py Tue Jul 13 13:23:36 2010 @@ -39,7 +39,7 @@ def _link_args_from_eci(self, eci, standalone): return Platform._link_args_from_eci(self, eci, standalone) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -50,6 +50,9 @@ f.write("%s;\n" % (sym,)) f.write("};") f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["-Wl,--export-dynamic,--version-script=%s" % (response_file,)] def _link(self, cc, ofiles, link_args, standalone, exe_name): @@ -90,7 +93,7 @@ if shared: linkflags = self._args_for_shared(linkflags) - linkflags += self._exportsymbols_link_flags(eci) + linkflags += self._exportsymbols_link_flags(eci, relto=path) if shared: libname = exe_name.new(ext='').basename Modified: pypy/branch/unicode_filename-2/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/translator/platform/windows.py (original) +++ pypy/branch/unicode_filename-2/pypy/translator/platform/windows.py Tue Jul 13 13:23:36 2010 @@ -141,7 +141,7 @@ # Windows needs to resolve all symbols even for DLLs return super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -150,6 +150,9 @@ for sym in eci.export_symbols: f.write("/EXPORT:%s\n" % (sym,)) f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["@%s" % (response_file,)] def _compile_c_file(self, cc, cfile, compile_args): @@ -219,7 +222,7 @@ if shared: linkflags = self._args_for_shared(linkflags) + [ '/EXPORT:$(PYPY_MAIN_FUNCTION)'] - linkflags += self._exportsymbols_link_flags(eci) + linkflags += self._exportsymbols_link_flags(eci, relto=path) if shared: so_name = exe_name.new(purebasename='lib' + exe_name.purebasename, From afa at codespeak.net Tue Jul 13 13:46:52 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Jul 2010 13:46:52 +0200 (CEST) Subject: [pypy-svn] r76177 - pypy/branch/unicode_filename-2/py/_cmdline Message-ID: <20100713114652.85E8E282B9E@codespeak.net> Author: afa Date: Tue Jul 13 13:46:50 2010 New Revision: 76177 Modified: pypy/branch/unicode_filename-2/py/_cmdline/pysvnwcrevert.py Log: py.path cannot handle filenames with unicode chars unencodable by filesystemencoding. Work around this in py.svnwcrevert: at least buildbot should be able to remove a directory containing such files. Modified: pypy/branch/unicode_filename-2/py/_cmdline/pysvnwcrevert.py ============================================================================== --- pypy/branch/unicode_filename-2/py/_cmdline/pysvnwcrevert.py (original) +++ pypy/branch/unicode_filename-2/py/_cmdline/pysvnwcrevert.py Tue Jul 13 13:46:50 2010 @@ -16,11 +16,16 @@ the following 'svn up' won't just crash. """ -import sys, py +import sys, os, py def kill(p, root): print('< %s' % (p.relto(root),)) - p.remove(rec=1) + try: + p.remove(rec=1) + except py.error.ENOTDIR: + if sys.platform != 'win32': + raise + os.system("rmdir /s/q %s" % (p,)) def svnwcrevert(path, root=None, precious=[]): if root is None: From fijal at codespeak.net Tue Jul 13 13:48:53 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Jul 2010 13:48:53 +0200 (CEST) Subject: [pypy-svn] r76178 - pypy/trunk/pypy/translator/platform Message-ID: <20100713114853.09AB9282B9E@codespeak.net> Author: fijal Date: Tue Jul 13 13:48:52 2010 New Revision: 76178 Modified: pypy/trunk/pypy/translator/platform/__init__.py pypy/trunk/pypy/translator/platform/darwin.py pypy/trunk/pypy/translator/platform/posix.py pypy/trunk/pypy/translator/platform/windows.py Log: Merge 76177 from unicode-filename-2 Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Tue Jul 13 13:48:52 2010 @@ -161,7 +161,7 @@ return (library_dirs + self.link_flags + export_flags + link_files + list(eci.link_extra) + libraries) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if eci.export_symbols: raise ValueError("This platform does not support export symbols") return [] Modified: pypy/trunk/pypy/translator/platform/darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/darwin.py (original) +++ pypy/trunk/pypy/translator/platform/darwin.py Tue Jul 13 13:48:52 2010 @@ -56,7 +56,7 @@ include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -65,6 +65,9 @@ for sym in eci.export_symbols: f.write("_%s\n" % (sym,)) f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["-Wl,-exported_symbols_list,%s" % (response_file,)] class Darwin_i386(Darwin): Modified: pypy/trunk/pypy/translator/platform/posix.py ============================================================================== --- pypy/trunk/pypy/translator/platform/posix.py (original) +++ pypy/trunk/pypy/translator/platform/posix.py Tue Jul 13 13:48:52 2010 @@ -39,7 +39,7 @@ def _link_args_from_eci(self, eci, standalone): return Platform._link_args_from_eci(self, eci, standalone) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -50,6 +50,9 @@ f.write("%s;\n" % (sym,)) f.write("};") f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["-Wl,--export-dynamic,--version-script=%s" % (response_file,)] def _link(self, cc, ofiles, link_args, standalone, exe_name): @@ -90,7 +93,7 @@ if shared: linkflags = self._args_for_shared(linkflags) - linkflags += self._exportsymbols_link_flags(eci) + linkflags += self._exportsymbols_link_flags(eci, relto=path) if shared: libname = exe_name.new(ext='').basename Modified: pypy/trunk/pypy/translator/platform/windows.py ============================================================================== --- pypy/trunk/pypy/translator/platform/windows.py (original) +++ pypy/trunk/pypy/translator/platform/windows.py Tue Jul 13 13:48:52 2010 @@ -141,7 +141,7 @@ # Windows needs to resolve all symbols even for DLLs return super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -150,6 +150,9 @@ for sym in eci.export_symbols: f.write("/EXPORT:%s\n" % (sym,)) f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["@%s" % (response_file,)] def _compile_c_file(self, cc, cfile, compile_args): @@ -219,7 +222,7 @@ if shared: linkflags = self._args_for_shared(linkflags) + [ '/EXPORT:$(PYPY_MAIN_FUNCTION)'] - linkflags += self._exportsymbols_link_flags(eci) + linkflags += self._exportsymbols_link_flags(eci, relto=path) if shared: so_name = exe_name.new(purebasename='lib' + exe_name.purebasename, From hakanardo at codespeak.net Tue Jul 13 14:07:38 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 13 Jul 2010 14:07:38 +0200 (CEST) Subject: [pypy-svn] r76179 - in pypy/branch/interplevel-array/pypy/jit: metainterp metainterp/test tl Message-ID: <20100713120738.0576A282B9E@codespeak.net> Author: hakanardo Date: Tue Jul 13 14:07:37 2010 New Revision: 76179 Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py pypy/branch/interplevel-array/pypy/jit/metainterp/resoperation.py pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py Log: guard removal by rewriting boolean expressions Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py Tue Jul 13 14:07:37 2010 @@ -1,7 +1,7 @@ from pypy.jit.metainterp.history import Box, BoxInt, LoopToken, BoxFloat,\ ConstFloat from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj, REF -from pypy.jit.metainterp.resoperation import rop, ResOperation +from pypy.jit.metainterp.resoperation import rop, ResOperation, opboolinvers, opboolreflex from pypy.jit.metainterp import jitprof from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.specnode import SpecNode, NotSpecNode, ConstantSpecNode @@ -611,12 +611,46 @@ assert oldop.opnum == op.opnum self.make_equal_to(op.result, self.getvalue(oldop.result)) return + elif self.find_rewriteable_constant(op, args): + return else: self.pure_operations[args] = op # otherwise, the operation remains self.emit_operation(op) + + def find_rewriteable_constant(self, op, args): + try: + oldopnum = opboolinvers[op.opnum] + targ = args[0:2] + [ConstInt(oldopnum)] + oldop = self.pure_operations.get(targ, None) + if oldop is not None and oldop.descr is op.descr: + value = self.getvalue(oldop.result) + if value.is_constant(): + if value.box is CONST_1: + self.make_constant(op.result, CONST_0) + return True + elif value.box is CONST_0: + self.make_constant(op.result, CONST_1) + return True + except KeyError: + pass + + try: + oldopnum = opboolreflex[op.opnum] + targ = args[1::-1] + [ConstInt(oldopnum)] + oldop = self.pure_operations.get(targ, None) + if oldop is not None and oldop.descr is op.descr: + value = self.getvalue(oldop.result) + if value.is_constant(): + self.make_constant(op.result, value.box) + return True + except KeyError: + pass + + return False + def optimize_JUMP(self, op): orgop = self.loop.operations[-1] exitargs = [] Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/resoperation.py Tue Jul 13 14:07:37 2010 @@ -274,3 +274,41 @@ setup(__name__ == '__main__') # print out the table when run directly del _oplist + +opboolinvers = { + rop.INT_EQ: rop.INT_NE, + rop.INT_NE: rop.INT_EQ, + rop.INT_LT: rop.INT_GE, + rop.INT_GE: rop.INT_LT, + rop.INT_GT: rop.INT_LE, + rop.INT_LE: rop.INT_GT, + + rop.FLOAT_EQ: rop.FLOAT_NE, + rop.FLOAT_NE: rop.FLOAT_EQ, + rop.FLOAT_LT: rop.FLOAT_GE, + rop.FLOAT_GE: rop.FLOAT_LT, + rop.FLOAT_GT: rop.FLOAT_LE, + rop.FLOAT_LE: rop.FLOAT_GT, + + rop.PTR_EQ: rop.PTR_NE, + rop.PTR_NE: rop.PTR_EQ, + } + +opboolreflex = { + rop.INT_EQ: rop.INT_EQ, + rop.INT_NE: rop.INT_NE, + rop.INT_LT: rop.INT_GT, + rop.INT_GE: rop.INT_LE, + rop.INT_GT: rop.INT_LT, + rop.INT_LE: rop.INT_GE, + + rop.FLOAT_EQ: rop.FLOAT_EQ, + rop.FLOAT_NE: rop.FLOAT_NE, + rop.FLOAT_LT: rop.FLOAT_GT, + rop.FLOAT_GE: rop.FLOAT_LE, + rop.FLOAT_GT: rop.FLOAT_LT, + rop.FLOAT_LE: rop.FLOAT_GE, + + rop.PTR_EQ: rop.PTR_EQ, + rop.PTR_NE: rop.PTR_NE, + } Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py Tue Jul 13 14:07:37 2010 @@ -364,6 +364,57 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_constant_boolrewrite1(self): + ops = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + i2 = int_ge(i0, 0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite2(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_le(i0, 0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite3(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_lt(0, i0) + guard_true(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + def test_remove_consecutive_guard_value_constfold(self): ops = """ [] Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py Tue Jul 13 14:07:37 2010 @@ -37,14 +37,24 @@ ## t2 = time.time() ## print t2 - t1 -from array import array -def f(img): +## from array import array +## def f(img): +## i=0 +## sa=0 +## while i<4: +## sa+=img[i] +## i+=1 +## return sa + +## img=array('i',(1,2,3,4)) +## print f(img) + +def f(): + a=7 i=0 - sa=0 while i<4: - sa+=img[i] + if i<0: break + if i<0: break i+=1 - return sa -img=array('i',(1,2,3,4)) -print f(img) +f() From antocuni at codespeak.net Tue Jul 13 14:29:23 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 13 Jul 2010 14:29:23 +0200 (CEST) Subject: [pypy-svn] r76180 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100713122923.E3DB4282B9E@codespeak.net> Author: antocuni Date: Tue Jul 13 14:29:22 2010 New Revision: 76180 Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py Log: fix translation Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Tue Jul 13 14:29:22 2010 @@ -39,8 +39,9 @@ def execute(self, space, func, cppthis, num_args, args): from pypy.module.cppyy import interp_cppyy - result = capi.c_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) - return interp_cppyy.W_CCPInstance(self.cpptype, result) + long_result = capi.c_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) + ptr_result = rffi.cast(rffi.VOIDP, long_result) + return interp_cppyy.W_CCPInstance(self.cpptype, ptr_result) def get_executor(space, name): From afa at codespeak.net Tue Jul 13 15:28:45 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Jul 2010 15:28:45 +0200 (CEST) Subject: [pypy-svn] r76181 - in pypy/branch/unicode_filename-2/pypy: module/posix rlib rlib/test rpython/module rpython/module/test Message-ID: <20100713132845.8E791282B9E@codespeak.net> Author: afa Date: Tue Jul 13 15:28:42 2010 New Revision: 76181 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/rlib/rposix.py pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os_stat.py pypy/branch/unicode_filename-2/pypy/rpython/module/test/test_ll_os_stat.py Log: Add unicode support to os.stat() and os.lstat() Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Tue Jul 13 15:28:42 2010 @@ -183,7 +183,7 @@ return build_stat_result(space, st) fstat.unwrap_spec = [ObjSpace, "c_int"] -def stat(space, path): +def stat(space, w_path): """Perform a stat system call on the given path. Return an object with (at least) the following attributes: st_mode @@ -199,22 +199,22 @@ """ try: - st = os.stat(path) + st = dispatch_filename(rposix.stat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -stat.unwrap_spec = [ObjSpace, 'path'] +stat.unwrap_spec = [ObjSpace, W_Root] -def lstat(space, path): +def lstat(space, w_path): "Like stat(path), but do no follow symbolic links." try: - st = os.lstat(path) + st = dispatch_filename(rposix.lstat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -lstat.unwrap_spec = [ObjSpace, 'path'] +lstat.unwrap_spec = [ObjSpace, W_Root] class StatState(object): def __init__(self, space): Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Tue Jul 13 15:28:42 2010 @@ -53,3 +53,17 @@ return os.open(path, flags, mode) else: return os.open(path.encode(), flags, mode) + + at specialize.argtype(0) +def stat(path): + if isinstance(path, str): + return os.stat(path) + else: + return os.stat(path.encode()) + + at specialize.argtype(0) +def lstat(path): + if isinstance(path, str): + return os.lstat(path) + else: + return os.lstat(path.encode()) Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Tue Jul 13 15:28:42 2010 @@ -7,13 +7,12 @@ return ''.join(s.chars) class TestPosixUnicode: - def test_access(self): - ufilename = (unicode(udir.join('test_open')) + + def setup_class(cls): + cls.ufilename = (unicode(udir.join('test_open')) + u'\u65e5\u672c.txt') # "Japan" - f = file(ufilename, 'w') + f = file(cls.ufilename, 'w') f.write("test") f.close() - filename = str(udir.join('test_open')) class UnicodeWithEncoding: def __init__(self, unistr): @@ -27,11 +26,12 @@ def gettext(self): return self.unistr - path = UnicodeWithEncoding(ufilename) + cls.path = UnicodeWithEncoding(cls.ufilename) + def test_access(self): def f(): try: - fd = rposix.open(path, os.O_RDONLY, 0777) + fd = rposix.open(self.path, os.O_RDONLY, 0777) try: text = os.read(fd, 50) return text @@ -41,3 +41,9 @@ return '' assert ll_to_string(interpret(f, [])) == "test" + + def test_stat(self): + def f(): + return rposix.stat(self.path).st_mtime + + assert interpret(f, []) == os.stat(self.ufilename).st_mtime Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Tue Jul 13 15:28:42 2010 @@ -1485,17 +1485,27 @@ @registering(os.fstat) def register_os_fstat(self): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('fstat') + return ll_os_stat.register_stat_variant('fstat') @registering(os.stat) def register_os_stat(self): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('stat') + return ll_os_stat.register_stat_variant('stat') @registering(os.lstat) def register_os_lstat(self): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('lstat') + return ll_os_stat.register_stat_variant('lstat') + + @registering_unicode_version(os.stat, 1, [0], sys.platform=='win32') + def register_os_stat_unicode(self): + from pypy.rpython.module import ll_os_stat + return ll_os_stat.register_stat_variant_unicode('stat') + + @registering_unicode_version(os.lstat, 1, [0], sys.platform=='win32') + def register_os_lstat_unicode(self): + from pypy.rpython.module import ll_os_stat + return ll_os_stat.register_stat_variant_unicode('lstat') # ------------------------------- os.W* --------------------------------- Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os_stat.py Tue Jul 13 15:28:42 2010 @@ -7,11 +7,12 @@ from pypy.tool.pairtype import pairtype from pypy.tool.sourcetools import func_with_new_name from pypy.rpython import extregistry -from pypy.rpython.extfunc import register_external +from pypy.rpython.extfunc import register_external, extdef from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.annlowlevel import hlstr @@ -271,8 +272,8 @@ [ARG1, STAT_STRUCT], rffi.INT, compilation_info=compilation_info) - register_external( - getattr(os, name), [s_arg], s_StatResult, + return extdef( + [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), llimpl=func_with_new_name(posix_stat_llimpl, 'os_%s_llimpl' % (name,)), @@ -281,13 +282,24 @@ ) else: # See Win32 implementation below - register_external( - getattr(os, name), [s_arg], s_StatResult, + return extdef( + [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), llimpl=func_with_new_name(globals()['win32_%s_llimpl' % (name,)], 'os_%s_llimpl' % (name,)), ) +def register_stat_variant_unicode(name): + assert name in ('stat', 'lstat') + + # See Win32 implementation below + return extdef( + [unicode], s_StatResult, + "ll_os.ll_os_%s" % (name,), + llimpl=func_with_new_name(globals()['win32_wstat_llimpl'], + 'os_wstat_llimpl') + ) + # ____________________________________________________________ if sys.platform == 'win32': # The CRT of Windows has a number of flaws wrt. its stat() implementation: @@ -344,7 +356,7 @@ ('ftLastAccessTime', rwin32.FILETIME), ('ftLastWriteTime', rwin32.FILETIME)]) - WIN32_FIND_DATA = platform.Struct( + WIN32_FIND_DATAA = platform.Struct( 'WIN32_FIND_DATAA', # Only interesting fields [('dwFileAttributes', rwin32.DWORD), @@ -353,17 +365,33 @@ ('ftCreationTime', rwin32.FILETIME), ('ftLastAccessTime', rwin32.FILETIME), ('ftLastWriteTime', rwin32.FILETIME)]) + WIN32_FIND_DATAW = platform.Struct( + 'WIN32_FIND_DATAW', + # Only interesting fields + [('dwFileAttributes', rwin32.DWORD), + ('nFileSizeHigh', rwin32.DWORD), + ('nFileSizeLow', rwin32.DWORD), + ('ftCreationTime', rwin32.FILETIME), + ('ftLastAccessTime', rwin32.FILETIME), + ('ftLastWriteTime', rwin32.FILETIME)]) globals().update(platform.configure(CConfig)) GET_FILEEX_INFO_LEVELS = rffi.ULONG # an enumeration - GetFileAttributesEx = rffi.llexternal( + GetFileAttributesExA = rffi.llexternal( 'GetFileAttributesExA', [rffi.CCHARP, GET_FILEEX_INFO_LEVELS, lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)], rwin32.BOOL, calling_conv='win') + GetFileAttributesExW = rffi.llexternal( + 'GetFileAttributesExW', + [rffi.CWCHARP, GET_FILEEX_INFO_LEVELS, + lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)], + rwin32.BOOL, + calling_conv='win') + GetFileInformationByHandle = rffi.llexternal( 'GetFileInformationByHandle', [rwin32.HANDLE, lltype.Ptr(BY_HANDLE_FILE_INFORMATION)], @@ -376,9 +404,15 @@ rwin32.DWORD, calling_conv='win') - FindFirstFile = rffi.llexternal( + FindFirstFileA = rffi.llexternal( 'FindFirstFileA', - [rffi.CCHARP, lltype.Ptr(WIN32_FIND_DATA)], + [rffi.CCHARP, lltype.Ptr(WIN32_FIND_DATAA)], + rwin32.HANDLE, + calling_conv='win') + + FindFirstFileW = rffi.llexternal( + 'FindFirstFileW', + [rffi.CWCHARP, lltype.Ptr(WIN32_FIND_DATAW)], rwin32.HANDLE, calling_conv='win') @@ -455,25 +489,37 @@ return make_stat_result(result) + @specialize.ll() + def findfirstfile(l_path): + if lltype.typeOf(l_path) == rffi.CWCHARP: + filedata = lltype.malloc(WIN32_FIND_DATAW, flavor='raw') + return FindFirstFileW(l_path, filedata), filedata + else: + filedata = lltype.malloc(WIN32_FIND_DATAA, flavor='raw') + return FindFirstFileA(l_path, filedata), filedata + + @specialize.ll() def attributes_from_dir(l_path, data): - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - hFindFile = FindFirstFile(l_path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - return 0 - FindClose(hFindFile) - data.c_dwFileAttributes = filedata.c_dwFileAttributes - rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) - rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) - rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) - data.c_nFileSizeHigh = filedata.c_nFileSizeHigh - data.c_nFileSizeLow = filedata.c_nFileSizeLow - return 1 + hFindFile, filedata = findfirstfile(l_path) + try: + if hFindFile == rwin32.INVALID_HANDLE_VALUE: + return 0 + FindClose(hFindFile) + data.c_dwFileAttributes = filedata.c_dwFileAttributes + rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) + rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) + rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) + data.c_nFileSizeHigh = filedata.c_nFileSizeHigh + data.c_nFileSizeLow = filedata.c_nFileSizeLow + return 1 + finally: + lltype.free(filedata, flavor='raw') def win32_stat_llimpl(path): data = lltype.malloc(WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') try: l_path = rffi.str2charp(path) - res = GetFileAttributesEx(l_path, GetFileExInfoStandard, data) + res = GetFileAttributesExA(l_path, GetFileExInfoStandard, data) errcode = rwin32.GetLastError() if res == 0: if errcode == ERROR_SHARING_VIOLATION: @@ -487,6 +533,23 @@ lltype.free(data, flavor='raw') win32_lstat_llimpl = win32_stat_llimpl + def win32_wstat_llimpl(path): + data = lltype.malloc(WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') + try: + l_path = rffi.unicode2wcharp(path) + res = GetFileAttributesExW(l_path, GetFileExInfoStandard, data) + errcode = rwin32.GetLastError() + if res == 0: + if errcode == ERROR_SHARING_VIOLATION: + res = attributes_from_dir(l_path, data) + errcode = rwin32.GetLastError() + rffi.free_wcharp(l_path) + if res == 0: + raise WindowsError(errcode, "os_stat failed") + return attribute_data_to_stat(data) + finally: + lltype.free(data, flavor='raw') + def win32_fstat_llimpl(fd): handle = rwin32._get_osfhandle(fd) Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/test/test_ll_os_stat.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/test/test_ll_os_stat.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/test/test_ll_os_stat.py Tue Jul 13 15:28:42 2010 @@ -9,8 +9,11 @@ def test_stat(self): stat = ll_os_stat.win32_stat_llimpl + wstat = ll_os_stat.win32_wstat_llimpl def check(f): - assert stat(f).st_mtime == os.stat(f).st_mtime + expected = os.stat(f).st_mtime + assert stat(f).st_mtime == expected + assert wstat(unicode(f)).st_mtime == expected check('c:/') check('c:/temp') From afa at codespeak.net Tue Jul 13 15:36:41 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Jul 2010 15:36:41 +0200 (CEST) Subject: [pypy-svn] r76182 - in pypy/branch/unicode_filename-2/pypy: module/posix rlib rlib/test rpython/module Message-ID: <20100713133641.C5C7C282B9E@codespeak.net> Author: afa Date: Tue Jul 13 15:36:40 2010 New Revision: 76182 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/rlib/rposix.py pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Unicode version of os.unlink() This should help the unit tests to remove their temp files... Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Tue Jul 13 15:36:40 2010 @@ -302,21 +302,21 @@ return space.wrap(rc) system.unwrap_spec = [ObjSpace, str] -def unlink(space, path): +def unlink(space, w_path): """Remove a file (same as remove(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -unlink.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +unlink.unwrap_spec = [ObjSpace, W_Root] -def remove(space, path): +def remove(space, w_path): """Remove a file (same as unlink(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -remove.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +remove.unwrap_spec = [ObjSpace, W_Root] def _getfullpathname(space, path): """helper for ntpath.abspath """ Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Tue Jul 13 15:36:40 2010 @@ -67,3 +67,10 @@ return os.lstat(path) else: return os.lstat(path.encode()) + + at specialize.argtype(0) +def unlink(path): + if isinstance(path, str): + return os.unlink(path) + else: + return os.unlink(path.encode()) Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Tue Jul 13 15:36:40 2010 @@ -7,10 +7,10 @@ return ''.join(s.chars) class TestPosixUnicode: - def setup_class(cls): - cls.ufilename = (unicode(udir.join('test_open')) + - u'\u65e5\u672c.txt') # "Japan" - f = file(cls.ufilename, 'w') + def setup_method(self, method): + self.ufilename = (unicode(udir.join('test_open')) + + u'\u65e5\u672c.txt') # "Japan" + f = file(self.ufilename, 'w') f.write("test") f.close() @@ -26,7 +26,7 @@ def gettext(self): return self.unistr - cls.path = UnicodeWithEncoding(cls.ufilename) + self.path = UnicodeWithEncoding(self.ufilename) def test_access(self): def f(): @@ -47,3 +47,10 @@ return rposix.stat(self.path).st_mtime assert interpret(f, []) == os.stat(self.ufilename).st_mtime + + def test_unlink(self): + def f(): + return rposix.unlink(self.path) + + interpret(f, []) + assert not os.path.exists(self.ufilename) Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Tue Jul 13 15:36:40 2010 @@ -1306,6 +1306,18 @@ return extdef([str], s_None, llimpl=unlink_llimpl, export_name="ll_os.ll_os_unlink") + @registering_unicode_version(os.unlink, 1, [0], sys.platform=='win32') + def register_os_unlink_unicode(self): + os_wunlink = self.llexternal(underscore_on_windows+'wunlink', [rffi.CWCHARP], rffi.INT) + + def wunlink_llimpl(pathname): + res = rffi.cast(lltype.Signed, os_wunlink(pathname)) + if res < 0: + raise OSError(rposix.get_errno(), "os_unlink failed") + + return extdef([unicode], s_None, llimpl=wunlink_llimpl, + export_name="ll_os.ll_os_wunlink") + @registering(os.chdir) def register_os_chdir(self): os_chdir = self.llexternal(underscore_on_windows+'chdir', [rffi.CCHARP], rffi.INT) From fijal at codespeak.net Tue Jul 13 15:37:54 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Jul 2010 15:37:54 +0200 (CEST) Subject: [pypy-svn] r76183 - in pypy/extradoc/pypy.org: . source Message-ID: <20100713133754.A454F282B9E@codespeak.net> Author: fijal Date: Tue Jul 13 15:37:52 2010 New Revision: 76183 Modified: pypy/extradoc/pypy.org/compat.html pypy/extradoc/pypy.org/download.html pypy/extradoc/pypy.org/source/download.txt Log: Windows binaries uploaded Modified: pypy/extradoc/pypy.org/compat.html ============================================================================== --- pypy/extradoc/pypy.org/compat.html (original) +++ pypy/extradoc/pypy.org/compat.html Tue Jul 13 15:37:52 2010 @@ -51,7 +51,7 @@ language, passing Python test suite (with minor modifications that were already accepted in the main python in newer versions). It supports most of the commonly used Python standard library modules; details below.

    -

    PyPy has alpha-level support of the CPython C API, however, as of 1.3 +

    PyPy has alpha-level support for the CPython C API, however, as of 1.3 release this feature is not yet complete. Most libraries will require a bit of effort to work, but there are known success stories. Check out PyPy blog for updates.

    @@ -131,4 +131,4 @@ - + \ No newline at end of file Modified: pypy/extradoc/pypy.org/download.html ============================================================================== --- pypy/extradoc/pypy.org/download.html (original) +++ pypy/extradoc/pypy.org/download.html Tue Jul 13 15:37:52 2010 @@ -50,7 +50,6 @@

    Here are the various binaries of PyPy 1.3 that we provide for x86 Linux, Mac OS/X or Windows. This release improves over 1.2 in terms of stability of the JIT. It also

    -

    Windows binaries are not ready yet

    • Download

      If your CPU is a 64-bit machine and you want to translate a 32-bit version of PyPy yourself, here are hints.

      @@ -121,17 +120,15 @@

      The Windows and the native 64-bit versions both need testing and careful reviewing; contact us!

      -
      +

      “Stackless” version

      Provides Stackless extensions, as well as greenlets. These binaries work on 32-bit x86 (IA-32) CPUs as well as x86-64 CPUs in the 32-bit compatibility mode.

      -
      -

      System Message: WARNING/2 ([dynamic-text], line 120)

      -Bullet list ends without a blank line; unexpected unindent.

      It is not possible right now to combine Stackless features with the JIT.

      If your CPU is a 64-bit machine and you want to translate a 32-bit version of PyPy yourself, here are hints.

      Modified: pypy/extradoc/pypy.org/source/download.txt ============================================================================== --- pypy/extradoc/pypy.org/source/download.txt (original) +++ pypy/extradoc/pypy.org/source/download.txt Tue Jul 13 15:37:52 2010 @@ -10,8 +10,6 @@ Mac OS/X or Windows. This release improves over 1.2 in terms of stability of the JIT. It also -**Windows binaries are not ready yet** - .. class:: download_menu * Download @@ -36,10 +34,11 @@ * `Linux binary`__ * `Mac OS/X binary`__ -* Windows binary - not yet ready +* `Windows binary`__ .. __: http://pypy.org/download/pypy-1.3-linux.tar.bz2 .. __: http://pypy.org/download/pypy-1.3-osx.tar.bz2 +.. __: http://pypy.org/download/pypy-1.3-win32.zip If your CPU is really old, it may not have SSE2. In this case, you need to translate_ yourself with the option ``--jit-backend=x86-without-sse2``. @@ -70,11 +69,12 @@ * `Linux binary`__ * `Mac OS/X binary`__ * `Linux 64bit binary`__ -* Windows binary - not yet ready +* `Windows binary`__ .. __: http://pypy.org/download/pypy-1.3-linux-nojit.tar.bz2 .. __: http://pypy.org/download/pypy-1.3-osx-nojit.tar.bz2 .. __: http://pypy.org/download/pypy-1.3-linux64-nojit.tar.bz2 +.. __: http://pypy.org/download/pypy-1.3-win32-nojit.zip If your CPU is a 64-bit machine and you want to translate_ a 32-bit version of PyPy yourself, `here are hints`_. @@ -122,7 +122,10 @@ in the 32-bit compatibility mode. * `Linux binary`__ +* `Windows binary`__ + .. __: http://pypy.org/download/pypy-1.3-linux-stackless.tar.bz2 +.. __: http://pypy.org/download/pypy-1.3-win32-stackless.zip It is not possible right now to combine Stackless features with the JIT. From hakanardo at codespeak.net Tue Jul 13 15:51:18 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 13 Jul 2010 15:51:18 +0200 (CEST) Subject: [pypy-svn] r76184 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100713135118.6928D282B9E@codespeak.net> Author: hakanardo Date: Tue Jul 13 15:51:13 2010 New Revision: 76184 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: less special cases Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Jul 13 15:51:13 2010 @@ -215,23 +215,9 @@ i = 0 try: - if isinstance(w_seq, W_ListObject): - while i < new: - item = self.item_w(w_seq.wrappeditems[i]) - self.buffer[oldlen + i ] = item - i += 1 - elif isinstance(w_seq, W_TupleObject): - while i < new: - item = self.item_w(w_seq.wrappeditems[i]) - self.buffer[oldlen + i ] = item - i += 1 - else: - getitem = space.getattr(w_seq, space.wrap('__getitem__')) - while i < new: - w_item = space.call_function(getitem, space.wrap(i)) - item=self.item_w(w_item) - self.buffer[oldlen + i ] = item - i += 1 + for w_i in space.listview(w_seq): + self.buffer[oldlen + i] = self.item_w(w_i) + i += 1 except OperationError: self.setlen(oldlen + i) raise @@ -257,15 +243,8 @@ isinstance(w_iterable, W_TupleObject)): self.descr_fromsequence(w_iterable) else: - w_iterator = space.iter(w_iterable) - while True: - try: - w_item = space.next(w_iterator) - except OperationError, e: - if not e.match(space, space.w_StopIteration): - raise - break - self.descr_append(w_item) + for w_i in space.listview(w_iterable): + self.descr_append(w_i) descr_extend.unwrap_spec = ['self', W_Root] def descr_setslice(self, w_idx, w_item): From fijal at codespeak.net Tue Jul 13 15:58:00 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Jul 2010 15:58:00 +0200 (CEST) Subject: [pypy-svn] r76185 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100713135800.BFB61282B9E@codespeak.net> Author: fijal Date: Tue Jul 13 15:57:59 2010 New Revision: 76185 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: Whitespace and pep-8 issues Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Jul 13 15:57:59 2010 @@ -21,7 +21,7 @@ #src=file(re.sub('.pyc$', '.py', mod.__file__)).read() #app = ApplevelClass(src) import app_array - f=getattr(app_array,n) + f = getattr(app_array,n) args = f.func_code.co_varnames[0:f.func_code.co_argcount] args = ', '.join(['space'] + ['w_'+s for s in args]) appfn = app.interphook(n) @@ -94,8 +94,10 @@ return val class W_Array(W_ArrayBase): - itemsize=mytype.bytes - typecode=mytype.typecode + + itemsize = mytype.bytes + typecode = mytype.typecode + def __init__(self, space): self.space = space self.len = 0 @@ -121,22 +123,24 @@ msg = None if mytype.signed: if item < -1 << (mytype.bytes * 8 - 1): - msg = 'signed %d-byte integer is less than minimum' % mytype.bytes + msg = ('signed %d-byte integer is less than minimum' % + mytype.bytes) elif item > (1 << (mytype.bytes * 8 - 1)) - 1: - msg = 'signed %d-byte integer is greater than maximum' % mytype.bytes + msg = ('signed %d-byte integer is greater than maximum' + % mytype.bytes) else: if item < 0: - msg = 'unsigned %d-byte integer is less than minimum' % mytype.bytes + msg = ('unsigned %d-byte integer is less than minimum' + % mytype.bytes) elif item > (1 << (mytype.bytes * 8)) - 1: - msg = 'unsigned %d-byte integer is greater than maximum' % mytype.bytes + msg = ('unsigned %d-byte integer is greater' + ' than maximum' % mytype.bytes) if msg is not None: raise OperationError(space.w_OverflowError, space.wrap(msg)) return rffi.cast(mytype.itemtype, item) - def __del__(self): self.setlen(0) - def setlen(self, size): if size > 0: @@ -157,7 +161,6 @@ return self.space.wrap(self.len) descr_len.unwrap_spec = ['self'] - def descr_getslice(self, w_idx): space = self.space start, stop, step = space.decode_index(w_idx, self.len) @@ -198,14 +201,12 @@ return self.descr_getslice(w_idx) descr_getitem.unwrap_spec = ['self', W_Root] - def descr_append(self, w_x): x = self.item_w(w_x) self.setlen(self.len + 1) self.buffer[self.len - 1] = x descr_append.unwrap_spec = ['self', W_Root] - def descr_fromsequence(self, w_seq): space = self.space w_new = space.call_function(space.getattr(w_seq, space.wrap('__len__'))) From hakanardo at codespeak.net Tue Jul 13 17:16:04 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 13 Jul 2010 17:16:04 +0200 (CEST) Subject: [pypy-svn] r76187 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100713151604.D99B5282B9E@codespeak.net> Author: hakanardo Date: Tue Jul 13 17:16:03 2010 New Revision: 76187 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: Less reallocing, better sequence itteration Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Jul 13 17:16:03 2010 @@ -3,17 +3,14 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.rpython.lltypesystem import lltype, rffi from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root, ApplevelClass -from pypy.objspace.std.listobject import W_ListObject -from pypy.objspace.std.tupleobject import W_TupleObject from pypy.rlib.jit import dont_look_inside from pypy.rlib import rgc from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rstruct.runpack import runpack -import os, types, re -path, _ = os.path.split(__file__) -app_array = os.path.join(path, 'app_array.py') -app = ApplevelClass(file(app_array).read()) +import py +app_array = py.path.local(__file__).dirpath().join('app_array.py') +app = ApplevelClass(file(str(app_array)).read()) def appmethod(n,allappfn={}): if not allappfn.has_key(n): @@ -101,6 +98,7 @@ def __init__(self, space): self.space = space self.len = 0 + self.allocated = 0 self.buffer = lltype.nullptr(mytype.arraytype) def item_w(self, w_item): @@ -143,17 +141,24 @@ self.setlen(0) def setlen(self, size): - if size > 0: - #new_buffer = lltype.malloc(mytype.arraytype, size) - new_buffer = lltype.malloc(mytype.arraytype, size, flavor='raw') - for i in range(min(size,self.len)): - new_buffer[i] = self.buffer[i] - else: - assert size == 0 - new_buffer = lltype.nullptr(mytype.arraytype) - if self.buffer != lltype.nullptr(mytype.arraytype): - lltype.free(self.buffer, flavor='raw') - self.buffer = new_buffer + if size > self.allocated or size < self.allocated/2: + if size > 0: + if size < 9: + some = 3 + else: + some = 6 + some += size >> 3 + self.allocated = size + some + new_buffer = lltype.malloc(mytype.arraytype, self.allocated, flavor='raw') + for i in range(min(size,self.len)): + new_buffer[i] = self.buffer[i] + else: + assert size == 0 + self.allocated = 0 + new_buffer = lltype.nullptr(mytype.arraytype) + if self.buffer != lltype.nullptr(mytype.arraytype): + lltype.free(self.buffer, flavor='raw') + self.buffer = new_buffer self.len = size setlen.unwrap_spec = ['self', int] @@ -209,19 +214,29 @@ def descr_fromsequence(self, w_seq): space = self.space - w_new = space.call_function(space.getattr(w_seq, space.wrap('__len__'))) - new = space.int_w(w_new) oldlen = self.len - self.setlen(self.len + new) + try: + new = space.int_w(space.len(w_seq)) + self.setlen(self.len + new) + except OperationError: + pass i = 0 try: - for w_i in space.listview(w_seq): - self.buffer[oldlen + i] = self.item_w(w_i) + if mytype.typecode == 'u': + myiter = space.unpackiterable + else: + myiter = space.listview + for w_i in myiter(w_seq): + if oldlen + i < self.len: + self.buffer[oldlen + i] = self.item_w(w_i) + else: + self.descr_append(w_i) i += 1 except OperationError: self.setlen(oldlen + i) raise + self.setlen(oldlen + i) descr_fromsequence.unwrap_spec = ['self', W_Root] @@ -240,12 +255,8 @@ self.buffer[oldlen + i] = w_iterable.buffer[i] else: assert False - elif (isinstance(w_iterable, W_ListObject) or - isinstance(w_iterable, W_TupleObject)): - self.descr_fromsequence(w_iterable) else: - for w_i in space.listview(w_iterable): - self.descr_append(w_i) + self.descr_fromsequence(w_iterable) descr_extend.unwrap_spec = ['self', W_Root] def descr_setslice(self, w_idx, w_item): Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Tue Jul 13 17:16:03 2010 @@ -563,7 +563,29 @@ for i in a: b.append(i) assert repr(b) == "array('i', [1, 2, 3])" - + + def test_lying_iterable(self): + class lier(object): + def __init__(self, n): + self.n = n + def __len__(self): + return 3 + def next(self): + self.n -= 1 + if self.n < 0: raise StopIteration + return self.n + def __iter__(self): + return self + + assert len(lier(2)) == 3 + assert len(tuple(lier(2))) == 2 + a = self.array('i', lier(2)) + assert repr(a) == "array('i', [1, 0])" + + assert len(lier(5)) == 3 + assert len(tuple(lier(5))) == 5 + a = self.array('i', lier(5)) + assert repr(a) == "array('i', [4, 3, 2, 1, 0])" #FIXME #def test_type(self): @@ -602,5 +624,5 @@ assert bi[0] != 0 assert bi[1] == 3 data = self.rffi.charp2string(bi[0]) - assert data == 'Hi!' + assert data[0:3] == 'Hi!' From wlav at codespeak.net Tue Jul 13 17:26:12 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Tue, 13 Jul 2010 17:26:12 +0200 (CEST) Subject: [pypy-svn] r76188 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100713152612.5BE6E282B9E@codespeak.net> Author: wlav Date: Tue Jul 13 17:26:09 2010 New Revision: 76188 Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Log: Allow methods to return instances. Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Tue Jul 13 17:26:09 2010 @@ -44,7 +44,7 @@ if w_cppinstance is not None: w_obj = w_cppinstance obj = space.interpclass_w(w_obj) - if isinstance(obj, interp_cppyy.W_CCPInstance): + if isinstance(obj, interp_cppyy.W_CPPInstance): if capi.c_is_subtype(obj.cppclass.handle, self.cpptype.handle): return obj.rawobject raise OperationError(space.w_TypeError, Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Tue Jul 13 17:26:09 2010 @@ -32,6 +32,7 @@ result = capi.charp2str_free(ccpresult) return space.wrap(result) + class InstancePtrExecutor(FunctionExecutor): _immutable_ = True def __init__(self, space, cpptype): @@ -41,7 +42,7 @@ from pypy.module.cppyy import interp_cppyy long_result = capi.c_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) ptr_result = rffi.cast(rffi.VOIDP, long_result) - return interp_cppyy.W_CCPInstance(self.cpptype, ptr_result) + return interp_cppyy.W_CPPInstance(self.cpptype, ptr_result) def get_executor(space, name): Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Tue Jul 13 17:26:09 2010 @@ -166,7 +166,7 @@ except Exception, e: capi.c_deallocate(self.cpptype.handle, newthis) raise - return W_CCPInstance(self.cpptype, newthis) + return W_CPPInstance(self.cpptype, newthis) class W_CPPOverload(Wrappable): @@ -180,6 +180,12 @@ def is_static(self): return self.space.wrap(isinstance(self.functions[0], CPPFunction)) + def get_returntype(self): + try: + return self.space.wrap(self.functions[0].executor.cpptype.name) + except AttributeError: + return None + @jit.unroll_safe def call(self, cppthis, args_w): space = self.space @@ -201,6 +207,7 @@ W_CPPOverload.typedef = TypeDef( 'CPPOverload', is_static = interp2app(W_CPPOverload.is_static, unwrap_spec=['self']), + get_returntype = interp2app(W_CPPOverload.get_returntype, unwrap_spec=['self']), ) @@ -268,8 +275,7 @@ ) - -class W_CCPInstance(Wrappable): +class W_CPPInstance(Wrappable): _immutable_ = True def __init__(self, cppclass, rawobject): self.space = cppclass.space @@ -290,8 +296,8 @@ capi.c_destruct(self.cppclass.handle, self.rawobject) self.rawobject = NULL_VOIDP -W_CCPInstance.typedef = TypeDef( +W_CPPInstance.typedef = TypeDef( 'CPPInstance', - invoke = interp2app(W_CCPInstance.invoke, unwrap_spec=['self', str, 'args_w']), - destruct = interp2app(W_CCPInstance.destruct, unwrap_spec=['self']), + invoke = interp2app(W_CPPInstance.invoke, unwrap_spec=['self', str, 'args_w']), + destruct = interp2app(W_CPPInstance.destruct, unwrap_spec=['self']), ) Modified: pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py Tue Jul 13 17:26:09 2010 @@ -5,25 +5,37 @@ class CppyyClass(type): pass +class CppyyObject(object): + def __init__(self, *args): + print '__init__ called', args + self._cppinstance = self._cppyyclass.construct(*args) + + def destruct(self): + self._cppinstance.destruct() + + def make_static_function(cpptype, name): def method(*args): return cpptype.invoke(name, *args) method.__name__ = name return staticmethod(method) -def make_method(name): - def method(self, *args): - return self._cppinstance.invoke(name, *args) - method.__name__ = name - return method +def make_method(name, rettype): + if rettype is None: # return builtin type + def method(self, *args): + return self._cppinstance.invoke(name, *args) + method.__name__ = name + return method + else: # return instance + def method(self, *args): + result = self._cppinstance.invoke(name, *args) + if not result is None: + bound_result = object.__new__(get_cppclass(rettype)) + bound_result._cppinstance = result + return bound_result + method.__name__ = name + return method -class CppyyObject(object): - def __init__(self, *args): - self._cppinstance = self._cppyyclass.construct(*args) - - def destruct(self): - self._cppinstance.destruct() - _existing_classes = {} def get_cppclass(name): @@ -42,7 +54,7 @@ if cppol.is_static(): d[f] = make_static_function(cpptype, f) else: - d[f] = make_method(f) + d[f] = make_method(f, cppol.get_returntype()) pycpptype = CppyyClass(name, (CppyyObject,), d) Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Tue Jul 13 17:26:09 2010 @@ -164,7 +164,7 @@ e.invoke("setPayload", pl); assert round(pl.invoke("getData")-50., 8) == 0 - e.invoke("cyclePayload", pl); + pl = e.invoke("cyclePayload", pl); assert round(pl.invoke("getData")-50., 8) == 0 e.destruct() Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Tue Jul 13 17:26:09 2010 @@ -137,9 +137,9 @@ e.staticSetPayload(pl, 45.) assert pl.getData() == 45. - e.setPayload(pl); + e.setPayload(pl) assert round(pl.getData()-14., 8) == 0 - e.cyclePayload(pl); + pl = e.cyclePayload(pl) assert round(pl.getData()-14., 8) == 0 pl.destruct() From afa at codespeak.net Tue Jul 13 18:41:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Jul 2010 18:41:22 +0200 (CEST) Subject: [pypy-svn] r76189 - in pypy/branch/unicode_filename-2/pypy: module/posix rlib rlib/test rpython/module Message-ID: <20100713164122.65028282BE8@codespeak.net> Author: afa Date: Tue Jul 13 18:41:20 2010 New Revision: 76189 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/rlib/rposix.py pypy/branch/unicode_filename-2/pypy/rlib/rwin32.py pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Implement win32 unicode version of os.listdir() Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Tue Jul 13 18:41:20 2010 @@ -434,7 +434,7 @@ unsetenv.unwrap_spec = [ObjSpace, str] -def listdir(space, dirname): +def listdir(space, w_dirname): """Return a list containing the names of the entries in the directory. \tpath: path of directory to list @@ -442,12 +442,18 @@ The list is in arbitrary order. It does not include the special entries '.' and '..' even if they are present in the directory.""" try: - result = os.listdir(dirname) + if space.isinstance_w(w_dirname, space.w_unicode): + dirname = FileEncoder(space, w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] + else: + dirname = space.str_w(w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] except OSError, e: - raise wrap_oserror(space, e, dirname) - result_w = [space.wrap(s) for s in result] + raise wrap_oserror2(space, e, w_dirname) return space.newlist(result_w) -listdir.unwrap_spec = [ObjSpace, str] +listdir.unwrap_spec = [ObjSpace, W_Root] def pipe(space): "Create a pipe. Returns (read_end, write_end)." Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Tue Jul 13 18:41:20 2010 @@ -74,3 +74,10 @@ return os.unlink(path) else: return os.unlink(path.encode()) + + at specialize.argtype(0) +def listdir(dirname): + if isinstance(dirname, str): + return os.listdir(dirname) + else: + return os.listdir(dirname.encode()) Modified: pypy/branch/unicode_filename-2/pypy/rlib/rwin32.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rwin32.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rwin32.py Tue Jul 13 18:41:20 2010 @@ -31,6 +31,7 @@ DWORD = rffi_platform.SimpleType("DWORD", rffi.UINT) BOOL = rffi_platform.SimpleType("BOOL", rffi.LONG) BYTE = rffi_platform.SimpleType("BYTE", rffi.UCHAR) + WCHAR = rffi_platform.SimpleType("WCHAR", rffi.UCHAR) INT = rffi_platform.SimpleType("INT", rffi.INT) LONG = rffi_platform.SimpleType("LONG", rffi.LONG) PLONG = rffi_platform.SimpleType("PLONG", rffi.LONGP) @@ -38,6 +39,7 @@ LPCVOID = rffi_platform.SimpleType("LPCVOID", rffi.VOIDP) LPSTR = rffi_platform.SimpleType("LPSTR", rffi.CCHARP) LPCSTR = rffi_platform.SimpleType("LPCSTR", rffi.CCHARP) + LPCWSTR = rffi_platform.SimpleType("LPCWSTR", rffi.CWCHARP) LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.INTP) SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T) ULONG_PTR = rffi_platform.SimpleType("ULONG_PTR", rffi.ULONG) @@ -87,6 +89,10 @@ GetLastError = winexternal('GetLastError', [], DWORD) SetLastError = winexternal('SetLastError', [DWORD], lltype.Void) + # In tests, the first call to GetLastError is always wrong, because error + # is hidden by operations in ll2ctypes. Call it now. + GetLastError() + LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], rffi.VOIDP) GetProcAddress = winexternal('GetProcAddress', [rffi.VOIDP, rffi.CCHARP], Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Tue Jul 13 18:41:20 2010 @@ -1,11 +1,23 @@ from pypy.rpython.test.test_llinterp import interpret from pypy.tool.udir import udir from pypy.rlib import rposix -import os +import os, sys def ll_to_string(s): return ''.join(s.chars) +class UnicodeWithEncoding: + def __init__(self, unistr): + self.unistr = unistr + + def encode(self): + from pypy.rlib.runicode import unicode_encode_utf_8 + return unicode_encode_utf_8(self.unistr, len(self.unistr), + "strict") + + def gettext(self): + return self.unistr + class TestPosixUnicode: def setup_method(self, method): self.ufilename = (unicode(udir.join('test_open')) + @@ -14,18 +26,6 @@ f.write("test") f.close() - class UnicodeWithEncoding: - def __init__(self, unistr): - self.unistr = unistr - - def encode(self): - from pypy.rlib.runicode import unicode_encode_utf_8 - return unicode_encode_utf_8(self.unistr, len(self.unistr), - "strict") - - def gettext(self): - return self.unistr - self.path = UnicodeWithEncoding(self.ufilename) def test_access(self): @@ -54,3 +54,11 @@ interpret(f, []) assert not os.path.exists(self.ufilename) + + def test_listdir(self): + udir = UnicodeWithEncoding(os.path.dirname(self.ufilename)) + def f(): + return u', '.join(rposix.listdir(udir)) + + result = interpret(f, []) + assert os.path.basename(self.ufilename) in ll_to_string(result) Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Tue Jul 13 18:41:20 2010 @@ -1035,10 +1035,10 @@ ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) - FindFirstFile = self.llexternal('FindFirstFile', + FindFirstFile = self.llexternal('FindFirstFileA', [rwin32.LPCSTR, LPWIN32_FIND_DATA], rwin32.HANDLE) - FindNextFile = self.llexternal('FindNextFile', + FindNextFile = self.llexternal('FindNextFileA', [rwin32.HANDLE, LPWIN32_FIND_DATA], rwin32.BOOL) FindClose = self.llexternal('FindClose', @@ -1122,6 +1122,73 @@ "ll_os.ll_os_listdir", llimpl=os_listdir_llimpl) + @registering_unicode_version(os.listdir, 1, [0], sys.platform=='win32') + def register_os_listdir_unicode(self): + from pypy.rlib import rwin32 + class CConfig: + _compilation_info_ = ExternalCompilationInfo( + includes = ['windows.h'] + ) + WIN32_FIND_DATA = platform.Struct('struct _WIN32_FIND_DATAW', + [('cFileName', lltype.FixedSizeArray(rwin32.WCHAR, 250))]) + ERROR_FILE_NOT_FOUND = platform.ConstantInteger( + 'ERROR_FILE_NOT_FOUND') + ERROR_NO_MORE_FILES = platform.ConstantInteger( + 'ERROR_NO_MORE_FILES') + + config = platform.configure(CConfig) + WIN32_FIND_DATA = config['WIN32_FIND_DATA'] + ERROR_FILE_NOT_FOUND = config['ERROR_FILE_NOT_FOUND'] + ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] + LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) + + FindFirstFile = self.llexternal('FindFirstFileW', + [rwin32.LPCWSTR, LPWIN32_FIND_DATA], + rwin32.HANDLE) + FindNextFile = self.llexternal('FindNextFileW', + [rwin32.HANDLE, LPWIN32_FIND_DATA], + rwin32.BOOL) + FindClose = self.llexternal('FindClose', + [rwin32.HANDLE], + rwin32.BOOL) + + def os_listdir_llimpl(path): + if path and path[-1] not in (u'/', u'\\', u':'): + path += u'/' + path += u'*.*' + filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') + try: + result = [] + hFindFile = FindFirstFile(path, filedata) + if hFindFile == rwin32.INVALID_HANDLE_VALUE: + error = rwin32.GetLastError() + if error == ERROR_FILE_NOT_FOUND: + return result + else: + raise WindowsError(error, "FindFirstFile failed") + while True: + name = rffi.wcharp2unicode(rffi.cast(rffi.CWCHARP, + filedata.c_cFileName)) + if name != u"." and name != u"..": # skip these + result.append(name) + if not FindNextFile(hFindFile, filedata): + break + # FindNextFile sets error to ERROR_NO_MORE_FILES if + # it got to the end of the directory + error = rwin32.GetLastError() + FindClose(hFindFile) + if error == ERROR_NO_MORE_FILES: + return result + else: + raise WindowsError(error, "FindNextFile failed") + finally: + lltype.free(filedata, flavor='raw') + + return extdef([unicode], # a single argument which is a unicode + [unicode], # returns a list of unicodes + "ll_os.ll_os_wlistdir", + llimpl=os_listdir_llimpl) + @registering(os.pipe) def register_os_pipe(self): # we need a different approach on Windows and on Posix From afa at codespeak.net Tue Jul 13 19:45:29 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Jul 2010 19:45:29 +0200 (CEST) Subject: [pypy-svn] r76190 - pypy/branch/unicode_filename-2/pypy/rpython/module Message-ID: <20100713174529.48D75282BF2@codespeak.net> Author: afa Date: Tue Jul 13 19:45:27 2010 New Revision: 76190 Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Fix translation on non-windows platforms Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Tue Jul 13 19:45:27 2010 @@ -37,7 +37,7 @@ @argnums is the list of positions of unicode strings """ if not condition: - registering(None, condition=False) + return registering(None, condition=False) def unicodefunc(*args): return func(*args) From hakanardo at codespeak.net Wed Jul 14 08:38:46 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 14 Jul 2010 08:38:46 +0200 (CEST) Subject: [pypy-svn] r76191 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100714063846.7C3BD282B90@codespeak.net> Author: hakanardo Date: Wed Jul 14 08:38:44 2010 New Revision: 76191 Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: one more layer of wrapping Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/__init__.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/__init__.py Wed Jul 14 08:38:44 2010 @@ -8,6 +8,7 @@ } interpleveldefs = { - 'array': 'interp_array.array', + 'array': 'interp_array.W_WrappedArray', + '_new_array': 'interp_array.new_array', 'simple_array': 'interp_simple.simple_array', } Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Wed Jul 14 08:38:44 2010 @@ -7,6 +7,7 @@ from pypy.rlib import rgc from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rstruct.runpack import runpack +from pypy.interpreter.argument import Arguments, Signature import py app_array = py.path.local(__file__).dirpath().join('app_array.py') @@ -142,7 +143,7 @@ def setlen(self, size): if size > self.allocated or size < self.allocated/2: - if size > 0: + if size > 0: #FIXME: allways free 0 if size < 9: some = 3 else: @@ -180,7 +181,6 @@ else: size = (stop - start) / step if (stop - start) % step > 0: size += 1 - print size if size < 0: size = 0 w_a=mytype.w_class(self.space) w_a.setlen(size) @@ -551,3 +551,170 @@ return a array.unwrap_spec = (ObjSpace, str, W_Root) + +class W_WrappedArray(Wrappable): + def __init__(self, typecode, w_initializer=None): + pass + __init__.unwrap_spec = ['self', str, W_Root] + + def descr_pop(self, i=-1): + return self._array.descr_pop(i) + descr_pop.unwrap_spec = ['self', int] + + def descr_getitem(self, w_i): + w_item = self._array.descr_getitem(w_i) + if isinstance(w_item, W_ArrayBase): + return wrap_array(w_item) + return w_item + descr_getitem.unwrap_spec = ['self', W_Root] + + def descr_iadd(self, w_i): + self._array.descr_iadd(w_i._array) + return self + descr_iadd.unwrap_spec = ['self', W_Root] + + def descr_imul(self, i): + self._array.descr_imul(i) + return self + descr_imul.unwrap_spec = ['self', int] + + def descr_reduce(self): + space=self._array.space + args=[space.wrap(self._array.typecode)] + if self._array.len>0: + args += [self._array.descr_tostring()] + from pypy.interpreter.mixedmodule import MixedModule + w_mod = space.getbuiltinmodule('array') + mod = space.interp_w(MixedModule, w_mod) + w_new_inst = mod.get('_new_array') + return space.newtuple([w_new_inst, space.newtuple(args)]) + + +def unwrap_array(w_a): + if isinstance(w_a, W_WrappedArray): + return w_a._array + return w_a + +def wrap_array(a): + w_a = W_WrappedArray(None, None) + w_a._array = a + return w_a + +def make_descr(fn, nargs, wrp): + if nargs == 0: + def descr(space, self): + ret = space.call_function( + space.getattr(self._array, space.wrap(fn))) + if (wrp): return wrap_array(ret) + return ret + + elif nargs == 1: + def descr(space, self, w_a): + ret = space.call_function( + space.getattr(self._array, space.wrap(fn)), + unwrap_array(w_a)) + if (wrp): return wrap_array(ret) + return ret + + elif nargs == 2: + def descr(space, self, w_a, w_b): + ret = space.call_function( + space.getattr(self._array, space.wrap(fn)), + unwrap_array(w_a), unwrap_array(w_b)) + if (wrp): return wrap_array(ret) + return ret + + elif nargs == 3: + def descr(space, self, w_a, w_b, w_c): + ret = space.call_function( + space.getattr(self._array, space.wrap(fn)), + unwrap_array(w_a), unwrap_array(w_b), unwrap_array(w_c)) + if (wrp): return wrap_array(ret) + return ret + + else: + raise NotImplementedError + + descr.unwrap_spec = (ObjSpace, W_WrappedArray) + (W_Root,) * nargs + descr.__name__ = 'W_WrappedArray_descr_' + fn + return interp2app(descr) + +typedefs={} +for fn, nargs, wrp in (('append', 1, False), + ('__len__', 0, False), + ('__setitem__', 2, False), + ('__delitem__', 1, False), + ('__getslice__', 2, True), + ('__setslice__', 3, False), + ('__delslice__', 2, False), + ('extend', 1, False), + ('fromstring', 1, False), + ('fromunicode', 1, False), + ('fromfile', 2, False), + ('read', 2, False), + ('fromlist', 1, False), + ('tolist', 0, False), + ('tounicode', 0, False), + ('tofile', 1, False), + ('write', 1, False), + ('tostring', 0, False), + ('__buffer__', 0, False), + ('__repr__', 0, False), + ('count', 1, False), + ('index', 1, False), + ('remove', 1, False), + ('reverse', 0, False), + ('insert', 2, False), + ('__eq__', 1, False), + ('__ne__', 1, False), + ('__lt__', 1, False), + ('__gt__', 1, False), + ('__le__', 1, False), + ('__ge__', 1, False), + ('__copy__', 0, True), + ('__add__', 1, True), + ('__mul__', 1, True), + ('__rmul__', 1, True), + ('buffer_info', 0, False), + ('byteswap', 0, False), + ('__iter__', 0, False), + ('__contains__', 1, False), + ): + typedefs[fn] = make_descr(fn, nargs, wrp) + +def new_array(space, typecode, w_initializer=None): + w_a = W_WrappedArray(None, None) + w_a._array = array(space, typecode, w_initializer) + return w_a +new_array.unwrap_spec = (ObjSpace, str, W_Root) + +def new_array_sub(space, w_cls, typecode, w_initializer=None, w_args=None): + w_array = space.allocate_instance(W_WrappedArray, w_cls) + w_array._array = array(space, typecode, w_initializer) + if len(w_args.arguments_w) > 0: + msg = 'array() takes at most 2 arguments' + raise OperationError(space.w_TypeError, space.wrap(msg)) + return w_array +new_array_sub.unwrap_spec = (ObjSpace, W_Root, str, W_Root, Arguments) + +def descr_wrapped_itemsize(space, self): + return space.wrap(self._array.itemsize) + +def descr_wrapped_typecode(space, self): + return space.wrap(self._array.typecode) + + +W_WrappedArray.typedef = TypeDef( + 'array', + __new__ = interp2app(new_array_sub), + __init__ = interp2app(W_WrappedArray.__init__), + pop = interp2app(W_WrappedArray.descr_pop), + __getitem__ = interp2app(W_WrappedArray.descr_getitem), + __iadd__ = interp2app(W_WrappedArray.descr_iadd), + __imul__ = interp2app(W_WrappedArray.descr_imul), + __reduce__ = interp2app(W_WrappedArray.descr_reduce), + itemsize = GetSetProperty(descr_wrapped_itemsize, cls=W_WrappedArray), + typecode = GetSetProperty(descr_wrapped_typecode, cls=W_WrappedArray), + + **typedefs +) Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Wed Jul 14 08:38:44 2010 @@ -532,12 +532,16 @@ assert repr(self.array('i') + self.array('i')) == "array('i')" a = self.array('i', [1, 2]) + assert type(a + a) is self.array + assert type(a * 2) is self.array + assert type(2 * a) is self.array b = a a += a assert repr(b) == "array('i', [1, 2, 1, 2])" b *= 3 assert repr(a) == "array('i', [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2])" assert a == b + print "\nok: ", type(a) a += self.array('i', (7,)) assert repr(a) == "array('i', [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 7])" @@ -587,10 +591,26 @@ a = self.array('i', lier(5)) assert repr(a) == "array('i', [4, 3, 2, 1, 0])" - #FIXME - #def test_type(self): - # for t in 'bBhHiIlLfdcu': - # assert type(self.array(t)) is self.array + def test_type(self): + for t in 'bBhHiIlLfdcu': + assert type(self.array(t)) is self.array + assert isinstance(self.array(t), self.array) + + def test_subclass(self): + assert len(self.array('b')) == 0 + + a=self.array('i') + a.append(7) + assert len(a) == 1 + + array=self.array + class adder(array): + def __getitem__(self, i): + return array.__getitem__(self, i) + 1 + + a=adder('i', (1,2,3)) + assert len(a) == 3 + assert a[0] == 2 class TestCPythonsOwnArray(BaseArrayTests): From dan at codespeak.net Wed Jul 14 10:46:17 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 14 Jul 2010 10:46:17 +0200 (CEST) Subject: [pypy-svn] r76192 - in pypy/branch/micronumpy/pypy/module/micronumpy: . test Message-ID: <20100714084617.00336282B90@codespeak.net> Author: dan Date: Wed Jul 14 10:46:16 2010 New Revision: 76192 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py pypy/branch/micronumpy/pypy/module/micronumpy/array.py pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py Log: Initial commit for GSoC changes, lots of skipped tests, needs lots of cleanup, needs more tests. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py Wed Jul 14 10:46:16 2010 @@ -1,15 +1,21 @@ ? from pypy.interpreter.mixedmodule import MixedModule +import dtype class Module(MixedModule): + def __init__(self, space, w_name): + super(Module, self).__init__(space, w_name) + from pypy.module.micronumpy import dtype + dtype.w_int_descr.w_native_type = space.w_int + dtype.w_float_descr.w_native_type = space.w_float - applevel_name = 'numpy' + applevel_name = 'micronumpy' appleveldefs = {} interpleveldefs = { - 'array' : 'array.array', + 'array' : 'microarray.array', + 'zeros' : 'microarray.zeros', 'ndarray' : 'array.ndarray', - 'zeros' : 'array.zeros', 'minimum' : 'ufunc.minimum', 'dot' : 'ufunc.dot', } Modified: pypy/branch/micronumpy/pypy/module/micronumpy/array.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/array.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/array.py Wed Jul 14 10:46:16 2010 @@ -4,14 +4,20 @@ from pypy.interpreter.gateway import NoneNotWrapped from pypy.interpreter.gateway import interp2app -from pypy.module.micronumpy.dtype import iterable_type +def iterable_type(space, xs): + raise NotImplementedError("Stub") -from pypy.module.micronumpy.dtype import get_dtype -from pypy.module.micronumpy.dtype import retrieve_dtype #FIXME: ambiguous name? +def get_dtype(space, t): + raise NotImplementedError("Stub") + +def retrieve_dtype(space, t): + raise NotImplementedError("Stub") class BaseNumArray(Wrappable): pass +from pypy.rpython.lltypesystem import lltype + def validate_index(array, space, w_i): index_dimensionality = space.int_w(space.len(w_i)) array_dimensionality = len(array.shape) @@ -90,14 +96,10 @@ return shape def construct_array(space, shape, w_dtype): - from pypy.module.micronumpy.sdarray import sdresult - from pypy.module.micronumpy.mdarray import mdresult + from pypy.module.micronumpy.microarray import MicroArray try: - if len(shape) == 1: - length = shape[0] - return sdresult(w_dtype.code)(space, length, w_dtype) - else: - return mdresult(w_dtype.code)(space, shape, w_dtype) + array = MicroArray(shape, w_dtype) + return space.wrap(array) except KeyError, e: raise OperationError(space.w_NotImplementedError, space.wrap("Haven't implemented generic array yet!")) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py Wed Jul 14 10:46:16 2010 @@ -1,139 +1,210 @@ ?from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable from pypy.interpreter.error import OperationError -from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import rffi -def unwrap_int(space, w_x): - return space.int_w(w_x) -def coerce_int(space, w_x): - return unwrap_int(space, space.int(w_x)) - -def unwrap_float(space, w_x): - return space.float_w(w_x) -def coerce_float(space, w_x): - return unwrap_float(space, space.float(w_x)) - -from pypy.rlib.rarithmetic import r_singlefloat as float32 - -def unwrap_float32(space, w_x): - return float32(space.float_w(w_x)) -def coerce_float32(space, w_x): - return unwrap_float32(space, space.float(w_x)) - -def create_factory(result_factory): - def factory(t): - return result_factory[t] - return factory - -class DynamicType(Wrappable): - def __init__(self, code, name, applevel_type): - self.code = code +class TypeDescr(Wrappable): + def __init__(self, dtype, name): + self.dtype = dtype self.name = name - self.applevel_type = applevel_type + self.w_native_type = None - def descr_eq(self, space, w_x): - if space.abstract_isinstance_w(w_x, space.w_type): - return space.eq(self.applevel_type, w_x) #FIXME: correct comparison? - else: - try: - code = space.str_w(w_x) - if self.code == code: - return space.w_True - elif self.name == code: - return space.w_True - else: - raise OperationError(space.w_TypeError, space.wrap("data type not understood")) - except OperationError, e: - raise OperationError(space.w_TypeError, space.wrap("data type not understood")) - descr_eq.unwrap_spec = ['self', ObjSpace, W_Root] -DynamicType.typedef = TypeDef('dtype', - __eq__ = interp2app(DynamicType.descr_eq), - ) - -class DynamicTypes(object): - result_types = { - ('i', 'i'): 'i', - ('i', 'd'): 'd', - ('d', 'i'): 'd', - ('d', 'd'): 'd', - } - - def __init__(self): - self.dtypes = {} + def descr_eq(self, space, w_type): + if space.eq_w(self.w_native_type, w_type): + return space.w_True - def typecode(self, space, w_type): try: - assert isinstance(w_type, DynamicType) - return w_type.code - except AssertionError, e: pass + typestr = space.str_w(w_type) + if self.dtype.typecode == typestr: return space.w_True + elif self.name == typestr: return space.w_True + else: return space.w_False + except OperationError, e: + if e.match(space, space.w_TypeError): pass + else: raise + + return space.w_False + descr_eq.unwrap_spec = ['self', ObjSpace, W_Root] + + def descr_itemsize(self, space): + return space.wrap(self.dtype.itemsize()) + descr_itemsize.unwrap_spec = ['self', ObjSpace] + + def descr_repr(self, space): + return space.wrap("dtype('%s')" % self.name) + descr_repr.unwrap_spec = ['self', ObjSpace] + + def result(self, other): + a = self.dtype.typeid + b = other.dtype.typeid + c = _result_types[(a, b)] + return _descriptors[c].wrappable_dtype() + +TypeDescr.typedef = TypeDef('dtype', + itemsize = GetSetProperty(TypeDescr.descr_itemsize), + __eq__ = interp2app(TypeDescr.descr_eq), + __repr__ = interp2app(TypeDescr.descr_repr), + ) + +class DescrBase(object): pass + +_typeindex = {} +_descriptors = [] +_w_descriptors = [] +def descriptor(code, name, ll_type): + arraytype = lltype.Array(ll_type) + class DescrImpl(DescrBase): + def __init__(self): + self.typeid = 0 + self.typecode = code + + def wrap(self, space, value): + return space.wrap(value) + + def w_getitem(self, space, data, index): + return space.wrap(self.getitem(data, index)) + + def w_setitem(self, space, data, index, w_value): + value = self.unwrap(space, w_value) + self.setitem(data, index, value) + + def itemsize(self): + return rffi.sizeof(ll_type) + + def getitem(self, data, index): + array = rffi.cast(lltype.Ptr(arraytype), data) + return array[index] + + def setitem(self, data, index, value): + array = rffi.cast(lltype.Ptr(arraytype), data) + #array[index] = rffi.cast(ll_type, value) + array[index] = value # XXX: let's see if this works + + def alloc(self, count): + return lltype.malloc(arraytype, count, flavor='raw') + + def free(self, data): + lltype.free(data, flavor='raw') + + def wrappable_dtype(self): + assert _w_descriptors[self.typeid].dtype is self, "This better be true." + return _w_descriptors[self.typeid] + + for type in [lltype.Signed, lltype.Float]: + def get_type(self, data, index): + value = self.getitem(data, index) + return rffi.cast(type, value) + get_type.__name__ = 'get_%s' % type + setattr(DescrImpl, 'get_%s' % type, get_type) + + def set_type(self, data, index, value): + value = rffi.cast(ll_type, value) + self.setitem(data, index, value) + set_type.__name__ = 'set_%s' % type + setattr(DescrImpl, 'set_%s' % type, set_type) + + DescrImpl.__name__ = 'Descr_%s' % name # XXX + + typeid = len(_descriptors) + + _typeindex[code] = typeid + descriptor = DescrImpl() + descriptor.typeid = typeid + + _descriptors.append(descriptor) + + w_descriptor = TypeDescr(descriptor, name) + _w_descriptors.append(w_descriptor) + + return descriptor + +_typestring = {} +# int, int32 is l +# i is ?? + +int_descr = descriptor('i', 'int32', lltype.Signed) +type(int_descr).unwrap = lambda self, space, value: space.int_w(value) +_int_index = _typeindex['i'] +_typestring['int32'] = _int_index +w_int_descr = _w_descriptors[_int_index] + +float_descr = descriptor('d', 'float64', lltype.Float) +type(float_descr).unwrap = lambda self, space, value: space.float_w(value) +_float_index = _typeindex['d'] +_typestring['float64'] = _float_index +w_float_descr = _w_descriptors[_float_index] + +_result_types = {(_int_index, _int_index): _int_index, + (_int_index, _float_index): _float_index, + (_float_index, _int_index): _float_index, + (_float_index, _float_index): _float_index, + } + +def from_typecode(s): + index = _typeindex[s] + return _descriptors[index] + +def from_typestring(s): + index = _typestring[s] + return _descriptors[index] + +def from_wrapped_type(space, w_type): + if w_type is space.w_int: + return int_descr + else: + return float_descr #XXX: only handles two types! + +def get(space, w_dtype): + try: + s = space.str_w(w_dtype) try: - return space.str_w(w_type) - except OperationError, e: - typecode_mapping = { - space.w_int: 'i', - space.w_float: 'd', - } + return from_typecode(s) + except KeyError, e: + return from_typestring(s) + + except KeyError, e: + raise OperationError(space.w_TypeError, + space.wrap("data type not understood") + ) + + except OperationError, e: + if e.match(space, space.w_TypeError): pass # XXX: ValueError? + + try: + i = space.int_w(w_dtype) + return _descriptors[i] + except OperationError, e: + if e.match(space, space.w_TypeError): pass + else: raise + + return from_wrapped_type(space, w_dtype) + +# FIXME: watch for wrapped typedescrs! +def infer_from_iterable(space, w_xs): + highest_type = None + dtype = None + w_i = space.iter(w_xs) + try: + while True: + w_element = space.next(w_i) try: - return typecode_mapping[w_type] - except IndexError, e: - raise OperationError(space.w_TypeError, - space.wrap("Can't understand type.")) - - def result_mapping(self, space, types): - types = (self.typecode(space, types[0]), - self.typecode(space, types[1])) - return self.result_types[types] - - def iterable_type(self, space, w_xs): - xs = space.fixedview(w_xs) - result_type = 'i' - for i in range(len(xs)): - try: - atype = self.iterable_type(space, xs[i]) + dtype = infer_from_iterable(space, w_element) except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - atype = self.typecode(space, space.type(xs[i])) - result_type = self.result_types[(result_type, atype)] - return result_type - - def verify_dtype_dict(self, space): - if not self.dtypes: - self.dtypes.update( - { - 'i': DynamicType('i', 'int32', space.w_int), - 'd': DynamicType('d', 'float64', space.w_float), - } - ) - - def retrieve_dtype(self, space, t): - self.verify_dtype_dict(space) - return self.dtypes[t] - - def get_dtype(self, space, w_type): - try: - t = space.str_w(w_type) - except OperationError, e: - if e.match(space, space.w_TypeError): - t = self.typecode(space, w_type) + if e.match(space, space.w_TypeError): # not iterable? + w_type = space.type(w_element) + dtype = from_wrapped_type(space, w_type) + else: raise + + if highest_type is not None: + a = highest_type.typeid + b = dtype.typeid + highest_typeid = _result_types[(a, b)] + highest_type = _descriptors[highest_typeid] else: - raise - return self.retrieve_dtype(space, t) - - def sdarray(self, space, length, w_dtype): - from pypy.module.micronumpy.sdarray import sdresult - return sdresult(self.typecode(space, w_dtype))(space, length, w_dtype) - - def mdarray(self, space, shape, w_dtype): - from pypy.module.micronumpy.mdarray import mdresult - return mdresult(self.typecode(space, w_dtype))(space, shape, w_dtype) - -dtypes = DynamicTypes() -iterable_type = dtypes.iterable_type -typecode = dtypes.typecode -result_mapping = dtypes.result_mapping -get_dtype = dtypes.get_dtype -retrieve_dtype = dtypes.retrieve_dtype -sdarray = dtypes.sdarray -mdarray = dtypes.mdarray + highest_type = dtype + except OperationError, e: + if e.match(space, space.w_StopIteration): + return highest_type + else: raise Modified: pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py Wed Jul 14 10:46:16 2010 @@ -12,16 +12,6 @@ from pypy.module.micronumpy.array import \ mul_operation, div_operation, add_operation, sub_operation -from pypy.module.micronumpy.dtype import unwrap_int, coerce_int -from pypy.module.micronumpy.dtype import unwrap_float, coerce_float -from pypy.module.micronumpy.dtype import result_mapping, iterable_type - -from pypy.module.micronumpy.dtype import create_factory - -from pypy.module.micronumpy.dtype import get_dtype -from pypy.module.micronumpy.dtype import retrieve_dtype #FIXME: ambiguously named? -from pypy.module.micronumpy.dtype import DynamicType - #TODO: merge unwrap_spec decorator # from pypy.interpreter.gateway import unwrap_spec @@ -60,6 +50,15 @@ next = interp2app(SingleDimIterator.descr_next) ) + def operation_name(type, operand_type, opname): + return '_'.join([type, operand_type, opname]) + + def fixedview_operation(opname, self, source, x): + return getattr(self, operation_name('client', 'fixedview', opname))(source, x) + + def client_operation(opname, self, source, x): + return getattr(self, operation_name('client', 'scalar', opname))(source, x) + def create_client_math_operation(f): def scalar_operation(self, source, x, inverse): for i in range(len(source.storage)): @@ -274,22 +273,22 @@ maxlen = max([len(x) for x in strings]) return strings, maxlen - def descr_str(self): + def descr_str(self, space): space = self.space #beautiful, as in numpy strings, maxlen = self.str() return space.wrap( "[%s]" % ' '.join(["%-*s"%(maxlen, s) for s in strings]) ) - descr_str.unwrap_spec = ['self'] + descr_str.unwrap_spec = ['self', ObjSpace] - def descr_repr(self): + def descr_repr(self, space): space = self.space strings, maxlen = self.str() return space.wrap( "array([%s])" % ', '.join(["%-*s"%(maxlen, s) for s in strings]) ) - descr_repr.unwrap_spec = ['self'] + descr_repr.unwrap_spec = ['self', ObjSpace] NumArray.typedef = TypeDef('ndarray', base_typedef, __mul__ = interp2app(NumArray.descr_mul), @@ -315,8 +314,3 @@ ) return NumArray - -IntArray = create_sdarray(int, unwrap_int, coerce_int) -FloatArray = create_sdarray(float, unwrap_float, coerce_float) - -sdresult = create_factory({'i': IntArray, 'd': FloatArray}) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Wed Jul 14 10:46:16 2010 @@ -1,8 +1,11 @@ ? +import py + from pypy.conftest import gettestobjspace class TestSDArray(object): def test_unwrap(self, space): + skip("Irrelevant interplevel test") w_int = space.wrap(1) w_float = space.wrap(1.0) @@ -27,6 +30,7 @@ #interp_raises((space.w_TypeError,), unwrap_float, space, w_int) #er, shouldn't this raise? def test_coerce(self, space): + skip("Irrelevant interplevel test") w_int = space.wrap(1) w_float = space.wrap(1.0) @@ -54,14 +58,15 @@ def test_type_array(self): compare = self.compare - from numpy import array + from micronumpy import array for data_type in (int, float): data = [data_type(x) for x in xrange(4)] ar = array(data) assert compare(ar, data) + @py.test.mark.xfail def test_sdarray_operators(self): - from numpy import array + from micronumpy import array from operator import mul, div, add, sub compare = self.compare d = range(1, self.length) @@ -76,9 +81,10 @@ assert compare(operator(value, ar2), [operator(value, x) for x in data]) assert compare(operator(ar, ar2), [operator(x, y) for (x, y) in zip(ar, ar2)]) + @py.test.mark.xfail def test_operator_result_types(self): from operator import mul, div, add, sub - from numpy import array + from micronumpy import array types = { (int, int): int, (int, float): float, @@ -116,7 +122,7 @@ test_type(f.dtype, result_type) def test_iter(self): - from numpy import array + from micronumpy import array for iterable_type in (list, tuple): for data_type in (int, float): data = iterable_type([data_type(x) for x in xrange(self.length)]) @@ -126,19 +132,20 @@ def test_iterable_construction(self): compare = self.compare - from numpy import array + from micronumpy import array ar = array(xrange(4)) assert compare(ar, xrange(4)) def test_zeroes(self): - from numpy import zeros + from micronumpy import zeros for data_type in (int, float): ar = zeros(3, dtype=int) assert ar[0] == data_type(0.0) + @py.test.mark.xfail def test_setitem_getitem(self): - from numpy import zeros + from micronumpy import zeros compare = self.compare ar = zeros(8, dtype=int) @@ -166,8 +173,9 @@ raises(ValueError, ar.__setitem__, 0, 'f') raises(ValueError, ar.__setitem__, slice(2, 3), [4, 5, 6]) + @py.test.mark.xfail def test_minimum(self): - from numpy import zeros, minimum + from micronumpy import zeros, minimum ar = zeros(5, dtype=int) ar2 = zeros(5, dtype=int) ar[0] = 3 @@ -184,9 +192,9 @@ assert len(x) == 5 raises(ValueError, minimum, ar, zeros(3, dtype=int)) + @py.test.mark.xfail def test_str(self): - skip("Perfect string formatting is going to be tough.") - from numpy import zeros, array + from micronumpy import zeros, array z = zeros(shape=(3,)) assert str(z) == '[ 0. 0. 0.]' @@ -228,14 +236,15 @@ return compare""") def test_multidim(self): - from numpy import zeros + from micronumpy import zeros ar = zeros((3, 3), dtype=int) assert ar[0, 2] == 0 raises(IndexError, ar.__getitem__, (3, 0)) assert ar[-2, 1] == 0 + @py.test.mark.xfail def test_construction(self): - from numpy import array + from micronumpy import array gen_array = self.gen_array #3x3 @@ -248,7 +257,7 @@ assert ar.shape == (2, 3) #3x2 - ar = array(gen_array((3,2))) + ar = array(gen_array((3,2))) # XXX: here's the problem assert len(ar) == 3 assert ar.shape == (3, 2) @@ -256,7 +265,7 @@ raises(ValueError, array, [2, [3, 4]]) def test_getset(self): - from numpy import zeros + from micronumpy import zeros ar = zeros((3, 3, 3), dtype=int) ar[1, 2, 1] = 3 assert ar[1, 2, 1] == 3 @@ -265,11 +274,11 @@ assert ar[-2, 2, -2] == 3 def test_len(self): - from numpy import zeros + from micronumpy import zeros assert len(zeros((3, 2, 1), dtype=int)) == 3 def test_shape_detect(self): - from numpy import array + from micronumpy import array ar = array([range(i*3, i*3+3) for i in range(3)]) assert len(ar) == 3 assert ar.shape == (3, 3) @@ -277,8 +286,9 @@ for j in range(3): assert ar[i, j] == i*3+j + @py.test.mark.xfail def test_get_set_slices(self): - from numpy import array + from micronumpy import array gen_array = self.gen_array compare = self.compare @@ -327,18 +337,20 @@ ar3[1:3] = [[0, 1, 2], [3, 4, 5]] assert compare(ar3[1, 2], [2, 2, 2]) + @py.test.mark.xfail def test_newaxis(self): - from numpy import array + from micronumpy import array gen_array = self.gen_array compare = self.compare ar = array(gen_array((3, 3))) assert compare(ar[[1, 2], 0], ar[1:3, 0]) - assert compare(ar[ [[1, 2], [0, 1]] ] [1, 0], ar[0]) + assert compare(ar[ [[1, 2], [0, 1]] ] [1, 0], ar[0]) # zip iterables into coordinates + @py.test.mark.xfail def test_mdarray_operators(self): - from numpy import array + from micronumpy import array import operator compare = self.compare @@ -377,9 +389,10 @@ class AppTestDType(object): def setup_class(cls): cls.space = gettestobjspace(usemodules=('micronumpy',)) - #FIXME: need DynamicType.__new__/__init__ to best test this + + @py.test.mark.xfail def test_eq(self): - from numpy import zeros + from micronumpy import zeros a = zeros((4,), dtype=int) assert a.dtype == int @@ -397,6 +410,7 @@ class TestDType(object): def test_lookups(self, space): + skip("Depends on interplevel stuff that doesn't exist really anymore.") from pypy.module.micronumpy.dtype import retrieve_dtype from pypy.module.micronumpy.dtype import get_dtype a = get_dtype(space, space.wrap('i')) @@ -406,6 +420,7 @@ assert b == retrieve_dtype(space, 'd') def test_result_types(self, space): + skip("Depends on interplevel stuff that doesn't exist really anymore.") from pypy.module.micronumpy.dtype import get_dtype from pypy.module.micronumpy.dtype import result_mapping w_typecode_a = space.wrap('i') @@ -419,9 +434,7 @@ assert 'd' == result_mapping(space, (w_typecode_b, w_typecode_b)) def test_iterable_type(self, space): - from pypy.module.micronumpy.dtype import iterable_type - w_int = space.wrap(1) - w_float = space.wrap(2.0) + from pypy.module.micronumpy.dtype import infer_from_iterable data = [(space.wrap([1, 2, 3, 4, 5]), 'i'), (space.wrap([1, 2, 3.0, 4, 5]), 'd'), @@ -431,4 +444,44 @@ (space.wrap([1.0, 2, 3, 4, 5.0]), 'd')] for w_xs, typecode in data: - assert typecode == iterable_type(space, w_xs) + assert typecode == infer_from_iterable(space, w_xs).typecode + +class TestMicroArray(object): + def test_strides(self, space): + from pypy.module.micronumpy.microarray import stride_row, stride_column + + shape = (2, 3) + assert stride_row(shape, 0) == 3 + assert stride_row(shape, 1) == 1 + + assert stride_column(shape, 0) == 1 + assert stride_column(shape, 1) == 2 + + # TODO: more, N-dimensional too + + def test_memory_layout(self, space): + from pypy.module.micronumpy.microarray import MicroArray + from pypy.module.micronumpy.microarray import array + from pypy.module.micronumpy.dtype import w_int_descr + + data = [[1, 2, 3], + [4, 5, 6]] + + w_data = space.wrap(data) + column_major = [1, 4, 2, 5, 3, 6] + row_major = [1, 2, 3, 4, 5, 6] + + assert len(column_major) == len(row_major) + memlen = len(column_major) + + ar = array(space, w_data, w_dtype=w_int_descr, order='C') #C for C not column + + for i in range(memlen): + array_element = space.unwrap(ar.getitem(space, i)) # ugly, but encapsulates everything + assert array_element == row_major[i] + + ar = array(space, w_data, w_dtype=w_int_descr, order='F') + for i in range(memlen): + array_element = space.unwrap(ar.getitem(space, i)) # ugly, but encapsulates everything + assert array_element == column_major[i] + Modified: pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py Wed Jul 14 10:46:16 2010 @@ -1,32 +1,34 @@ ?from pypy.module.micronumpy.array import array, zeros, ndarray from pypy.module.micronumpy.array import construct_array -from pypy.module.micronumpy.dtype import result_mapping -from pypy.module.micronumpy.sdarray import sdresult -from pypy.module.micronumpy.mdarray import mdresult, compute_pos - -from pypy.module.micronumpy.dtype import retrieve_dtype from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.error import OperationError from pypy.rlib.debug import make_sure_not_resized +from pypy.module.micronumpy.microarray import MicroArray, size_from_shape + def minimum(space, w_a, w_b): - if not isinstance(w_a, ndarray) or not isinstance(w_b, ndarray): + if not isinstance(w_a, MicroArray) or not isinstance(w_b, MicroArray): raise OperationError(space.w_TypeError, space.wrap("expecting ndarray object")) - if w_a.len()!= w_b.len(): + if w_a.shape != w_b.shape: raise OperationError(space.w_ValueError, space.wrap("minimum of arrays of different length")) - dtype = result_mapping(space, (w_a.dtype, w_b.dtype)) - res = construct_array(space, w_a.shape, retrieve_dtype(space, dtype)) - for i in range(len(w_a.storage)): - one = w_a.storage[i] - two = w_b.storage[i] + + result_type = w_a.dtype.result(w_b.dtype) + + result = MicroArray(w_a.shape, result_type) + + size = size_from_shape(w_a.shape) + + for i in range(size): # FIXME: specialized paths for Signed and Float ? wraps right now... + one = w_a.getitem(space, i) + two = w_b.getitem(space, i) if one < two: - res.storage[i] = one + result.setitem(space, i, one) else: - res.storage[i] = two - return space.wrap(res) + result.setitem(space, i, two) + return space.wrap(result) minimum.unwrap_spec = [ObjSpace, W_Root, W_Root] def dot(space, w_a, w_b): From dan at codespeak.net Wed Jul 14 10:51:24 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 14 Jul 2010 10:51:24 +0200 (CEST) Subject: [pypy-svn] r76193 - pypy/branch/micronumpy/pypy/module/micronumpy Message-ID: <20100714085124.BEBD6282B90@codespeak.net> Author: dan Date: Wed Jul 14 10:51:23 2010 New Revision: 76193 Added: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Modified: pypy/branch/micronumpy/pypy/module/micronumpy/array.py Log: That's an important file... Modified: pypy/branch/micronumpy/pypy/module/micronumpy/array.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/array.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/array.py Wed Jul 14 10:51:23 2010 @@ -13,9 +13,6 @@ def retrieve_dtype(space, t): raise NotImplementedError("Stub") -class BaseNumArray(Wrappable): - pass - from pypy.rpython.lltypesystem import lltype def validate_index(array, space, w_i): @@ -33,52 +30,6 @@ raise OperationError(space.w_IndexError, space.wrap("invalid index")) # all as in numpy -def mul_operation(): - def mul(x, y): return x * y - return mul - -def div_operation(): - def div(x, y): return x / y - return div - -def add_operation(): - def add(x, y): return x + y - return add - -def sub_operation(): - def sub(x, y): return x - y - return sub - -def copy_operation(): - def copy(x, y): return x #XXX: I sure hope GCC can optimize this - return copy - -def app_mul_operation(): - def mul(space, x, y): - return space.mul(x, y) - return mul - -def app_div_operation(): - def div(space, x, y): - return space.div(x, y) - return div - -def app_add_operation(): - def add(space, x, y): - return space.add(x, y) - return add - -def app_sub_operation(): - def sub(space, x, y): - return space.sub(x, y) - return sub - -def unpack_shape(space, w_shape): - if space.is_true(space.isinstance(w_shape, space.w_int)): - return [space.int_w(w_shape)] - shape_w = space.fixedview(w_shape) - return [space.int_w(w_i) for w_i in shape_w] - def infer_shape(space, w_values): shape = [] while True: @@ -104,6 +55,9 @@ raise OperationError(space.w_NotImplementedError, space.wrap("Haven't implemented generic array yet!")) +class BaseNumArray(Wrappable): + pass + def descr_new(space, w_cls, w_shape, w_dtype=NoneNotWrapped, w_buffer=NoneNotWrapped, w_offset=NoneNotWrapped, w_strides=NoneNotWrapped, order='C'): @@ -121,29 +75,3 @@ ) base_typedef = BaseNumArray.typedef ndarray = BaseNumArray - -def array(space, w_values, w_dtype=NoneNotWrapped, - copy=True, order='C', - subok=False, ndim=1): - shape = infer_shape(space, w_values) - - if w_dtype is None: - dtype_w = retrieve_dtype(space, iterable_type(space, w_values)) - else: - dtype_w = get_dtype(space, w_dtype) - result = construct_array(space, shape, dtype_w) - result.load_iterable(w_values) - return space.wrap(result) -array.unwrap_spec = [ObjSpace, W_Root, W_Root, - bool, str, - bool, int] - -def zeros(space, w_shape, w_dtype=NoneNotWrapped, order='C'): - shape_w = unpack_shape(space, w_shape) - if w_dtype is None: - dtype_w = retrieve_dtype(space, 'd') - else: - dtype_w = get_dtype(space, w_dtype) - result = construct_array(space, shape_w, dtype_w) - return space.wrap(result) -zeros.unwrap_spec = [ObjSpace, W_Root, W_Root, str] Added: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- (empty file) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Wed Jul 14 10:51:23 2010 @@ -0,0 +1,354 @@ +?from pypy.interpreter.baseobjspace import ObjSpace +from pypy.interpreter.error import OperationError +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.gateway import NoneNotWrapped + +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.baseobjspace import W_Root, UnpackValueError +from pypy.objspace.std.sliceobject import W_SliceObject +from pypy.objspace.std.listobject import W_ListObject +from pypy.objspace.std.tupleobject import W_TupleObject +from pypy.rlib.debug import make_sure_not_resized + +from pypy.module import micronumpy +from pypy.module.micronumpy.array import BaseNumArray +from pypy.module.micronumpy.array import base_typedef +from pypy.module.micronumpy.array import construct_array, infer_shape +from pypy.module.micronumpy.array import validate_index + +def size_from_shape(shape): + size = 1 + for dimension in shape: + size *= dimension + return size + +def index_w(space, w_index): + return space.int_w(space.index(w_index)) + +def stride_row(shape, i): + stride = 1 + ndim = len(shape) + for s in shape[i + 1:]: + stride *= s + return stride + +def stride_column(shape, i): + if i < 1: return 1 + elif i == 1: + return shape[0] + stride = 1 + for s in shape[:i-1]: + stride *= s + return stride + +class MicroIter(Wrappable): + def __init__(self, array): + self.array = array + self.i = 0 + + def descr_iter(self, space): + return space.wrap(self) + descr_iter.unwrap_spec = ['self', ObjSpace] + + def descr_next(self, space): + if self.i < space.int_w(space.len(self.array)): + next = self.array.getitem(space, self.i) # FIXME: wrong for multi dimensional! (would be easy applevel) + self.i += 1 + return next + else: + raise OperationError(space.w_StopIteration, space.wrap("")) + descr_next.unwrap_spec = ['self', ObjSpace] + +MicroIter.typedef = TypeDef('iterator', + __iter__ = interp2app(MicroIter.descr_iter), + next = interp2app(MicroIter.descr_next), + ) + +class MicroArray(BaseNumArray): + def __init__(self, shape, dtype, parent=None, offset = 0): + self.shape = shape + self.dtype = dtype + self.parent = parent + self.offset = offset + self.order = 'C' #XXX: more forgiving to give it valid default + + assert self.dtype is not None + dtype = dtype.dtype #XXX: ugly + + size = size_from_shape(shape) + + if size > 0 and parent is None: + self.data = dtype.alloc(size) + elif parent is not None: + self.data = parent.data + else: + self.data = None + + def descr_len(self, space): + return space.wrap(self.shape[0]) + descr_len.unwrap_spec = ['self', ObjSpace] + + def getitem(self, space, index): + try: + dtype = self.dtype.dtype #XXX: kinda ugly + return dtype.w_getitem(space, self.data, self.offset + index) + except IndexError, e: + raise OperationError(space.w_IndexError, + space.wrap("index out of bounds")) + + def setitem(self, space, index, w_value): + dtype = self.dtype.dtype #XXX: kinda ugly + dtype.w_setitem(space, self.data, self.offset + index, w_value) #maybe hang onto w_dtype separately? + + def flatten_applevel_index(self, space, w_index): + try: + index = space.int_w(w_index) + return index + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + + index = space.fixedview(w_index) + index = [index_w(space, w_x) for w_x in index] + # XXX: normalize + for i in range(len(index)): + if index[i] < 0: + index[i] = self.shape[i] + index[i] + return self.flatten_index(space, index) + + def create_flatten_index(stride_function): + def flatten_index(self, index): + offset = 0 + for i in range(len(index)): + stride = stride_function(self.shape, i) + offset += index[i] * stride + return offset + return flatten_index + + flatten_index_r = create_flatten_index(stride_row) + flatten_index_c = create_flatten_index(stride_column) + + # FIXME: when different types are supported + # this function will change + def flatten_index(self, space, index): + if self.order == 'C': + return self.flatten_index_r(index) # row order for C + elif self.order == 'F': + return self.flatten_index_c(index) # + else: + raise OperationError(space.w_NotImplementedError, + space.wrap("Unknown order: '%s'" % self.order)) + + def descr_getitem(self, space, w_index): + try: + index = self.flatten_applevel_index(space, w_index) + + try: + #w_iter = space.iter(w_index) # XXX: I guess len() should throw TypeError + # FIXME: what about slices? + index_dimension = space.int_w(space.len(w_index)) + except OperationError, e: + if e.match(space, space.w_TypeError): + index_dimension = 1 + else: raise + + if index_dimension == len(self.shape): + return self.getitem(space, index) + elif index_dimension < len(self.shape): + array = MicroArray(self.shape[index_dimension:], self.dtype, + parent=self, offset=self.offset + index) + return space.wrap(array) + else: + raise OperationError(space.w_IndexError, + space.wrap("invalid index")) + + except OperationError, e: + if e.match(space, space.w_TypeError): pass # is there any way this can be caught here? + else: raise + + # XXX: here be demons + + try: + indices = [] + index = space.fixedview(w_index) + if len(index) > len(self.shape): + raise OperationError(space.w_ValueError, + space.wrap("Index has more dimensions (%d) than array (%d)" % (len(index), len(self.shape)))) + + for i in range(len(index)): + indices.append([]) + + for subindex in index: + try: + raise OperationError(space.w_NotImplementedError, + space.wrap("Haven't implemented newaxis.")) + except OperationError, e: + pass + + except OperationError, e: + if e.match(space, space.w_StopIteration): pass + else: raise + + + descr_getitem.unwrap_spec = ['self', ObjSpace, W_Root] + + def descr_setitem(self, space, w_index, w_value): + index = self.flatten_applevel_index(space, w_index) + self.setitem(space, index, w_value) + descr_setitem.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] + + def descr_repr(self, space): + return space.wrap("") + descr_repr.unwrap_spec = ['self', ObjSpace] + + def descr_iter(self, space): + return space.wrap(MicroIter(self)) + descr_iter.unwrap_spec = ['self', ObjSpace] + + def __del__(self): + if self.parent is None and self.data is not None: + dtype = self.dtype.dtype + dtype.free(self.data) + +from pypy.interpreter import gateway + +app_formatting = gateway.applevel(""" + from StringIO import StringIO + def str(out, array): + out.write("[") + if len(array.shape) > 1: + out.write(',\\n'.join([str(x) for x in array])) + else: + out.write(', '.join([str(x) for x in array])) + out.write("]") + + def descr_str(self): + out = StringIO() + str(out, self) + result = out.getvalue() + out.close() + return result + + def descr_repr(self): + out = StringIO() + out.write("array(") + str(out, self) + out.write(")") + result = out.getvalue() + out.close() + return result + """) + +app_descr_repr = app_formatting.interphook('descr_repr') + +def microarray_descr_repr(self, space): + return app_descr_repr(space, space.wrap(self)) +microarray_descr_repr.unwrap_spec = ['self', ObjSpace] + +# Getters, strange GetSetProperty behavior +# forced them out of the class +def descr_get_dtype(space, self): + return space.wrap(self.dtype) + +def descr_get_shape(space, self): + return space.newtuple([space.wrap(x) for x in self.shape]) + +MicroArray.typedef = TypeDef('uarray', + dtype = GetSetProperty(descr_get_dtype, cls=MicroArray), + shape = GetSetProperty(descr_get_shape, cls=MicroArray), + __getitem__ = interp2app(MicroArray.descr_getitem), + __setitem__ = interp2app(MicroArray.descr_setitem), + __len__ = interp2app(MicroArray.descr_len), + __repr__ = microarray_descr_repr, + __iter__ = interp2app(MicroArray.descr_iter), + ) + +def reconcile_shapes(space, a, b): + assert a == b, "Invalid assertion I think" # FIXME + return a + +#def infer_shape(space, w_xs): +def infer_shape_slow(space, w_xs): # gonna kill this after it's been svn for one revision + length = 0 + shape = [] + old = None + try: + w_i = space.iter(w_xs) + while True: + element = space.next(w_i) + try: + shape = infer_shape(space, element) + except OperationError, e: + if e.match(space, space.w_TypeError): pass + else: raise + + if old is not None: + shape = reconcile_shapes(space, old, shape) # use to process jagged arrays, if Numpy even allows them + + length += 1 + old = shape + except OperationError, e: + if e.match(space, space.w_StopIteration): + return [length] + shape + else: + raise + +#XXX: don't like having to pass around setitem +app_fill_array = gateway.applevel(""" + def fill_array(start, a, b): + i = 0 + for x in b: + print i + try: + fill_array(start + [i], a, x) + except TypeError, e: + a[start + [i]] = x + i += 1 + """) + +fill_array = app_fill_array.interphook('fill_array') + +def array(space, w_xs, w_dtype=NoneNotWrapped, copy=True, order='C', subok=False, w_ndim=NoneNotWrapped): + if w_dtype is None: + dtype = micronumpy.dtype.infer_from_iterable(space, w_xs) + else: + dtype = micronumpy.dtype.get(space, w_dtype) + + assert dtype is not None + wrappable_dtype = dtype.wrappable_dtype() + + shape = infer_shape(space, w_xs) + + ar = MicroArray(shape, wrappable_dtype) + ar.order = order + w_ar = space.wrap(ar) + + fill_array(space, + space.wrap([]), w_ar, w_xs) + + return w_ar +array.unwrap_spec = [ObjSpace, W_Root, W_Root, bool, str, bool, W_Root] + +def zeros(space, w_shape, w_dtype=NoneNotWrapped, order='C'): + try: + shape_w = space.fixedview(w_shape) + shape = [space.int_w(x) for x in shape_w] + except OperationError, e: + if e.match(space, space.w_TypeError): + shape = [space.int_w(w_shape)] + else: raise + + if w_dtype: + dtype = micronumpy.dtype.get(space, w_dtype) + else: + dtype = micronumpy.dtype.int_descr + + ar = MicroArray(shape, dtype.wrappable_dtype()) + ar.order = order + + for i in range(size_from_shape(shape)): + ar.setitem(space, i, space.wrap(0)) + + return space.wrap(ar) +zeros.unwrap_spec = [ObjSpace, W_Root, W_Root] From wlav at codespeak.net Wed Jul 14 11:09:14 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Wed, 14 Jul 2010 11:09:14 +0200 (CEST) Subject: [pypy-svn] r76194 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100714090914.2FD31282B90@codespeak.net> Author: wlav Date: Wed Jul 14 11:09:13 2010 New Revision: 76194 Modified: pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/example01.h pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Log: Enable returning objects from static classes and some code cleanup/cutification. Modified: pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py Wed Jul 14 11:09:13 2010 @@ -7,16 +7,26 @@ class CppyyObject(object): def __init__(self, *args): - print '__init__ called', args self._cppinstance = self._cppyyclass.construct(*args) def destruct(self): self._cppinstance.destruct() - -def make_static_function(cpptype, name): - def method(*args): - return cpptype.invoke(name, *args) +def bind_object(cppobj, cppclass): + if cppobj is None: + return None + bound_obj = object.__new__(cppclass) + bound_obj._cppinstance = cppobj + return bound_obj + +def make_static_function(cpptype, name, rettype): + if rettype is None: + def method(*args): + return cpptype.invoke(name, *args) + else: + cppclass = get_cppclass(rettype) + def method(*args): + return bind_object(cpptype.invoke(name, *args), cppclass) method.__name__ = name return staticmethod(method) @@ -24,17 +34,12 @@ if rettype is None: # return builtin type def method(self, *args): return self._cppinstance.invoke(name, *args) - method.__name__ = name - return method else: # return instance + cppclass = get_cppclass(rettype) def method(self, *args): - result = self._cppinstance.invoke(name, *args) - if not result is None: - bound_result = object.__new__(get_cppclass(rettype)) - bound_result._cppinstance = result - return bound_result - method.__name__ = name - return method + return bind_object(self._cppinstance.invoke(name, *args), cppclass) + method.__name__ = name + return method _existing_classes = {} @@ -52,7 +57,7 @@ for f in cpptype.get_function_members(): cppol = cpptype.get_overload(f) if cppol.is_static(): - d[f] = make_static_function(cpptype, f) + d[f] = make_static_function(cpptype, f, cppol.get_returntype()) else: d[f] = make_method(f, cppol.get_returntype()) Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.cxx Wed Jul 14 11:09:13 2010 @@ -15,20 +15,20 @@ //=========================================================================== -example01::example01() : somedata(-99) { +example01::example01() : m_somedata(-99) { count++; } -example01::example01(int a) : somedata(a) { +example01::example01(int a) : m_somedata(a) { count++; std::cout << "constructor called" << std::endl; } -example01::example01(const example01& e) : somedata(e.somedata) { +example01::example01(const example01& e) : m_somedata(e.m_somedata) { count++; std::cout << "copy constructor called" << std::endl; } example01& example01::operator=(const example01& e) { if (this != &e) { - somedata = e.somedata; + m_somedata = e.m_somedata; } return *this; } @@ -58,6 +58,11 @@ p->setData(d); } +payload* example01::staticCyclePayload(payload* p, double d) { + staticSetPayload(p, d); + return p; +} + int example01::getCount() { std::cout << "getcount called" << std::endl; return count; @@ -65,19 +70,19 @@ // instance methods int example01::addDataToInt(int a) { - return somedata + a; + return m_somedata + a; } double example01::addDataToDouble(double a) { - return somedata + a; + return m_somedata + a; } int example01::addDataToAtoi(const char* str) { - return ::atoi(str) + somedata; + return ::atoi(str) + m_somedata; } char* example01::addToStringValue(const char* str) { - int out = ::atoi(str) + somedata; + int out = ::atoi(str) + m_somedata; std::ostringstream ss; ss << out << std::ends; std::string result = ss.str(); @@ -87,7 +92,7 @@ } void example01::setPayload(payload* p) { - p->setData(somedata); + p->setData(m_somedata); } payload* example01::cyclePayload(payload* p) { Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/example01.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/example01.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/example01.h Wed Jul 14 11:09:13 2010 @@ -14,7 +14,7 @@ class example01 { public: static int count; - int somedata; + int m_somedata; example01(); example01(int a); @@ -29,6 +29,7 @@ static int staticAtoi(const char* str); static char* staticStrcpy(const char* strin); static void staticSetPayload(payload* p, double d); + static payload* staticCyclePayload(payload* p, double d); static int getCount(); // instance methods Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Wed Jul 14 11:09:13 2010 @@ -153,19 +153,20 @@ assert t.invoke("getCount") == 0 def testReturningOfAnObjectByPointer(self): - """Test passing of an instance as an argument.""" + """Test returing of an instance as an argument.""" t = self.example01 pl = self.payload.construct(3.14) assert round(pl.invoke("getData")-3.14, 8) == 0 + pl2 = t.invoke("staticCyclePayload", pl, 38.) + assert pl2.invoke("getData") == 38. + e = t.construct(50) - e.invoke("setPayload", pl); - assert round(pl.invoke("getData")-50., 8) == 0 - pl = e.invoke("cyclePayload", pl); - assert round(pl.invoke("getData")-50., 8) == 0 + pl2 = e.invoke("cyclePayload", pl); + assert round(pl2.invoke("getData")-50., 8) == 0 e.destruct() pl.destruct() Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Wed Jul 14 11:09:13 2010 @@ -139,9 +139,27 @@ e.setPayload(pl) assert round(pl.getData()-14., 8) == 0 - pl = e.cyclePayload(pl) - assert round(pl.getData()-14., 8) == 0 pl.destruct() e.destruct() assert example01_class.getCount() == 0 + + def testReturningOfAnObjectByPointer(self): + import cppyy + example01_class = cppyy.gbl.example01 + payload_class = cppyy.gbl.payload + + pl = payload_class(3.14) + assert round(pl.getData()-3.14, 8) == 0 + + pl2 = example01_class.staticCyclePayload(pl, 38.) + assert pl2.getData() == 38. + + e = example01_class(14) + + pl2 = e.cyclePayload(pl) + assert round(pl2.getData()-14., 8) == 0 + + pl.destruct() + e.destruct() + assert example01_class.getCount() == 0 From dan at codespeak.net Wed Jul 14 11:51:16 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 14 Jul 2010 11:51:16 +0200 (CEST) Subject: [pypy-svn] r76195 - pypy/branch/micronumpy/pypy/module/micronumpy Message-ID: <20100714095116.686A1282B90@codespeak.net> Author: dan Date: Wed Jul 14 11:51:14 2010 New Revision: 76195 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/array.py pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py Log: More cleaning, time to start passing tests again. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/array.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/array.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/array.py Wed Jul 14 11:51:14 2010 @@ -58,20 +58,5 @@ class BaseNumArray(Wrappable): pass -def descr_new(space, w_cls, w_shape, w_dtype=NoneNotWrapped, - w_buffer=NoneNotWrapped, w_offset=NoneNotWrapped, - w_strides=NoneNotWrapped, order='C'): - shape_w = unpack_shape(space, w_shape) - dtype_w = get_dtype(space, w_dtype) - result = construct_array(space, shape_w, dtype_w) - #TODO: load from buffer? - return space.wrap(result) -descr_new.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root, - W_Root, W_Root, - W_Root, str] - -BaseNumArray.typedef = TypeDef("ndarray", - __new__ = interp2app(descr_new), - ) -base_typedef = BaseNumArray.typedef +BaseNumArray.typedef = TypeDef("ndarray") ndarray = BaseNumArray Modified: pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py Wed Jul 14 11:51:14 2010 @@ -8,17 +8,8 @@ from pypy.rlib.debug import make_sure_not_resized from pypy.module.micronumpy.array import BaseNumArray -from pypy.module.micronumpy.array import base_typedef from pypy.module.micronumpy.array import construct_array, infer_shape -from pypy.module.micronumpy.array import array as array_fromseq from pypy.module.micronumpy.array import validate_index -from pypy.module.micronumpy.array import \ - mul_operation, div_operation, add_operation, sub_operation - -from pypy.module.micronumpy.dtype import unwrap_int, coerce_int -from pypy.module.micronumpy.dtype import unwrap_float, coerce_float -from pypy.module.micronumpy.dtype import create_factory, result_mapping -from pypy.module.micronumpy.dtype import retrieve_dtype def compute_pos(space, indexes, dim): current = 1 @@ -192,11 +183,6 @@ def descr_shape(space, self): return space.newtuple([space.wrap(dim) for dim in self.shape]) -mul = mul_operation() -div = div_operation() -add = add_operation() -sub = sub_operation() - def create_mdarray(data_type, unwrap, coerce): def create_math_operation(f): @@ -464,7 +450,7 @@ descr_str.unwrap_spec = ['self'] MultiDimArray.typedef = \ - TypeDef('ndarray', base_typedef, + TypeDef('mdarray', #XXX: condemned __len__ = interp2app(MultiDimArray.descr_len), __getitem__ = interp2app(MultiDimArray.descr_getitem), __setitem__ = interp2app(MultiDimArray.descr_setitem), @@ -485,8 +471,3 @@ shape = GetSetProperty(descr_shape, cls = MultiDimArray), ) return MultiDimArray - -MultiDimIntArray = create_mdarray(int, unwrap_int, coerce_int) -MultiDimFloatArray = create_mdarray(float, unwrap_float, coerce_float) - -mdresult = create_factory({'i': MultiDimIntArray, 'd': MultiDimFloatArray}) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Wed Jul 14 11:51:14 2010 @@ -1,21 +1,17 @@ ?from pypy.interpreter.baseobjspace import ObjSpace +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError + from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app from pypy.interpreter.gateway import NoneNotWrapped -from pypy.interpreter.baseobjspace import Wrappable -from pypy.interpreter.baseobjspace import W_Root, UnpackValueError -from pypy.objspace.std.sliceobject import W_SliceObject -from pypy.objspace.std.listobject import W_ListObject -from pypy.objspace.std.tupleobject import W_TupleObject from pypy.rlib.debug import make_sure_not_resized from pypy.module import micronumpy from pypy.module.micronumpy.array import BaseNumArray -from pypy.module.micronumpy.array import base_typedef from pypy.module.micronumpy.array import construct_array, infer_shape -from pypy.module.micronumpy.array import validate_index def size_from_shape(shape): size = 1 @@ -254,6 +250,19 @@ def descr_get_shape(space, self): return space.newtuple([space.wrap(x) for x in self.shape]) +#TODO: add to typedef when ready +def descr_new(space, w_cls, w_shape, w_dtype=NoneNotWrapped, + w_buffer=NoneNotWrapped, w_offset=NoneNotWrapped, + w_strides=NoneNotWrapped, order='C'): + shape_w = unpack_shape(space, w_shape) + dtype_w = get_dtype(space, w_dtype) + result = construct_array(space, shape_w, dtype_w) + #TODO: load from buffer + return space.wrap(result) +descr_new.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root, + W_Root, W_Root, + W_Root, str] + MicroArray.typedef = TypeDef('uarray', dtype = GetSetProperty(descr_get_dtype, cls=MicroArray), shape = GetSetProperty(descr_get_shape, cls=MicroArray), @@ -268,38 +277,10 @@ assert a == b, "Invalid assertion I think" # FIXME return a -#def infer_shape(space, w_xs): -def infer_shape_slow(space, w_xs): # gonna kill this after it's been svn for one revision - length = 0 - shape = [] - old = None - try: - w_i = space.iter(w_xs) - while True: - element = space.next(w_i) - try: - shape = infer_shape(space, element) - except OperationError, e: - if e.match(space, space.w_TypeError): pass - else: raise - - if old is not None: - shape = reconcile_shapes(space, old, shape) # use to process jagged arrays, if Numpy even allows them - - length += 1 - old = shape - except OperationError, e: - if e.match(space, space.w_StopIteration): - return [length] + shape - else: - raise - -#XXX: don't like having to pass around setitem app_fill_array = gateway.applevel(""" def fill_array(start, a, b): i = 0 for x in b: - print i try: fill_array(start + [i], a, x) except TypeError, e: Modified: pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py Wed Jul 14 11:51:14 2010 @@ -7,11 +7,6 @@ from pypy.objspace.std.sliceobject import W_SliceObject from pypy.objspace.std.tupleobject import W_TupleObject -from pypy.module.micronumpy.array import BaseNumArray -from pypy.module.micronumpy.array import base_typedef -from pypy.module.micronumpy.array import \ - mul_operation, div_operation, add_operation, sub_operation - #TODO: merge unwrap_spec decorator # from pypy.interpreter.gateway import unwrap_spec From dan at codespeak.net Wed Jul 14 12:06:41 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 14 Jul 2010 12:06:41 +0200 (CEST) Subject: [pypy-svn] r76196 - in pypy/branch/micronumpy/pypy/module/micronumpy: . test Message-ID: <20100714100641.3010F282B90@codespeak.net> Author: dan Date: Wed Jul 14 12:06:39 2010 New Revision: 76196 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Log: More passing, less skipping. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py Wed Jul 14 12:06:39 2010 @@ -35,12 +35,6 @@ return space.wrap("dtype('%s')" % self.name) descr_repr.unwrap_spec = ['self', ObjSpace] - def result(self, other): - a = self.dtype.typeid - b = other.dtype.typeid - c = _result_types[(a, b)] - return _descriptors[c].wrappable_dtype() - TypeDescr.typedef = TypeDef('dtype', itemsize = GetSetProperty(TypeDescr.descr_itemsize), __eq__ = interp2app(TypeDescr.descr_eq), @@ -141,6 +135,15 @@ (_float_index, _float_index): _float_index, } +def result(a, b): + a = a.typeid + b = b.typeid + c = _result_types[(a, b)] + return _descriptors[c] + +def w_result(w_a, w_b): + return result(w_a.dtype, w_b.dtype).wrappable_dtype() + def from_typecode(s): index = _typeindex[s] return _descriptors[index] Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Wed Jul 14 12:06:39 2010 @@ -254,9 +254,10 @@ def descr_new(space, w_cls, w_shape, w_dtype=NoneNotWrapped, w_buffer=NoneNotWrapped, w_offset=NoneNotWrapped, w_strides=NoneNotWrapped, order='C'): + from pypy.module.micronumpy import dtype shape_w = unpack_shape(space, w_shape) - dtype_w = get_dtype(space, w_dtype) - result = construct_array(space, shape_w, dtype_w) + dtype_w = dtype.get(space, w_dtype) + result = MicroArray(shape_w, dtype_w) #TODO: load from buffer return space.wrap(result) descr_new.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root, Modified: pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Wed Jul 14 12:06:39 2010 @@ -3,45 +3,6 @@ from pypy.conftest import gettestobjspace -class TestSDArray(object): - def test_unwrap(self, space): - skip("Irrelevant interplevel test") - w_int = space.wrap(1) - w_float = space.wrap(1.0) - - from pypy.interpreter.error import OperationError - def interp_raises(exceptions, f, *args, **kwargs): - try: - f(*args, **kwargs) - except OperationError, e: - for ex in exceptions: - if e.match(space, ex): - return - raise - else: - raise AssertionError("Expected one of %s to be raised" % str(exceptions)) - - from pypy.module.micronumpy.dtype import unwrap_int - assert 1 == unwrap_int(space, w_int) - interp_raises((space.w_TypeError,), unwrap_int, space, w_float) - - from pypy.module.micronumpy.dtype import unwrap_float - assert 1.0 == unwrap_float(space, w_float) - #interp_raises((space.w_TypeError,), unwrap_float, space, w_int) #er, shouldn't this raise? - - def test_coerce(self, space): - skip("Irrelevant interplevel test") - w_int = space.wrap(1) - w_float = space.wrap(1.0) - - from pypy.module.micronumpy.dtype import coerce_int - assert 1 == coerce_int(space, w_int) - assert 1 == coerce_int(space, w_float) - - from pypy.module.micronumpy.dtype import coerce_float - assert 1.0 == coerce_float(space, w_int) - assert 1.0 == coerce_float(space, w_float) - class AppTestSDArray(object): def setup_class(cls): cls.space = gettestobjspace(usemodules=('micronumpy',)) @@ -410,28 +371,21 @@ class TestDType(object): def test_lookups(self, space): - skip("Depends on interplevel stuff that doesn't exist really anymore.") - from pypy.module.micronumpy.dtype import retrieve_dtype - from pypy.module.micronumpy.dtype import get_dtype - a = get_dtype(space, space.wrap('i')) - b = get_dtype(space, space.wrap('d')) + from pypy.module.micronumpy import dtype + a = dtype.get(space, space.wrap('i')) + b = dtype.get(space, space.wrap('d')) - assert a == retrieve_dtype(space, 'i') - assert b == retrieve_dtype(space, 'd') + assert a == dtype.int_descr + assert b == dtype.float_descr def test_result_types(self, space): - skip("Depends on interplevel stuff that doesn't exist really anymore.") - from pypy.module.micronumpy.dtype import get_dtype - from pypy.module.micronumpy.dtype import result_mapping - w_typecode_a = space.wrap('i') - w_typecode_b = space.wrap('d') - a = get_dtype(space, w_typecode_a) - b = get_dtype(space, w_typecode_b) - - assert 'i' == result_mapping(space, (w_typecode_a, w_typecode_a)) - assert 'd' == result_mapping(space, (w_typecode_b, w_typecode_a)) - assert 'd' == result_mapping(space, (w_typecode_a, w_typecode_b)) - assert 'd' == result_mapping(space, (w_typecode_b, w_typecode_b)) + from pypy.module.micronumpy import dtype + from pypy.module.micronumpy.dtype import int_descr, float_descr + + assert int_descr == dtype.result(int_descr, int_descr) + assert float_descr == dtype.result(int_descr, float_descr) + assert float_descr == dtype.result(float_descr, int_descr) + assert float_descr == dtype.result(float_descr, float_descr) def test_iterable_type(self, space): from pypy.module.micronumpy.dtype import infer_from_iterable From fijal at codespeak.net Wed Jul 14 12:53:33 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Jul 2010 12:53:33 +0200 (CEST) Subject: [pypy-svn] r76197 - in pypy/extradoc/pypy.org: . source Message-ID: <20100714105333.CB9C7282B90@codespeak.net> Author: fijal Date: Wed Jul 14 12:53:32 2010 New Revision: 76197 Modified: pypy/extradoc/pypy.org/download.html pypy/extradoc/pypy.org/source/download.txt Log: Strike unfinished sentence Modified: pypy/extradoc/pypy.org/download.html ============================================================================== --- pypy/extradoc/pypy.org/download.html (original) +++ pypy/extradoc/pypy.org/download.html Wed Jul 14 12:53:32 2010 @@ -49,7 +49,7 @@

      Download and install

      Here are the various binaries of PyPy 1.3 that we provide for x86 Linux, Mac OS/X or Windows. This release improves over 1.2 in terms of stability -of the JIT. It also

      +of the JIT.

      • Download
        • With a JIT Compiler (recommended!)
          • Modified: pypy/extradoc/pypy.org/source/download.txt ============================================================================== --- pypy/extradoc/pypy.org/source/download.txt (original) +++ pypy/extradoc/pypy.org/source/download.txt Wed Jul 14 12:53:32 2010 @@ -8,7 +8,7 @@ Here are the various binaries of **PyPy 1.3** that we provide for x86 Linux, Mac OS/X or Windows. This release improves over 1.2 in terms of stability -of the JIT. It also +of the JIT. .. class:: download_menu From hakanardo at codespeak.net Wed Jul 14 13:20:20 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 14 Jul 2010 13:20:20 +0200 (CEST) Subject: [pypy-svn] r76198 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100714112020.4CE33282BF8@codespeak.net> Author: hakanardo Date: Wed Jul 14 13:20:18 2010 New Revision: 76198 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/interp_simple.py Log: potential memory leak fixed and some rpython fixes Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Wed Jul 14 13:20:18 2010 @@ -140,10 +140,10 @@ def __del__(self): self.setlen(0) - + def setlen(self, size): - if size > self.allocated or size < self.allocated/2: - if size > 0: #FIXME: allways free 0 + if size > 0: + if size > self.allocated or size < self.allocated/2: if size < 9: some = 3 else: @@ -154,14 +154,19 @@ for i in range(min(size,self.len)): new_buffer[i] = self.buffer[i] else: - assert size == 0 - self.allocated = 0 - new_buffer = lltype.nullptr(mytype.arraytype) - if self.buffer != lltype.nullptr(mytype.arraytype): - lltype.free(self.buffer, flavor='raw') - self.buffer = new_buffer + self.len = size + return + else: + assert size == 0 + self.allocated = 0 + new_buffer = lltype.nullptr(mytype.arraytype) + + if self.buffer != lltype.nullptr(mytype.arraytype): + lltype.free(self.buffer, flavor='raw') + self.buffer = new_buffer self.len = size - setlen.unwrap_spec = ['self', int] + + setlen.unwrap_spec = ['self', int] def descr_len(self): return self.space.wrap(self.len) @@ -553,6 +558,8 @@ array.unwrap_spec = (ObjSpace, str, W_Root) class W_WrappedArray(Wrappable): + _immutable_fields_ = ['_array'] + def __init__(self, typecode, w_initializer=None): pass __init__.unwrap_spec = ['self', str, W_Root] @@ -562,21 +569,31 @@ descr_pop.unwrap_spec = ['self', int] def descr_getitem(self, w_i): - w_item = self._array.descr_getitem(w_i) + space = self._array.space + #w_item = self._array.descr_getitem(w_i) + w_item = space.call_function( + space.getattr(self._array, space.wrap('__getitem__')), w_i) if isinstance(w_item, W_ArrayBase): return wrap_array(w_item) return w_item descr_getitem.unwrap_spec = ['self', W_Root] def descr_iadd(self, w_i): - self._array.descr_iadd(w_i._array) + space = self._array.space + #self._array.descr_iadd(w_i._array) + w_i = space.interp_w(W_WrappedArray, w_i) + w_item = space.call_function( + space.getattr(self._array, space.wrap('__iadd__')), w_i._array) return self descr_iadd.unwrap_spec = ['self', W_Root] - def descr_imul(self, i): - self._array.descr_imul(i) + def descr_imul(self, w_i): + space = self._array.space + #self._array.descr_imul(i) + w_item = space.call_function( + space.getattr(self._array, space.wrap('__imul__')), w_i) return self - descr_imul.unwrap_spec = ['self', int] + descr_imul.unwrap_spec = ['self', W_Root] def descr_reduce(self): space=self._array.space @@ -588,6 +605,8 @@ mod = space.interp_w(MixedModule, w_mod) w_new_inst = mod.get('_new_array') return space.newtuple([w_new_inst, space.newtuple(args)]) + return space.newtuple([space.gettypeobject(W_WrappedArray.typedef), + space.newtuple(args)]) def unwrap_array(w_a): Modified: pypy/branch/interplevel-array/pypy/module/array/interp_simple.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_simple.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_simple.py Wed Jul 14 13:20:18 2010 @@ -11,6 +11,8 @@ class W_SizedFloatArray(Wrappable): + _immutable_fields_ = ['size', 'buffer'] + @dont_look_inside def __init__(self, space, size): self.space = space From hakanardo at codespeak.net Wed Jul 14 13:39:55 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 14 Jul 2010 13:39:55 +0200 (CEST) Subject: [pypy-svn] r76199 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100714113955.557AD282BF8@codespeak.net> Author: hakanardo Date: Wed Jul 14 13:39:53 2010 New Revision: 76199 Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: reduce Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/__init__.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/__init__.py Wed Jul 14 13:39:53 2010 @@ -9,6 +9,5 @@ interpleveldefs = { 'array': 'interp_array.W_WrappedArray', - '_new_array': 'interp_array.new_array', 'simple_array': 'interp_simple.simple_array', } Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Wed Jul 14 13:39:53 2010 @@ -600,13 +600,7 @@ args=[space.wrap(self._array.typecode)] if self._array.len>0: args += [self._array.descr_tostring()] - from pypy.interpreter.mixedmodule import MixedModule - w_mod = space.getbuiltinmodule('array') - mod = space.interp_w(MixedModule, w_mod) - w_new_inst = mod.get('_new_array') - return space.newtuple([w_new_inst, space.newtuple(args)]) - return space.newtuple([space.gettypeobject(W_WrappedArray.typedef), - space.newtuple(args)]) + return space.newtuple([space.type(self), space.newtuple(args)]) def unwrap_array(w_a): @@ -725,6 +719,7 @@ W_WrappedArray.typedef = TypeDef( 'array', + __module__ = 'array', __new__ = interp2app(new_array_sub), __init__ = interp2app(W_WrappedArray.__init__), pop = interp2app(W_WrappedArray.descr_pop), From hakanardo at codespeak.net Wed Jul 14 14:05:23 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 14 Jul 2010 14:05:23 +0200 (CEST) Subject: [pypy-svn] r76200 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100714120523.35CDC282B90@codespeak.net> Author: hakanardo Date: Wed Jul 14 14:05:21 2010 New Revision: 76200 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: constructor mess cleaned up Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Wed Jul 14 14:05:21 2010 @@ -560,35 +560,40 @@ class W_WrappedArray(Wrappable): _immutable_fields_ = ['_array'] - def __init__(self, typecode, w_initializer=None): + def __init__(self, space, a): + self.space = space + self._array = a + + def descr_init(self, typecode, w_initializer=None): pass - __init__.unwrap_spec = ['self', str, W_Root] + descr_init.unwrap_spec = ['self', str, W_Root] - def descr_pop(self, i=-1): - return self._array.descr_pop(i) - descr_pop.unwrap_spec = ['self', int] + def descr_pop(self, w_i=-1): + space = self.space + return space.call_function(space.getattr(self._array, space.wrap('pop')), w_i) + descr_pop.unwrap_spec = ['self', W_Root] def descr_getitem(self, w_i): - space = self._array.space + space = self.space #w_item = self._array.descr_getitem(w_i) w_item = space.call_function( space.getattr(self._array, space.wrap('__getitem__')), w_i) if isinstance(w_item, W_ArrayBase): - return wrap_array(w_item) + return W_WrappedArray(space, w_item) return w_item descr_getitem.unwrap_spec = ['self', W_Root] def descr_iadd(self, w_i): - space = self._array.space + space = self.space #self._array.descr_iadd(w_i._array) w_i = space.interp_w(W_WrappedArray, w_i) w_item = space.call_function( space.getattr(self._array, space.wrap('__iadd__')), w_i._array) return self descr_iadd.unwrap_spec = ['self', W_Root] - + def descr_imul(self, w_i): - space = self._array.space + space = self.space #self._array.descr_imul(i) w_item = space.call_function( space.getattr(self._array, space.wrap('__imul__')), w_i) @@ -596,7 +601,7 @@ descr_imul.unwrap_spec = ['self', W_Root] def descr_reduce(self): - space=self._array.space + space=self.space args=[space.wrap(self._array.typecode)] if self._array.len>0: args += [self._array.descr_tostring()] @@ -608,17 +613,12 @@ return w_a._array return w_a -def wrap_array(a): - w_a = W_WrappedArray(None, None) - w_a._array = a - return w_a - def make_descr(fn, nargs, wrp): if nargs == 0: def descr(space, self): ret = space.call_function( space.getattr(self._array, space.wrap(fn))) - if (wrp): return wrap_array(ret) + if (wrp): return W_WrappedArray(space, ret) return ret elif nargs == 1: @@ -626,7 +626,7 @@ ret = space.call_function( space.getattr(self._array, space.wrap(fn)), unwrap_array(w_a)) - if (wrp): return wrap_array(ret) + if (wrp): return W_WrappedArray(space, ret) return ret elif nargs == 2: @@ -634,7 +634,7 @@ ret = space.call_function( space.getattr(self._array, space.wrap(fn)), unwrap_array(w_a), unwrap_array(w_b)) - if (wrp): return wrap_array(ret) + if (wrp): return W_WrappedArray(space, ret) return ret elif nargs == 3: @@ -642,7 +642,7 @@ ret = space.call_function( space.getattr(self._array, space.wrap(fn)), unwrap_array(w_a), unwrap_array(w_b), unwrap_array(w_c)) - if (wrp): return wrap_array(ret) + if (wrp): return W_WrappedArray(space, ret) return ret else: @@ -695,20 +695,14 @@ ): typedefs[fn] = make_descr(fn, nargs, wrp) -def new_array(space, typecode, w_initializer=None): - w_a = W_WrappedArray(None, None) - w_a._array = array(space, typecode, w_initializer) - return w_a -new_array.unwrap_spec = (ObjSpace, str, W_Root) - -def new_array_sub(space, w_cls, typecode, w_initializer=None, w_args=None): - w_array = space.allocate_instance(W_WrappedArray, w_cls) - w_array._array = array(space, typecode, w_initializer) +def new_array(space, w_cls, typecode, w_initializer=None, w_args=None): if len(w_args.arguments_w) > 0: msg = 'array() takes at most 2 arguments' raise OperationError(space.w_TypeError, space.wrap(msg)) + w_array = space.allocate_instance(W_WrappedArray, w_cls) + w_array.__init__(space, array(space, typecode, w_initializer)) return w_array -new_array_sub.unwrap_spec = (ObjSpace, W_Root, str, W_Root, Arguments) +new_array.unwrap_spec = (ObjSpace, W_Root, str, W_Root, Arguments) def descr_wrapped_itemsize(space, self): return space.wrap(self._array.itemsize) @@ -720,8 +714,8 @@ W_WrappedArray.typedef = TypeDef( 'array', __module__ = 'array', - __new__ = interp2app(new_array_sub), - __init__ = interp2app(W_WrappedArray.__init__), + __new__ = interp2app(new_array), + __init__ = interp2app(W_WrappedArray.descr_init), pop = interp2app(W_WrappedArray.descr_pop), __getitem__ = interp2app(W_WrappedArray.descr_getitem), __iadd__ = interp2app(W_WrappedArray.descr_iadd), From hakanardo at codespeak.net Wed Jul 14 15:42:57 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 14 Jul 2010 15:42:57 +0200 (CEST) Subject: [pypy-svn] r76201 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100714134257.69307282B90@codespeak.net> Author: hakanardo Date: Wed Jul 14 15:42:55 2010 New Revision: 76201 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: compiling Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Wed Jul 14 15:42:55 2010 @@ -602,9 +602,11 @@ def descr_reduce(self): space=self.space - args=[space.wrap(self._array.typecode)] - if self._array.len>0: - args += [self._array.descr_tostring()] + if space.int_w(space.len(self._array)) > 0: + w_s = space.call_function(space.getattr(self._array, space.wrap('tostring'))) + args = [space.wrap(self._array.typecode), w_s] + else: + args = [space.wrap(self._array.typecode)] return space.newtuple([space.type(self), space.newtuple(args)]) Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Wed Jul 14 15:42:55 2010 @@ -497,10 +497,14 @@ assert a == b a = self.array('l') - s = pickle.dumps(a,1) + s = pickle.dumps(a, 1) b = pickle.loads(s) assert len(b) == 0 and b.typecode == 'l' + a = self.array('i', [1, 2, 4]) + i = iter(a) + raises(TypeError, pickle.dumps, i, 1) + def test_copy_swap(self): a = self.array('i', [1, 2, 3]) from copy import copy From antocuni at codespeak.net Wed Jul 14 16:55:13 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 14 Jul 2010 16:55:13 +0200 (CEST) Subject: [pypy-svn] r76203 - pypy/trunk/pypy/jit/tool Message-ID: <20100714145513.ABCCC282B90@codespeak.net> Author: antocuni Date: Wed Jul 14 16:55:11 2010 New Revision: 76203 Added: pypy/trunk/pypy/jit/tool/gen-trace-mode-keywords.py (contents, props changed) pypy/trunk/pypy/jit/tool/trace-mode.el Log: an emacs mode to look at JIT traces Added: pypy/trunk/pypy/jit/tool/gen-trace-mode-keywords.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/tool/gen-trace-mode-keywords.py Wed Jul 14 16:55:11 2010 @@ -0,0 +1,8 @@ +import autopath +from pypy.jit.metainterp.resoperation import opname + +for name in opname.values(): + name = name.lower() + if not name.startswith('guard') and name != 'debug_merge_point': + print '"%s"' % name, + Added: pypy/trunk/pypy/jit/tool/trace-mode.el ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/tool/trace-mode.el Wed Jul 14 16:55:11 2010 @@ -0,0 +1,30 @@ +(require 'generic-x) ;; we need this + +(defun set-truncate-lines () + (setq truncate-lines t)) + + +(define-generic-mode + 'pypytrace-mode ;; name of the mode to create + nil ;; comments start with '!!' + '("jump" "finish" "int_add" "int_sub" "int_mul" "int_floordiv" "uint_floordiv" "int_mod" "int_and" "int_or" "int_xor" "int_rshift" "int_lshift" "uint_rshift" "float_add" "float_sub" "float_mul" "float_truediv" "float_neg" "float_abs" "cast_float_to_int" "cast_int_to_float" "int_lt" "int_le" "int_eq" "int_ne" "int_gt" "int_ge" "uint_lt" "uint_le" "uint_gt" "uint_ge" "float_lt" "float_le" "float_eq" "float_ne" "float_gt" "float_ge" "int_is_zero" "int_is_true" "int_neg" "int_invert" "same_as" "ptr_eq" "ptr_ne" "arraylen_gc" "strlen" "strgetitem" "getfield_gc_pure" "getfield_raw_pure" "getarrayitem_gc_pure" "unicodelen" "unicodegetitem" "getarrayitem_gc" "getarrayitem_raw" "getfield_gc" "getfield_raw" "new" "new_with_vtable" "new_array" "force_token" "virtual_ref" "setarrayitem_gc" "setarrayitem_raw" "setfield_gc" "setfield_raw" "arraycopy" "newstr" "strsetitem" "unicodesetitem" "newunicode" "cond_call_gc_wb" "virtual_ref_finish" "call" "call_assembler" "call_may_force" "call_loopinvariant" "call_pure" "int_add_ovf" "int_sub_ovf" "int_mul_ovf") ;; keywords + '( ;; additional regexps + ("^# Loop.*" . 'hi-blue) + ("\\[.*\\]" . 'font-lock-comment-face) ;; comment out argument lists + ("guard_[a-z_]*" . 'widget-button-pressed) + ;; comment out debug_merge_point, but then highlight specific part of it + ("^debug_merge_point.*" . font-lock-comment-face) + ("^\\(debug_merge_point\\).*code object\\(.*\\), file \\('.*'\\), \\(line .*\\)> \\(.*\\)')" + (1 'compilation-warning t) + (2 'compilation-line-number t) + (3 'font-lock-string-face t) + (4 'compilation-line-number t) + (5 'custom-variable-tag t))) + '("\\.trace$") + '(set-truncate-lines) + "A mode for pypy traces files") + +;; debug helpers +(switch-to-buffer-other-window "strslice.log") +(pypytrace-mode) + From antocuni at codespeak.net Wed Jul 14 17:00:10 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 14 Jul 2010 17:00:10 +0200 (CEST) Subject: [pypy-svn] r76204 - pypy/trunk/pypy/jit/tool Message-ID: <20100714150010.21ABC282B90@codespeak.net> Author: antocuni Date: Wed Jul 14 17:00:08 2010 New Revision: 76204 Modified: pypy/trunk/pypy/jit/tool/trace-mode.el Log: minor update Modified: pypy/trunk/pypy/jit/tool/trace-mode.el ============================================================================== --- pypy/trunk/pypy/jit/tool/trace-mode.el (original) +++ pypy/trunk/pypy/jit/tool/trace-mode.el Wed Jul 14 17:00:08 2010 @@ -6,25 +6,26 @@ (define-generic-mode 'pypytrace-mode ;; name of the mode to create - nil ;; comments start with '!!' + nil '("jump" "finish" "int_add" "int_sub" "int_mul" "int_floordiv" "uint_floordiv" "int_mod" "int_and" "int_or" "int_xor" "int_rshift" "int_lshift" "uint_rshift" "float_add" "float_sub" "float_mul" "float_truediv" "float_neg" "float_abs" "cast_float_to_int" "cast_int_to_float" "int_lt" "int_le" "int_eq" "int_ne" "int_gt" "int_ge" "uint_lt" "uint_le" "uint_gt" "uint_ge" "float_lt" "float_le" "float_eq" "float_ne" "float_gt" "float_ge" "int_is_zero" "int_is_true" "int_neg" "int_invert" "same_as" "ptr_eq" "ptr_ne" "arraylen_gc" "strlen" "strgetitem" "getfield_gc_pure" "getfield_raw_pure" "getarrayitem_gc_pure" "unicodelen" "unicodegetitem" "getarrayitem_gc" "getarrayitem_raw" "getfield_gc" "getfield_raw" "new" "new_with_vtable" "new_array" "force_token" "virtual_ref" "setarrayitem_gc" "setarrayitem_raw" "setfield_gc" "setfield_raw" "arraycopy" "newstr" "strsetitem" "unicodesetitem" "newunicode" "cond_call_gc_wb" "virtual_ref_finish" "call" "call_assembler" "call_may_force" "call_loopinvariant" "call_pure" "int_add_ovf" "int_sub_ovf" "int_mul_ovf") ;; keywords '( ;; additional regexps ("^# Loop.*" . 'hi-blue) ("\\[.*\\]" . 'font-lock-comment-face) ;; comment out argument lists ("guard_[a-z_]*" . 'widget-button-pressed) + ("[ip][0-9][0-9]*" . 'font-lock-variable-name-face) ;; comment out debug_merge_point, but then highlight specific part of it ("^debug_merge_point.*" . font-lock-comment-face) ("^\\(debug_merge_point\\).*code object\\(.*\\), file \\('.*'\\), \\(line .*\\)> \\(.*\\)')" (1 'compilation-warning t) - (2 'compilation-line-number t) + (2 'escape-glyph t) (3 'font-lock-string-face t) - (4 'compilation-line-number t) + (4 'escape-glyph t) (5 'custom-variable-tag t))) '("\\.trace$") '(set-truncate-lines) "A mode for pypy traces files") ;; debug helpers -(switch-to-buffer-other-window "strslice.log") -(pypytrace-mode) +;; (switch-to-buffer-other-window "strslice.log") +;; (pypytrace-mode) From antocuni at codespeak.net Wed Jul 14 17:20:18 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 14 Jul 2010 17:20:18 +0200 (CEST) Subject: [pypy-svn] r76205 - pypy/trunk/pypy/jit/tool Message-ID: <20100714152018.CC3D2282B90@codespeak.net> Author: antocuni Date: Wed Jul 14 17:20:17 2010 New Revision: 76205 Added: pypy/trunk/pypy/jit/tool/pypytrace-mode.el - copied, changed from r76204, pypy/trunk/pypy/jit/tool/trace-mode.el Removed: pypy/trunk/pypy/jit/tool/trace-mode.el Log: make sure that it works when (require)d by my ~/.emacs Copied: pypy/trunk/pypy/jit/tool/pypytrace-mode.el (from r76204, pypy/trunk/pypy/jit/tool/trace-mode.el) ============================================================================== --- pypy/trunk/pypy/jit/tool/trace-mode.el (original) +++ pypy/trunk/pypy/jit/tool/pypytrace-mode.el Wed Jul 14 17:20:17 2010 @@ -1,9 +1,13 @@ -(require 'generic-x) ;; we need this +(provide 'pypytrace-mode) +(eval-when-compile + (require 'generic-x) + (require 'hi-lock) + (require 'compile) + (require 'cus-edit)) (defun set-truncate-lines () (setq truncate-lines t)) - (define-generic-mode 'pypytrace-mode ;; name of the mode to create nil @@ -12,7 +16,8 @@ ("^# Loop.*" . 'hi-blue) ("\\[.*\\]" . 'font-lock-comment-face) ;; comment out argument lists ("guard_[a-z_]*" . 'widget-button-pressed) - ("[ip][0-9][0-9]*" . 'font-lock-variable-name-face) + ("\\(ptr\\|p\\)[0-9][0-9]*" . 'font-lock-variable-name-face) + ("i[0-9][0-9]*" . 'custom-button-pressed-unraised) ;; comment out debug_merge_point, but then highlight specific part of it ("^debug_merge_point.*" . font-lock-comment-face) ("^\\(debug_merge_point\\).*code object\\(.*\\), file \\('.*'\\), \\(line .*\\)> \\(.*\\)')" @@ -26,6 +31,6 @@ "A mode for pypy traces files") ;; debug helpers -;; (switch-to-buffer-other-window "strslice.log") +;; (switch-to-buffer-other-window "strslice.trace") ;; (pypytrace-mode) From wlav at codespeak.net Wed Jul 14 17:29:53 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Wed, 14 Jul 2010 17:29:53 +0200 (CEST) Subject: [pypy-svn] r76208 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src test Message-ID: <20100714152953.57B3F282B90@codespeak.net> Author: wlav Date: Wed Jul 14 17:29:51 2010 New Revision: 76208 Added: pypy/branch/reflex-support/pypy/module/cppyy/test/datatypes.cxx (contents, props changed) pypy/branch/reflex-support/pypy/module/cppyy/test/datatypes.h (contents, props changed) pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py (contents, props changed) Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Log: Preparation of new test for the builtin datatypes (modified from roottest), and a bool executor/converter. Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Wed Jul 14 17:29:51 2010 @@ -45,6 +45,10 @@ "cppyy_call_v", [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], lltype.Void, compilation_info=eci) +c_call_b = rffi.llexternal( + "cppyy_call_b", + [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.INT, + compilation_info=eci) c_call_l = rffi.llexternal( "cppyy_call_l", [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.LONG, Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Wed Jul 14 17:29:51 2010 @@ -13,12 +13,22 @@ lltype.free(arg, flavor='raw') +class BoolConverter(TypeConverter): + def convert_argument(self, space, w_obj): + arg = space.c_int_w(w_obj) + if arg != False and arg != True: + raise OperationError(space.w_TypeError, + space.wrap("boolean value should be bool, or integer 1 or 0")) + x = lltype.malloc(rffi.LONGP.TO, 1, flavor='raw') + x[0] = arg + return rffi.cast(rffi.VOIDP, x) + class IntConverter(TypeConverter): def convert_argument(self, space, w_obj): arg = space.c_int_w(w_obj) x = lltype.malloc(rffi.LONGP.TO, 1, flavor='raw') x[0] = arg - return rffi.cast(rffi.VOIDP, x) + return rffi.cast(rffi.VOIDP, x) class DoubleConverter(TypeConverter): def convert_argument(self, space, w_obj): @@ -33,6 +43,7 @@ x = rffi.str2charp(arg) return rffi.cast(rffi.VOIDP, x) + class InstancePtrConverter(TypeConverter): _immutable_ = True def __init__(self, space, cpptype): @@ -76,6 +87,7 @@ raise OperationError(space.w_TypeError, space.wrap("no clue what %s is" % name)) +_converters["bool"] = BoolConverter() _converters["int"] = IntConverter() _converters["double"] = DoubleConverter() _converters["const char*"] = CStringConverter() Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Wed Jul 14 17:29:51 2010 @@ -15,6 +15,11 @@ capi.c_call_v(func.cpptype.handle, func.method_index, cppthis, num_args, args) return space.w_None +class BoolExecutor(FunctionExecutor): + def execute(self, space, func, cppthis, num_args, args): + result = capi.c_call_b(func.cpptype.handle, func.method_index, cppthis, num_args, args) + return space.wrap(result) + class LongExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): result = capi.c_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) @@ -63,7 +68,8 @@ # raise TypeError("no clue what %s is" % name) _executors["void"] = VoidExecutor() +_executors["bool"] = BoolExecutor() _executors["int"] = LongExecutor() -_executors["long"] = LongExecutor() +_executors["long int"] = LongExecutor() _executors["double"] = DoubleExecutor() _executors["char*"] = CStringExecutor() Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Wed Jul 14 17:29:51 2010 @@ -14,8 +14,9 @@ void* cppyy_allocate(cppyy_typehandle_t handle); void cppyy_deallocate(cppyy_typehandle_t handle, cppyy_object_t instance); - void cppyy_call_v(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); - long cppyy_call_l(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); + void cppyy_call_v(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); + int cppyy_call_b(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); + long cppyy_call_l(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); double cppyy_call_d(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); void cppyy_destruct(cppyy_typehandle_t handle, cppyy_object_t self); cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_typehandle_t handle, int method_index); Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Wed Jul 14 17:29:51 2010 @@ -31,9 +31,10 @@ } } -long cppyy_call_l(cppyy_typehandle_t handle, int method_index, - cppyy_object_t self, int numargs, void* args[]) { - long result; +template +static inline T cppyy_call_T(cppyy_typehandle_t handle, int method_index, + cppyy_object_t self, int numargs, void* args[]) { + T result; std::vector arguments(args, args+numargs); Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); @@ -46,19 +47,19 @@ return result; } +int cppyy_call_b(cppyy_typehandle_t handle, int method_index, + cppyy_object_t self, int numargs, void* args[]) { + return (int)cppyy_call_T(handle, method_index, self, numargs, args); +} + +long cppyy_call_l(cppyy_typehandle_t handle, int method_index, + cppyy_object_t self, int numargs, void* args[]) { + return cppyy_call_T(handle, method_index, self, numargs, args); +} + double cppyy_call_d(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]) { - double result; - std::vector arguments(args, args+numargs); - Reflex::Type t((Reflex::TypeName*)handle); - Reflex::Member m = t.FunctionMemberAt(method_index); - if (self) { - Reflex::Object o(t, self); - m.Invoke(o, result, arguments); - } else { - m.Invoke(result, arguments); - } - return result; + return cppyy_call_T(handle, method_index, self, numargs, args); } void cppyy_destruct(cppyy_typehandle_t handle, cppyy_object_t self) { Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/Makefile Wed Jul 14 17:29:51 2010 @@ -1,3 +1,5 @@ +all: example01Dict.so datatypesDict.so + ROOTSYS := ${ROOTSYS} ifeq ($(ROOTSYS),) @@ -19,3 +21,7 @@ example01Dict.so: example01.cxx example01.h $(genreflex) example01.h $(genreflexflags) g++ -o $@ example01_rflx.cpp example01.cxx -shared -lReflex $(cppflags) $(cppflags2) + +datatypesDict.so: datatypes.cxx datatypes.h + $(genreflex) datatypes.h $(genreflexflags) + g++ -o $@ datatypes_rflx.cpp datatypes.cxx -shared -lReflex $(cppflags) $(cppflags2) Added: pypy/branch/reflex-support/pypy/module/cppyy/test/datatypes.cxx ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/datatypes.cxx Wed Jul 14 17:29:51 2010 @@ -0,0 +1,143 @@ +#include "datatypes.h" + + +cppyy_test_data::cppyy_test_data() : m_owns_arrays(false) +{ + m_bool = false; + m_char = 'a'; + m_uchar = 'c'; + m_short = -11; + m_ushort = 11u; + m_int = -22; + m_uint = 22u; + m_long = -33l; + m_ulong = 33ul; + m_float = -44.f; + m_double = -55.; + + m_short_array2 = new short[N]; + m_ushort_array2 = new unsigned short[N]; + m_int_array2 = new int[N]; + m_uint_array2 = new unsigned int[N]; + m_long_array2 = new long[N]; + m_ulong_array2 = new unsigned long[N]; + + m_float_array2 = new float[N]; + m_double_array2 = new double[N]; + + for (int i = 0; i < N; ++i) { + m_short_array[i] = -1*i; + m_short_array2[i] = -2*i; + m_ushort_array[i] = 3u*i; + m_ushort_array2[i] = 4u*i; + m_int_array[i] = -5*i; + m_int_array2[i] = -6*i; + m_uint_array[i] = 7u*i; + m_uint_array2[i] = 8u*i; + m_long_array[i] = -9l*i; + m_long_array2[i] = -10l*i; + m_ulong_array[i] = 11ul*i; + m_ulong_array2[i] = 12ul*i; + + m_float_array[i] = -13.f*i; + m_float_array2[i] = -14.f*i; + m_double_array[i] = -15.*i; + m_double_array2[i] = -16.*i; + } + + m_owns_arrays = true; + + m_pod.m_int = 888; + m_pod.m_double = 3.14; +}; + +cppyy_test_data::~cppyy_test_data() +{ + destroy_arrays(); +} + +void cppyy_test_data::destroy_arrays() { + if (m_owns_arrays == true) { + delete[] m_short_array2; + delete[] m_ushort_array2; + delete[] m_int_array2; + delete[] m_uint_array2; + delete[] m_long_array2; + delete[] m_ulong_array2; + + delete[] m_float_array2; + delete[] m_double_array2; + + m_owns_arrays = false; + } +} + +// getters +bool cppyy_test_data::get_bool() { return m_bool; } +char cppyy_test_data::get_char() { return m_char; } +unsigned char cppyy_test_data::get_uchar() { return m_uchar; } +short cppyy_test_data::get_short() { return m_short; } +unsigned short cppyy_test_data::get_ushort() { return m_ushort; } +int cppyy_test_data::get_int() { return m_int; } +unsigned int cppyy_test_data::get_uint() { return m_uint; } +long cppyy_test_data::get_long() { return m_long; } +unsigned long cppyy_test_data::get_ulong() { return m_ulong; } +float cppyy_test_data::get_float() { return m_float; } +double cppyy_test_data::get_double() { return m_double; } + +short* cppyy_test_data::get_short_array() { return m_short_array; } +short* cppyy_test_data::get_short_array2() { return m_short_array2; } +unsigned short* cppyy_test_data::get_ushort_array() { return m_ushort_array; } +unsigned short* cppyy_test_data::get_ushort_array2() { return m_ushort_array2; } +int* cppyy_test_data::get_int_array() { return m_int_array; } +int* cppyy_test_data::get_int_array2() { return m_int_array2; } +unsigned int* cppyy_test_data::get_uint_array() { return m_uint_array; } +unsigned int* cppyy_test_data::get_uint_array2() { return m_uint_array2; } +long* cppyy_test_data::get_long_array() { return m_long_array; } +long* cppyy_test_data::get_long_array2() { return m_long_array2; } +unsigned long* cppyy_test_data::get_ulong_array() { return m_ulong_array; } +unsigned long* cppyy_test_data::get_ulong_array2() { return m_ulong_array2; } + +float* cppyy_test_data::get_float_array() { return m_float_array; } +float* cppyy_test_data::get_float_array2() { return m_float_array2; } +double* cppyy_test_data::get_double_array() { return m_double_array; } +double* cppyy_test_data::get_double_array2() { return m_double_array2; } + +// setters +void cppyy_test_data::set_bool(bool b) { m_bool = b; } +void cppyy_test_data::set_char(char c) { m_char = c; } +void cppyy_test_data::set_uchar(unsigned char uc) { m_uchar = uc; } +void cppyy_test_data::set_short(short s) { m_short = s; } +void cppyy_test_data::set_ushort(unsigned short us) { m_ushort = us; } +void cppyy_test_data::set_int(int i) { m_int = i; } +void cppyy_test_data::set_uint(unsigned int ui) { m_uint = ui; } +void cppyy_test_data::set_long(long l) { m_long = l; } +void cppyy_test_data::set_ulong(unsigned long ul) { m_ulong = ul; } +void cppyy_test_data::set_float(float f) { m_float = f; } +void cppyy_test_data::set_double(double d) { m_double = d; } + +char cppyy_test_data::s_char = 's'; +unsigned char cppyy_test_data::s_uchar = 'u'; +short cppyy_test_data::s_short = -101; +unsigned short cppyy_test_data::s_ushort = 255u; +int cppyy_test_data::s_int = -202; +unsigned int cppyy_test_data::s_uint = 202u; +long cppyy_test_data::s_long = -303l; +unsigned long cppyy_test_data::s_ulong = 303ul; +float cppyy_test_data::s_float = -404.f; +double cppyy_test_data::s_double = -505.; + +long get_pod_address(cppyy_test_data& c) +{ + return (long)&c.m_pod; +} + +long get_int_address(cppyy_test_data& c) +{ + return (long)&c.m_pod.m_int; +} + +long get_double_address(cppyy_test_data& c) +{ + return (long)&c.m_pod.m_double; +} Added: pypy/branch/reflex-support/pypy/module/cppyy/test/datatypes.h ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/datatypes.h Wed Jul 14 17:29:51 2010 @@ -0,0 +1,120 @@ +const int N = 5; + + +//=========================================================================== +struct cppyy_test_pod { + int m_int; + double m_double; +}; + + +//=========================================================================== +class cppyy_test_data { +public: + cppyy_test_data(); + ~cppyy_test_data(); + +// getters + bool get_bool(); + char get_char(); + unsigned char get_uchar(); + short get_short(); + unsigned short get_ushort(); + int get_int(); + unsigned int get_uint(); + long get_long(); + unsigned long get_ulong(); + float get_float(); + double get_double(); + + short* get_short_array(); + short* get_short_array2(); + unsigned short* get_ushort_array(); + unsigned short* get_ushort_array2(); + int* get_int_array(); + int* get_int_array2(); + unsigned int* get_uint_array(); + unsigned int* get_uint_array2(); + long* get_long_array(); + long* get_long_array2(); + unsigned long* get_ulong_array(); + unsigned long* get_ulong_array2(); + + float* get_float_array(); + float* get_float_array2(); + double* get_double_array(); + double* get_double_array2(); + +// setters + void set_bool(bool b); + void set_char(char c); + void set_uchar(unsigned char uc); + void set_short(short s); + void set_ushort(unsigned short us); + void set_int(int i); + void set_uint(unsigned int ui); + void set_long(long l); + void set_ulong(unsigned long ul); + void set_float(float f); + void set_double(double d); + +public: +// basic types + bool m_bool; + char m_char; + unsigned char m_uchar; + short m_short; + unsigned short m_ushort; + int m_int; + unsigned int m_uint; + long m_long; + unsigned long m_ulong; + float m_float; + double m_double; + +// array types + short m_short_array[N]; + short* m_short_array2; + unsigned short m_ushort_array[N]; + unsigned short* m_ushort_array2; + int m_int_array[N]; + int* m_int_array2; + unsigned int m_uint_array[N]; + unsigned int* m_uint_array2; + long m_long_array[N]; + long* m_long_array2; + unsigned long m_ulong_array[N]; + unsigned long* m_ulong_array2; + + float m_float_array[N]; + float* m_float_array2; + double m_double_array[N]; + double* m_double_array2; + +// object types + cppyy_test_pod m_pod; + +public: + static char s_char; + static unsigned char s_uchar; + static short s_short; + static unsigned short s_ushort; + static int s_int; + static unsigned int s_uint; + static long s_long; + static unsigned long s_ulong; + static float s_float; + static double s_double; + +private: + void destroy_arrays(); + +private: + bool m_owns_arrays; +}; + + +// global functions +long get_pod_address(cppyy_test_data& c); +long get_int_address(cppyy_test_data& c); +long get_double_address(cppyy_test_data& c); Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Wed Jul 14 17:29:51 2010 @@ -11,7 +11,7 @@ def setup_module(mod): if sys.platform == 'win32': py.test.skip("win32 not supported so far") - err = os.system("cd '%s' && make" % currpath) + err = os.system("cd '%s' && make example01Dict.so" % currpath) if err: raise OSError("'make' failed (see stderr)") Added: pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py ============================================================================== --- (empty file) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Wed Jul 14 17:29:51 2010 @@ -0,0 +1,105 @@ +import py, os, sys +from pypy.conftest import gettestobjspace +from pypy.module.cppyy import interp_cppyy, executor + + +currpath = py.path.local(__file__).dirpath() +shared_lib = str(currpath.join("datatypesDict.so")) + +space = gettestobjspace(usemodules=['cppyy']) + +def setup_module(mod): + if sys.platform == 'win32': + py.test.skip("win32 not supported so far") + err = os.system("cd '%s' && make datatypesDict.so" % currpath) + if err: + raise OSError("'make' failed (see stderr)") + +class AppTestDATATYPES: + N = 5 # should be imported from the dictionary + + def setup_class(cls): + cls.space = space + env = os.environ + cls.w_shared_lib = space.wrap(shared_lib) + cls.w_datatypes = cls.space.appexec([], """(): + import cppyy + return cppyy.load_lib(%r)""" % (shared_lib, )) + + def testLoadLibCache(self): + """Test whether loading a library twice results in the same object.""" + import cppyy + lib2 = cppyy.load_lib(self.shared_lib) + assert self.datatypes is lib2 + + def test2WriteAccess( self ): + """Test write access to instance public data and verify values""" + + import cppyy, sys + cppyy_test_data = cppyy.gbl.cppyy_test_data + + c = cppyy_test_data() + assert isinstance(c, cppyy_test_data) + + # boolean types + c.set_bool(True); + assert c.get_bool() == True + c.set_bool(0); assert c.get_bool() == False + """ + c.m_bool = True; assert c.get_bool() == True + c.set_bool(True); assert c.m_bool == True + c.m_bool = 0; assert c.get_bool() == False + c.set_bool(0); assert c.m_bool == False + """ + raises(TypeError, 'c.set_bool(10)') + + """ + # char types + c.m_char = 'b'; assert c.get_char() == 'b' + c.m_char = 40; assert c.get_char() == chr(40) + c.set_char('c'); assert c.m_char == 'c' + c.set_char(41); assert c.m_char == chr(41) + c.m_uchar = 'd'; assert c.get_uchar() == 'd' + c.m_uchar = 42; assert c.get_uchar() == chr(42) + c.set_uchar('e'); assert c.m_uchar == 'e' + c.set_uchar(43); assert c.m_uchar == chr(43) + + raises(TypeError, 'c.set_char("string")') + raises(TypeError, 'c.set_uchar(-1)') + raises(TypeError, 'c.set_uchar("string")') + + # integer types + names = ['short', 'ushort', 'int', 'uint', 'long', 'ulong'] + for i in range(len(names)): + exec 'c.m_%s = %d' % (names[i],i) + assert eval('c.get_%s()' % names[i]) == i + + for i in range(len(names)): + exec 'c.set_%s = %d' % (names[i],2*i) + assert eval('c.m_%s' % names[i]) == i + + # float types + c.m_float = 0.123; assert round(c.get_float() - 0.123, 5) == 0 + c.set_float( 0.234 ); assert round(c.m_float - 0.234, 5) == 0 + c.m_double = 0.456; assert round(c.get_double() - 0.456, 8) == 0 + c.set_double( 0.567 ); assert round(c.m_double - 0.567, 8) == 0 + + # arrays; there will be pointer copies, so destroy the current ones + c.destroy_arrays() + + # integer arrays + a = range(N) + atypes = [ 'h', 'H', 'i', 'I', 'l', 'L' ] + for j in range(len(names)): + b = array(atypes[j], a) + exec 'c.m_%s_array = b' % names[j] # buffer copies + for i in range(N): + assert eval('c.m_%s_array[i]' % names[j]) == b[i] + + exec 'c.m_%s_array2 = b' % names[j] # pointer copies + b[i] = 28 + for i in range(N): + assert eval('c.m_%s_array2[i]' % names[j]) == b[i] + """ + + c.destruct() Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_pythonify.py Wed Jul 14 17:29:51 2010 @@ -1,4 +1,4 @@ -import py, os +import py, os, sys from pypy.conftest import gettestobjspace from pypy.module.cppyy import interp_cppyy, executor @@ -9,7 +9,11 @@ space = gettestobjspace(usemodules=['cppyy']) def setup_module(mod): - os.system("make") + if sys.platform == 'win32': + py.test.skip("win32 not supported so far") + err = os.system("cd '%s' && make example01Dict.so" % currpath) + if err: + raise OSError("'make' failed (see stderr)") class AppTestPYTHONIFY: def setup_class(cls): From hakanardo at codespeak.net Wed Jul 14 17:44:37 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 14 Jul 2010 17:44:37 +0200 (CEST) Subject: [pypy-svn] r76209 - in pypy/branch/interplevel-array/pypy: jit/metainterp jit/tl module/array/test module/pypyjit/test Message-ID: <20100714154437.6CDF2282B90@codespeak.net> Author: hakanardo Date: Wed Jul 14 17:44:35 2010 New Revision: 76209 Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py Log: extracted 2.6 incompatible parts into a separate test with a skip Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py Wed Jul 14 17:44:35 2010 @@ -623,7 +623,7 @@ def find_rewriteable_constant(self, op, args): try: oldopnum = opboolinvers[op.opnum] - targ = args[0:2] + [ConstInt(oldopnum)] + targ = [args[0], args[1], ConstInt(oldopnum)] oldop = self.pure_operations.get(targ, None) if oldop is not None and oldop.descr is op.descr: value = self.getvalue(oldop.result) @@ -639,7 +639,7 @@ try: oldopnum = opboolreflex[op.opnum] - targ = args[1::-1] + [ConstInt(oldopnum)] + targ = [args[1], args[0], ConstInt(oldopnum)] oldop = self.pure_operations.get(targ, None) if oldop is not None and oldop.descr is op.descr: value = self.getvalue(oldop.result) Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py Wed Jul 14 17:44:35 2010 @@ -37,24 +37,24 @@ ## t2 = time.time() ## print t2 - t1 -## from array import array -## def f(img): -## i=0 -## sa=0 -## while i<4: -## sa+=img[i] -## i+=1 -## return sa - -## img=array('i',(1,2,3,4)) -## print f(img) - -def f(): - a=7 +from array import array +def f(img): i=0 + sa=0 while i<4: - if i<0: break - if i<0: break + sa+=img[i] i+=1 + return sa + +img=array('i',(1,2,3,4)) +print f(img) + +## def f(): +## a=7 +## i=0 +## while i<4: +## if i<0: break +## if i<0: break +## i+=1 -f() +## f() Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Wed Jul 14 17:44:35 2010 @@ -343,6 +343,21 @@ assert repr(arr) == repr(self.array('i', lst)) except ValueError: assert not ok + + def test_reversingslice_pre26(self): + import sys + if sys.version_info >= (2, 6): + py.test.skip('arrays can handle more slice opps than lists in 2.6') + + for a in range(-4,5): + for b in range(-4,5): + for c in [-4, -3, -2, -1, 1, 2, 3, 4]: + lst = [1, 2, 3] + arr=self.array('i', lst) + for vals in ([4,5], [6], []): + try: + lst[a:b:c]=vals + except ValueError: raises(ValueError, "arr[a:b:c]=self.array('i', vals)") Modified: pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py Wed Jul 14 17:44:35 2010 @@ -615,6 +615,17 @@ return total ''', 170, ([], 4999450000L)) + def test_boolrewrite(self): + self.run_source(''' + def main(a,b,c): + sa = 0 + for i in range(100000): + if b < a: sa += 1 + else: sa += 2 + if b >= c: sa += 3 + else: sa += 4 + ''', 0, ([0, 1, 0], 7)) + class AppTestJIT(PyPyCJITTests): def setup_class(cls): if not option.runappdirect: From getxsick at codespeak.net Wed Jul 14 20:04:13 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Wed, 14 Jul 2010 20:04:13 +0200 (CEST) Subject: [pypy-svn] r76212 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100714180413.D05F2282B90@codespeak.net> Author: getxsick Date: Wed Jul 14 20:04:10 2010 New Revision: 76212 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: push_funcaddr is equivalent push_int Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Wed Jul 14 20:04:10 2010 @@ -114,13 +114,10 @@ self.esp = 0 self.push_funcaddr(self.funcaddr) - def push_funcaddr(self, value): - self.cpu.set_future_value_int(self.esp, value) - self.esp += 1 - def push_int(self, value): self.cpu.set_future_value_int(self.esp, value) self.esp += 1 + push_funcaddr = push_int def push_float(self, value): self.cpu.set_future_value_float(self.esp, value) From getxsick at codespeak.net Wed Jul 14 20:15:15 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Wed, 14 Jul 2010 20:15:15 +0200 (CEST) Subject: [pypy-svn] r76213 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100714181515.A68A2282B90@codespeak.net> Author: getxsick Date: Wed Jul 14 20:15:13 2010 New Revision: 76213 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: be more RPython Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Wed Jul 14 20:15:13 2010 @@ -25,11 +25,12 @@ def __init__(self, name): name_ptr = rffi.str2charp(name) try: - self.handler = rdynload.dlopen(name_ptr) + handler = rdynload.dlopen(name_ptr) except rdynload.DLOpenError, e: raise OSError('%s: %s', name, e.msg or 'unspecified error') finally: rffi.free_charp(name_ptr) + self.handler = handler class _Get(object): def __init__(self, cpu, lib, func, args_type, res_type='v'): From afa at codespeak.net Wed Jul 14 20:17:41 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 14 Jul 2010 20:17:41 +0200 (CEST) Subject: [pypy-svn] r76214 - in pypy/branch/unicode_filename-2/pypy: module/posix rlib rlib/test rpython/module Message-ID: <20100714181741.97899282B90@codespeak.net> Author: afa Date: Wed Jul 14 20:17:39 2010 New Revision: 76214 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/rlib/rposix.py pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Unicode version of os.access Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Wed Jul 14 20:17:39 2010 @@ -255,7 +255,7 @@ raise wrap_oserror(space, e) dup2.unwrap_spec = [ObjSpace, "c_int", "c_int"] -def access(space, path, mode): +def access(space, w_path, mode): """ access(path, mode) -> 1 if granted, 0 otherwise @@ -266,12 +266,12 @@ existence, or the inclusive-OR of R_OK, W_OK, and X_OK. """ try: - ok = os.access(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) + ok = dispatch_filename(rposix.access)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) else: return space.wrap(ok) -access.unwrap_spec = [ObjSpace, str, "c_int"] +access.unwrap_spec = [ObjSpace, W_Root, "c_int"] def times(space): Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Wed Jul 14 20:17:39 2010 @@ -81,3 +81,10 @@ return os.listdir(dirname) else: return os.listdir(dirname.encode()) + + at specialize.argtype(0) +def access(path, mode): + if isinstance(path, str): + return os.access(path, mode) + else: + return os.access(path.encode(), mode) Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Wed Jul 14 20:17:39 2010 @@ -28,7 +28,7 @@ self.path = UnicodeWithEncoding(self.ufilename) - def test_access(self): + def test_open(self): def f(): try: fd = rposix.open(self.path, os.O_RDONLY, 0777) @@ -48,6 +48,12 @@ assert interpret(f, []) == os.stat(self.ufilename).st_mtime + def test_access(self): + def f(): + return rposix.access(self.path, os.R_OK) + + assert interpret(f, []) == 1 + def test_unlink(self): def f(): return rposix.unlink(self.path) Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Wed Jul 14 20:17:39 2010 @@ -946,6 +946,21 @@ export_name="ll_os.ll_os_access", oofakeimpl=os_access_oofakeimpl) + @registering_unicode_version(os.access, 2, [0], sys.platform=='win32') + def register_os_access_unicode(self): + os_waccess = self.llexternal(underscore_on_windows + 'waccess', + [rffi.CWCHARP, rffi.INT], + rffi.INT) + + def access_llimpl(path, mode): + # All files are executable on Windows + mode = mode & ~os.X_OK + error = rffi.cast(lltype.Signed, os_waccess(path, mode)) + return error == 0 + + return extdef([unicode, int], s_Bool, llimpl=access_llimpl, + export_name="ll_os.ll_os_waccess") + @registering_if(posix, '_getfullpathname') def register_posix__getfullpathname(self): from pypy.rlib import rwin32 From afa at codespeak.net Wed Jul 14 21:43:55 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 14 Jul 2010 21:43:55 +0200 (CEST) Subject: [pypy-svn] r76215 - in pypy/branch/unicode_filename-2/pypy: module/posix rlib rlib/test rpython/module Message-ID: <20100714194355.CD9CB282B90@codespeak.net> Author: afa Date: Wed Jul 14 21:43:52 2010 New Revision: 76215 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/rlib/rposix.py pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Unicode version of os.chdir() Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Wed Jul 14 21:43:52 2010 @@ -345,13 +345,13 @@ return space.call_method(getcwd(space), 'decode') getcwdu.unwrap_spec = [ObjSpace] -def chdir(space, path): +def chdir(space, w_path): """Change the current working directory to the specified path.""" try: - os.chdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -chdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.chdir)(space, wpath) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chdir.unwrap_spec = [ObjSpace, W_Root] def mkdir(space, path, mode=0777): """Create a directory.""" Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Wed Jul 14 21:43:52 2010 @@ -88,3 +88,10 @@ return os.access(path, mode) else: return os.access(path.encode(), mode) + + at specialize.argtype(0) +def chdir(path): + if isinstance(path, str): + return os.access(path) + else: + return os.access(path.encode()) Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Wed Jul 14 21:43:52 2010 @@ -68,3 +68,17 @@ result = interpret(f, []) assert os.path.basename(self.ufilename) in ll_to_string(result) + + def test_chdir(self): + os.unlink(self.ufilename) + os.mkdir(self.ufilename) + + def f(): + rposix.chdir(self.path) + + curdir = os.getcwd() + try: + interpret(f, []) + assert os.getcwdu() == self.ufilename + finally: + os.chdir(curdir) Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Wed Jul 14 21:43:52 2010 @@ -1412,6 +1412,18 @@ return extdef([str], s_None, llimpl=chdir_llimpl, export_name="ll_os.ll_os_chdir") + @registering_unicode_version(os.chdir, 1, [0], sys.platform=='win32') + def register_os_chdir(self): + os_wchdir = self.llexternal(underscore_on_windows+'wchdir', [rffi.CWCHARP], rffi.INT) + + def chdir_llimpl(path): + res = rffi.cast(lltype.Signed, os_wchdir(path)) + if res < 0: + raise OSError(rposix.get_errno(), "os_chdir failed") + + return extdef([unicode], s_None, llimpl=chdir_llimpl, + export_name="ll_os.ll_os_wchdir") + @registering(os.mkdir) def register_os_mkdir(self): if os.name == 'nt': From afa at codespeak.net Wed Jul 14 21:54:04 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 14 Jul 2010 21:54:04 +0200 (CEST) Subject: [pypy-svn] r76216 - pypy/branch/unicode_filename-2/pypy/module/posix Message-ID: <20100714195404.40388282B90@codespeak.net> Author: afa Date: Wed Jul 14 21:54:02 2010 New Revision: 76216 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Log: Fix translation Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Wed Jul 14 21:54:02 2010 @@ -348,7 +348,7 @@ def chdir(space, w_path): """Change the current working directory to the specified path.""" try: - dispatch_filename(rposix.chdir)(space, wpath) + dispatch_filename(rposix.chdir)(space, w_path) except OSError, e: raise wrap_oserror2(space, e, w_path) chdir.unwrap_spec = [ObjSpace, W_Root] From afa at codespeak.net Wed Jul 14 22:06:33 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 14 Jul 2010 22:06:33 +0200 (CEST) Subject: [pypy-svn] r76217 - in pypy/branch/unicode_filename-2/pypy: rlib rpython/module Message-ID: <20100714200633.8BE25282B90@codespeak.net> Author: afa Date: Wed Jul 14 22:06:32 2010 New Revision: 76217 Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Fix fix Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Wed Jul 14 22:06:32 2010 @@ -92,6 +92,6 @@ @specialize.argtype(0) def chdir(path): if isinstance(path, str): - return os.access(path) + return os.chdir(path) else: - return os.access(path.encode()) + return os.chdir(path.encode()) Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Wed Jul 14 22:06:32 2010 @@ -1413,7 +1413,7 @@ export_name="ll_os.ll_os_chdir") @registering_unicode_version(os.chdir, 1, [0], sys.platform=='win32') - def register_os_chdir(self): + def register_os_chdir_unicode(self): os_wchdir = self.llexternal(underscore_on_windows+'wchdir', [rffi.CWCHARP], rffi.INT) def chdir_llimpl(path): From dan at codespeak.net Thu Jul 15 01:04:23 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Thu, 15 Jul 2010 01:04:23 +0200 (CEST) Subject: [pypy-svn] r76218 - pypy/branch/micronumpy/pypy/module/micronumpy Message-ID: <20100714230423.98302282B90@codespeak.net> Author: dan Date: Thu Jul 15 01:04:21 2010 New Revision: 76218 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py Log: checkmodule.py approved Modified: pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py Thu Jul 15 01:04:21 2010 @@ -1,6 +1,5 @@ ? from pypy.interpreter.mixedmodule import MixedModule -import dtype class Module(MixedModule): def __init__(self, space, w_name): @@ -16,7 +15,7 @@ 'array' : 'microarray.array', 'zeros' : 'microarray.zeros', 'ndarray' : 'array.ndarray', - 'minimum' : 'ufunc.minimum', - 'dot' : 'ufunc.dot', + #'minimum' : 'ufunc.minimum', + #'dot' : 'ufunc.dot', } Modified: pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py Thu Jul 15 01:04:21 2010 @@ -40,7 +40,10 @@ __eq__ = interp2app(TypeDescr.descr_eq), __repr__ = interp2app(TypeDescr.descr_repr), ) - + +storage_type = lltype.Ptr(lltype.Array(lltype.Char)) +null_storage = lltype.nullptr(storage_type.TO) + class DescrBase(object): pass _typeindex = {} @@ -76,7 +79,9 @@ array[index] = value # XXX: let's see if this works def alloc(self, count): - return lltype.malloc(arraytype, count, flavor='raw') + #return lltype.malloc(arraytype, count, flavor='raw') + mem = lltype.malloc(arraytype, count, flavor='raw') + return rffi.cast(storage_type, mem) def free(self, data): lltype.free(data, flavor='raw') @@ -136,12 +141,16 @@ } def result(a, b): + assert isinstance(a, DescrBase) + assert isinstance(b, DescrBase) a = a.typeid b = b.typeid c = _result_types[(a, b)] return _descriptors[c] def w_result(w_a, w_b): + assert isinstance(w_a, TypeDescr) + assert isinstance(w_b, TypeDescr) return result(w_a.dtype, w_b.dtype).wrappable_dtype() def from_typecode(s): Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Thu Jul 15 01:04:21 2010 @@ -12,6 +12,7 @@ from pypy.module import micronumpy from pypy.module.micronumpy.array import BaseNumArray from pypy.module.micronumpy.array import construct_array, infer_shape +from pypy.module.micronumpy.dtype import null_storage def size_from_shape(shape): size = 1 @@ -79,7 +80,7 @@ elif parent is not None: self.data = parent.data else: - self.data = None + self.data = null_storage def descr_len(self, space): return space.wrap(self.shape[0]) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py Thu Jul 15 01:04:21 2010 @@ -1,4 +1,4 @@ -?from pypy.module.micronumpy.array import array, zeros, ndarray +?from pypy.module.micronumpy.microarray import array, zeros from pypy.module.micronumpy.array import construct_array from pypy.interpreter.baseobjspace import ObjSpace, W_Root @@ -15,7 +15,8 @@ raise OperationError(space.w_ValueError, space.wrap("minimum of arrays of different length")) - result_type = w_a.dtype.result(w_b.dtype) + from pypy.module.micronumpy.dtype import result + result_type = result(w_a.dtype, w_b.dtype) result = MicroArray(w_a.shape, result_type) @@ -32,30 +33,29 @@ minimum.unwrap_spec = [ObjSpace, W_Root, W_Root] def dot(space, w_a, w_b): - if not isinstance(w_a, ndarray) or not isinstance(w_b, ndarray): + if not isinstance(w_a, MicroArray) or not isinstance(w_b, MicroArray): raise OperationError(space.w_TypeError, space.wrap("expecting ndarray object")) if len(w_b.shape) == 1: - the_b = mdresult(space, w_b.dtype)(space, [w_b.shape[0], 1], w_b.dtype) - the_b.storage[:] = w_b.storage + the_b = MicroArray([w_b.shape[0], 1], w_b.dtype) + #the_b.storage[:] = w_b.storage #FIXME: copy storage w_b = the_b if len(w_a.shape)%2: - the_a = mdresult(space, w_a.dtype)(space, [1]+w_a.shape, w_a.dtype) - the_a.storage[:] = w_a.storage + the_a = MicroArray([1]+w_a.shape, w_a.dtype) + #the_a.storage[:] = w_a.storage #FIXME: ditto w_a = the_a ah, aw = w_a.shape[0], w_a.shape[1] als = len(w_a.shape) if len(w_b.shape)%2: - the_b = mdresult(space, w_b.dtype)(space, [1]+w_b.shape, w_b.dtype) - the_b.storage[:] = w_b.storage + the_b = MicroArray([1]+w_b.shape, w_b.dtype) + #the_b.storage[:] = w_b.storage #FIXME: and again w_b = the_b bh, bw = w_a.shape[0], w_a.shape[1] bls = len(w_b.shape) - if aw == bh == 1: return dot(space, w_b, w_a) @@ -68,8 +68,8 @@ if als != bls: raise shapemismatch - - data = _dotit(space, w_a, w_b, als, [], []) + #data = _dotit(space, w_a, w_b, als, [], []) #FIXME: ok this definitely won't work here + data = [0] #FIXME: obviously wrong... if len(data) == 1: return space.wrap(data[0]) shape = make_sure_not_resized([0]*als) @@ -77,10 +77,11 @@ shape[i] = w_a.shape[i] shape[i+1] = w_b.shape[i+1] - dtype = result_mapping(space, (w_a.dtype, w_b.dtype)) + from pypy.module.micronumpy.dtype import w_result + dtype = w_result(w_a.dtype, w_b.dtype) - res = construct_array(space, shape, retrieve_dtype(space, dtype)) - res.storage[:] = data + res = construct_array(space, shape, dtype) + #res.storage[:] = data #FIXME: more copying return space.wrap(res) dot.unwrap_spec = [ObjSpace, W_Root, W_Root] From dan at codespeak.net Thu Jul 15 01:06:35 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Thu, 15 Jul 2010 01:06:35 +0200 (CEST) Subject: [pypy-svn] r76219 - pypy/branch/micronumpy/pypy/tool Message-ID: <20100714230635.1B0C0282B90@codespeak.net> Author: dan Date: Thu Jul 15 01:06:33 2010 New Revision: 76219 Added: pypy/branch/micronumpy/pypy/tool/numpybench.py Log: Ooops, forgot the benchmark. I want everything under control before merging trunk into here. Added: pypy/branch/micronumpy/pypy/tool/numpybench.py ============================================================================== --- (empty file) +++ pypy/branch/micronumpy/pypy/tool/numpybench.py Thu Jul 15 01:06:33 2010 @@ -0,0 +1,54 @@ +from __future__ import division +try: + import numpy as np +except ImportError, e: + import micronumpy as np + +def naive_convolve(f, g): + # f is an image and is indexed by (v, w) + # g is a filter kernel and is indexed by (s, t), + # it needs odd dimensions + # h is the output image and is indexed by (x, y), + # it is not cropped + if g.shape[0] % 2 != 1 or g.shape[1] % 2 != 1: + raise ValueError("Only odd dimensions on filter supported") + # smid and tmid are number of pixels between the center pixel + # and the edge, ie for a 5x5 filter they will be 2. + # + # The output size is calculated by adding smid, tmid to each + # side of the dimensions of the input image. + vmax = f.shape[0] + wmax = f.shape[1] + smax = g.shape[0] + tmax = g.shape[1] + smid = smax // 2 + tmid = tmax // 2 + xmax = vmax + 2*smid + ymax = wmax + 2*tmid + # Allocate result image. + h = np.zeros([xmax, ymax], dtype=f.dtype) + # Do convolution + for x in range(xmax): + for y in range(ymax): + print "(%d, %d)" % (x, y) + # Calculate pixel value for h at (x,y). Sum one component + # for each pixel (s, t) of the filter g. + s_from = max(smid - x, -smid) + s_to = min((xmax - x) - smid, smid + 1) + t_from = max(tmid - y, -tmid) + t_to = min((ymax - y) - tmid, tmid + 1) + value = 0 + for s in range(s_from, s_to): + for t in range(t_from, t_to): + v = x - smid + s + w = y - tmid + t + value += g[smid - s, tmid - t] * f[v, w] + h[x, y] = value + return h + +if __name__ == '__main__': + image = np.array([[x + y for x in range(200)] for y in range(200)]) + kernel = np.array([[0, 1, 0], + [1, 1, 1], + [0, 1, 0]]) + result = naive_convolve(image, kernel) From afa at codespeak.net Thu Jul 15 01:08:55 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 01:08:55 +0200 (CEST) Subject: [pypy-svn] r76220 - in pypy/branch/unicode_filename-2/pypy: module/posix rlib rlib/test rpython/module Message-ID: <20100714230855.76742282B90@codespeak.net> Author: afa Date: Thu Jul 15 01:08:53 2010 New Revision: 76220 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/rlib/rposix.py pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Unicode versions of os.mkdir & os.rmdir Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Thu Jul 15 01:08:53 2010 @@ -353,21 +353,21 @@ raise wrap_oserror2(space, e, w_path) chdir.unwrap_spec = [ObjSpace, W_Root] -def mkdir(space, path, mode=0777): +def mkdir(space, w_path, mode=0777): """Create a directory.""" try: - os.mkdir(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -mkdir.unwrap_spec = [ObjSpace, str, "c_int"] + dispatch_filename(rposix.mkdir)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +mkdir.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rmdir(space, path): +def rmdir(space, w_path): """Remove a directory.""" try: - os.rmdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -rmdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.rmdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +rmdir.unwrap_spec = [ObjSpace, W_Root] def strerror(space, errno): """Translate an error code to a message string.""" Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Thu Jul 15 01:08:53 2010 @@ -95,3 +95,17 @@ return os.chdir(path) else: return os.chdir(path.encode()) + + at specialize.argtype(0) +def mkdir(path, mode=0777): + if isinstance(path, str): + return os.mkdir(path, mode) + else: + return os.mkdir(path.encode(), mode) + + at specialize.argtype(0) +def rmdir(path): + if isinstance(path, str): + return os.rmdir(path) + else: + return os.rmdir(path.encode()) Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Thu Jul 15 01:08:53 2010 @@ -71,9 +71,9 @@ def test_chdir(self): os.unlink(self.ufilename) - os.mkdir(self.ufilename) def f(): + rposix.mkdir(self.path, 0777) rposix.chdir(self.path) curdir = os.getcwd() @@ -82,3 +82,14 @@ assert os.getcwdu() == self.ufilename finally: os.chdir(curdir) + + def g(): + rposix.rmdir(self.path) + + try: + interpret(g, []) + finally: + try: + os.rmdir(self.ufilename) + except Exception: + pass Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Thu Jul 15 01:08:53 2010 @@ -1446,6 +1446,28 @@ return extdef([str, int], s_None, llimpl=mkdir_llimpl, export_name="ll_os.ll_os_mkdir") + @registering_unicode_version(os.mkdir, 2, [0], sys.platform=='win32') + def register_os_mkdir_unicode(self): + if os.name == 'nt': + ARG2 = [] # no 'mode' argument on Windows - just ignored + else: + ARG2 = [rffi.MODE_T] + os_wmkdir = self.llexternal(underscore_on_windows+'wmkdir', + [rffi.CWCHARP]+ARG2, rffi.INT) + IGNORE_MODE = len(ARG2) == 0 + + def mkdir_llimpl(pathname, mode): + if IGNORE_MODE: + res = os_wmkdir(pathname) + else: + res = os_wmkdir(pathname, mode) + res = rffi.cast(lltype.Signed, res) + if res < 0: + raise OSError(rposix.get_errno(), "os_mkdir failed") + + return extdef([unicode, int], s_None, llimpl=mkdir_llimpl, + export_name="ll_os.ll_os_wmkdir") + @registering(os.rmdir) def register_os_rmdir(self): os_rmdir = self.llexternal(underscore_on_windows+'rmdir', [rffi.CCHARP], rffi.INT) @@ -1458,6 +1480,18 @@ return extdef([str], s_None, llimpl=rmdir_llimpl, export_name="ll_os.ll_os_rmdir") + @registering_unicode_version(os.rmdir, 1, [0], sys.platform=='win32') + def register_os_rmdir_unicode(self): + os_wrmdir = self.llexternal(underscore_on_windows+'wrmdir', [rffi.CWCHARP], rffi.INT) + + def rmdir_llimpl(pathname): + res = rffi.cast(lltype.Signed, os_wrmdir(pathname)) + if res < 0: + raise OSError(rposix.get_errno(), "os_rmdir failed") + + return extdef([unicode], s_None, llimpl=rmdir_llimpl, + export_name="ll_os.ll_os_wrmdir") + @registering(os.chmod) def register_os_chmod(self): os_chmod = self.llexternal(underscore_on_windows+'chmod', [rffi.CCHARP, rffi.MODE_T], From afa at codespeak.net Thu Jul 15 01:21:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 01:21:20 +0200 (CEST) Subject: [pypy-svn] r76221 - in pypy/branch/unicode_filename-2/pypy: module/posix rlib rpython/module Message-ID: <20100714232120.40BFB282B90@codespeak.net> Author: afa Date: Thu Jul 15 01:21:18 2010 New Revision: 76221 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/rlib/rposix.py pypy/branch/unicode_filename-2/pypy/rlib/rwin32.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: unicode version of nt._getfullpathname() Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Thu Jul 15 01:21:18 2010 @@ -320,14 +320,13 @@ def _getfullpathname(space, path): """helper for ntpath.abspath """ - posix = __import__(os.name) # nt specific try: - fullpath = posix._getfullpathname(path) + dispatch_filename(rposix._getfullpathname)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) - else: + raise wrap_oserror2(space, e, w_path) + else: return space.wrap(fullpath) -_getfullpathname.unwrap_spec = [ObjSpace, str] +_getfullpathname.unwrap_spec = [ObjSpace, W_Root] def getcwd(space): """Return the current working directory.""" Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Thu Jul 15 01:21:18 2010 @@ -109,3 +109,11 @@ return os.rmdir(path) else: return os.rmdir(path.encode()) + +if os.name == 'nt': + import nt + def _getfullpathname(path): + if isinstance(path, str): + return nt._getfullpathname(path) + else: + return nt._getfullpathname(path.encode()) Modified: pypy/branch/unicode_filename-2/pypy/rlib/rwin32.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rwin32.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rwin32.py Thu Jul 15 01:21:18 2010 @@ -39,6 +39,7 @@ LPCVOID = rffi_platform.SimpleType("LPCVOID", rffi.VOIDP) LPSTR = rffi_platform.SimpleType("LPSTR", rffi.CCHARP) LPCSTR = rffi_platform.SimpleType("LPCSTR", rffi.CCHARP) + LPWSTR = rffi_platform.SimpleType("LPWSTR", rffi.CWCHARP) LPCWSTR = rffi_platform.SimpleType("LPCWSTR", rffi.CWCHARP) LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.INTP) SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T) Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Thu Jul 15 01:21:18 2010 @@ -968,7 +968,6 @@ # to get a correct implementation of os.abspath # XXX why do we ignore WINAPI conventions everywhere? LPSTRP = rffi.CArrayPtr(rwin32.LPSTR) - # XXX unicode? GetFullPathName = self.llexternal( 'GetFullPathNameA', [rwin32.LPCSTR, @@ -996,6 +995,41 @@ "ll_os.posix__getfullpathname", llimpl=_getfullpathname_llimpl) + @registering_unicode_version(getattr(posix, '_getfullpathname', None), + 1, [0], sys.platform=='win32') + def register_posix__getfullpathname_unicode(self): + from pypy.rlib import rwin32 + # this nt function is not exposed via os, but needed + # to get a correct implementation of os.abspath + # XXX why do we ignore WINAPI conventions everywhere? + LPWSTRP = rffi.CArrayPtr(rwin32.LPWSTR) + GetFullPathName = self.llexternal( + 'GetFullPathNameW', + [rwin32.LPCWSTR, + rwin32.DWORD, + rwin32.LPSTR, + rffi.CArrayPtr(rwin32.LPWSTR)], + rwin32.DWORD) + + def _getfullpathname_llimpl(lpFileName): + nBufferLength = rwin32.MAX_PATH + 1 + lpBuffer = lltype.malloc(rwin32.LPWSTR.TO, nBufferLength, flavor='raw') + try: + res = GetFullPathName( + lpFileName, rffi.cast(rwin32.DWORD, nBufferLength), + lpBuffer, lltype.nullptr(LPWSTRP.TO)) + if res == 0: + raise rwin32.lastWindowsError("_getfullpathname failed") + result = rffi.wcharp2unicode(lpBuffer) + return result + finally: + lltype.free(lpBuffer, flavor='raw') + + return extdef([unicode], # a single argument which is a str + unicode, # returns a string + "ll_os.posix__wgetfullpathname", + llimpl=_getfullpathname_llimpl) + @registering(os.getcwd) def register_os_getcwd(self): os_getcwd = self.llexternal(underscore_on_windows + 'getcwd', From afa at codespeak.net Thu Jul 15 06:50:36 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 06:50:36 +0200 (CEST) Subject: [pypy-svn] r76222 - pypy/branch/unicode_filename-2/pypy/module/posix Message-ID: <20100715045036.C93E7282BFE@codespeak.net> Author: afa Date: Thu Jul 15 06:50:33 2010 New Revision: 76222 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Log: fix Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Thu Jul 15 06:50:33 2010 @@ -318,7 +318,7 @@ raise wrap_oserror2(space, e, w_path) remove.unwrap_spec = [ObjSpace, W_Root] -def _getfullpathname(space, path): +def _getfullpathname(space, w_path): """helper for ntpath.abspath """ try: dispatch_filename(rposix._getfullpathname)(space, w_path) From afa at codespeak.net Thu Jul 15 07:06:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 07:06:22 +0200 (CEST) Subject: [pypy-svn] r76223 - in pypy/branch/unicode_filename-2/pypy: module/posix rpython/module Message-ID: <20100715050622.33EFE282BFE@codespeak.net> Author: afa Date: Thu Jul 15 07:06:20 2010 New Revision: 76223 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: A real implementation of os.getcwdu() Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Thu Jul 15 07:06:20 2010 @@ -340,8 +340,17 @@ def getcwdu(space): """Return the current working directory as a unicode string.""" - # XXX ascii encoding for now - return space.call_method(getcwd(space), 'decode') + if sys.platform == 'win32': + try: + cur = os.getcwdu() + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(cur) + else: + filesystemencoding = space.sys.filesystemencoding + return space.call_method(getcwd(space), 'decode', + space.wrap(filesystemencoding)) getcwdu.unwrap_spec = [ObjSpace] def chdir(space, w_path): Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Thu Jul 15 07:06:20 2010 @@ -1062,6 +1062,34 @@ "ll_os.ll_os_getcwd", llimpl=os_getcwd_llimpl, oofakeimpl=os_getcwd_oofakeimpl) + @registering(os.getcwdu, condition=sys.platform=='win32') + def register_os_getcwdu(self): + os_wgetcwd = self.llexternal(underscore_on_windows + 'wgetcwd', + [rffi.CWCHARP, rffi.SIZE_T], + rffi.CWCHARP) + + def os_getcwd_llimpl(): + bufsize = 256 + while True: + buf = lltype.malloc(rffi.CWCHARP.TO, bufsize, flavor='raw') + res = os_wgetcwd(buf, rffi.cast(rffi.SIZE_T, bufsize)) + if res: + break # ok + error = rposix.get_errno() + lltype.free(buf, flavor='raw') + if error != errno.ERANGE: + raise OSError(error, "getcwd failed") + # else try again with a larger buffer, up to some sane limit + bufsize *= 4 + if bufsize > 1024*1024: # xxx hard-coded upper limit + raise OSError(error, "getcwd result too large") + result = rffi.wcharp2unicode(res) + lltype.free(buf, flavor='raw') + return result + + return extdef([], unicode, + "ll_os.ll_os_wgetcwd", llimpl=os_getcwd_llimpl) + @registering(os.listdir) def register_os_listdir(self): # we need a different approach on Windows and on Posix From afa at codespeak.net Thu Jul 15 07:07:35 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 07:07:35 +0200 (CEST) Subject: [pypy-svn] r76224 - pypy/branch/unicode_filename-2/pypy/module/posix Message-ID: <20100715050735.E5D23282BFE@codespeak.net> Author: afa Date: Thu Jul 15 07:07:34 2010 New Revision: 76224 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Log: Grrr. Run tests and fix. Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Thu Jul 15 07:07:34 2010 @@ -321,7 +321,7 @@ def _getfullpathname(space, w_path): """helper for ntpath.abspath """ try: - dispatch_filename(rposix._getfullpathname)(space, w_path) + fullpath = dispatch_filename(rposix._getfullpathname)(space, w_path) except OSError, e: raise wrap_oserror2(space, e, w_path) else: From afa at codespeak.net Thu Jul 15 07:27:33 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 07:27:33 +0200 (CEST) Subject: [pypy-svn] r76225 - pypy/branch/unicode_filename-2/pypy/module/posix Message-ID: <20100715052733.305DA282BFC@codespeak.net> Author: afa Date: Thu Jul 15 07:27:31 2010 New Revision: 76225 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Log: "if sys.platform..." is not RPython Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Thu Jul 15 07:27:31 2010 @@ -338,16 +338,18 @@ return space.wrap(cur) getcwd.unwrap_spec = [ObjSpace] -def getcwdu(space): - """Return the current working directory as a unicode string.""" - if sys.platform == 'win32': +if sys.platform == 'win32': + def getcwdu(space): + """Return the current working directory as a unicode string.""" try: cur = os.getcwdu() except OSError, e: raise wrap_oserror(space, e) else: return space.wrap(cur) - else: +else: + def getcwdu(space): + """Return the current working directory as a unicode string.""" filesystemencoding = space.sys.filesystemencoding return space.call_method(getcwd(space), 'decode', space.wrap(filesystemencoding)) From afa at codespeak.net Thu Jul 15 09:07:44 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 09:07:44 +0200 (CEST) Subject: [pypy-svn] r76226 - pypy/branch/unicode_filename-2/pypy/module/posix Message-ID: <20100715070744.52291282BDB@codespeak.net> Author: afa Date: Thu Jul 15 09:07:40 2010 New Revision: 76226 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Log: Fix translation: the return type can be either str or unicode, and must be wrapped inside each branch. Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Thu Jul 15 09:07:40 2010 @@ -321,11 +321,18 @@ def _getfullpathname(space, w_path): """helper for ntpath.abspath """ try: - fullpath = dispatch_filename(rposix._getfullpathname)(space, w_path) + if space.isinstance_w(w_path, space.w_unicode): + path = FileEncoder(space, w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) + else: + path = space.str_w(w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) except OSError, e: raise wrap_oserror2(space, e, w_path) else: - return space.wrap(fullpath) + return w_fullpath _getfullpathname.unwrap_spec = [ObjSpace, W_Root] def getcwd(space): From afa at codespeak.net Thu Jul 15 09:23:08 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 09:23:08 +0200 (CEST) Subject: [pypy-svn] r76227 - pypy/branch/unicode_filename-2/pypy/rpython/module Message-ID: <20100715072308.F27DB282BDB@codespeak.net> Author: afa Date: Thu Jul 15 09:23:07 2010 New Revision: 76227 Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: fix rtype phase Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Thu Jul 15 09:23:07 2010 @@ -1007,7 +1007,7 @@ 'GetFullPathNameW', [rwin32.LPCWSTR, rwin32.DWORD, - rwin32.LPSTR, + rwin32.LPWSTR, rffi.CArrayPtr(rwin32.LPWSTR)], rwin32.DWORD) From antocuni at codespeak.net Thu Jul 15 09:57:21 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 15 Jul 2010 09:57:21 +0200 (CEST) Subject: [pypy-svn] r76228 - in pypy/trunk/pypy/jit/backend: llsupport llsupport/test x86/test Message-ID: <20100715075721.AF2CF282BDB@codespeak.net> Author: antocuni Date: Thu Jul 15 09:57:20 2010 New Revision: 76228 Modified: pypy/trunk/pypy/jit/backend/llsupport/descr.py pypy/trunk/pypy/jit/backend/llsupport/gc.py pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py Log: store the name of the field in the FieldDescr, it makes the tracelog immensely more readable Modified: pypy/trunk/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/descr.py Thu Jul 15 09:57:20 2010 @@ -67,9 +67,11 @@ class BaseFieldDescr(AbstractDescr): offset = 0 # help translation + name = '' _clsname = '' - def __init__(self, offset): + def __init__(self, name, offset): + self.name = name self.offset = offset def sort_key(self): @@ -88,7 +90,7 @@ return self._is_float_field def repr_of_descr(self): - return '<%s %s>' % (self._clsname, self.offset) + return '<%s %s %s>' % (self._clsname, self.name, self.offset) class NonGcPtrFieldDescr(BaseFieldDescr): @@ -113,7 +115,8 @@ offset, _ = symbolic.get_field_token(STRUCT, fieldname, gccache.translate_support_code) FIELDTYPE = getattr(STRUCT, fieldname) - fielddescr = getFieldDescrClass(FIELDTYPE)(offset) + name = '%s.%s' % (STRUCT._name, fieldname) + fielddescr = getFieldDescrClass(FIELDTYPE)(name, offset) cachedict = cache.setdefault(STRUCT, {}) cachedict[fieldname] = fielddescr return fielddescr Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/gc.py Thu Jul 15 09:57:20 2010 @@ -351,7 +351,7 @@ gcrootmap = cls() self.gcrootmap = gcrootmap self.gcrefs = GcRefList() - self.single_gcref_descr = GcPtrFieldDescr(0) + self.single_gcref_descr = GcPtrFieldDescr('', 0) # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py Thu Jul 15 09:57:20 2010 @@ -53,6 +53,10 @@ assert descr_y.__class__ is GcPtrFieldDescr assert descr_z.__class__ is NonGcPtrFieldDescr assert descr_f.__class__ is clsf + assert descr_x.name == 'S.x' + assert descr_y.name == 'S.y' + assert descr_z.name == 'S.z' + assert descr_f.name == 'S.f' if not tsc: assert descr_x.offset < descr_y.offset < descr_z.offset assert descr_x.sort_key() < descr_y.sort_key() < descr_z.sort_key() Modified: pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py Thu Jul 15 09:57:20 2010 @@ -51,7 +51,7 @@ def initialize(self): self.gcrefs = GcRefList() self.gcrefs.initialize() - self.single_gcref_descr = GcPtrFieldDescr(0) + self.single_gcref_descr = GcPtrFieldDescr('', 0) rewrite_assembler = GcLLDescr_framework.rewrite_assembler.im_func From afa at codespeak.net Thu Jul 15 10:14:26 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 10:14:26 +0200 (CEST) Subject: [pypy-svn] r76229 - in pypy/branch/unicode_filename-2/pypy: module/posix rlib rlib/test rpython/module Message-ID: <20100715081426.CBAF8282BDB@codespeak.net> Author: afa Date: Thu Jul 15 10:14:24 2010 New Revision: 76229 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/rlib/rposix.py pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Unicode version of os.chmod() Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Thu Jul 15 10:14:24 2010 @@ -481,13 +481,13 @@ return space.newtuple([space.wrap(fd1), space.wrap(fd2)]) pipe.unwrap_spec = [ObjSpace] -def chmod(space, path, mode): +def chmod(space, w_path, mode): "Change the access permissions of a file." - try: - os.chmod(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -chmod.unwrap_spec = [ObjSpace, str, "c_int"] + try: + dispatch_filename(rposix.chmod)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chmod.unwrap_spec = [ObjSpace, W_Root, "c_int"] def rename(space, old, new): "Rename a file or directory." Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Thu Jul 15 10:14:24 2010 @@ -90,6 +90,13 @@ return os.access(path.encode(), mode) @specialize.argtype(0) +def chmod(path, mode): + if isinstance(path, str): + return os.chmod(path, mode) + else: + return os.chmod(path.encode(), mode) + + at specialize.argtype(0) def chdir(path): if isinstance(path, str): return os.chdir(path) Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Thu Jul 15 10:14:24 2010 @@ -10,10 +10,16 @@ def __init__(self, unistr): self.unistr = unistr - def encode(self): - from pypy.rlib.runicode import unicode_encode_utf_8 - return unicode_encode_utf_8(self.unistr, len(self.unistr), - "strict") + if sys.platform == 'win32': + def encode(self): + from pypy.rlib.runicode import unicode_encode_mbcs + return unicode_encode_mbcs(self.unistr, len(self.unistr), + "strict") + else: + def encode(self): + from pypy.rlib.runicode import unicode_encode_utf_8 + return unicode_encode_utf_8(self.unistr, len(self.unistr), + "strict") def gettext(self): return self.unistr @@ -54,6 +60,12 @@ assert interpret(f, []) == 1 + def test_chmod(self): + def f(): + return rposix.chmod(self.path, 0777) + + interpret(f, []) # does not crash + def test_unlink(self): def f(): return rposix.unlink(self.path) Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Thu Jul 15 10:14:24 2010 @@ -1567,6 +1567,19 @@ return extdef([str, int], s_None, llimpl=chmod_llimpl, export_name="ll_os.ll_os_chmod") + @registering_unicode_version(os.chmod, 2, [0], sys.platform=='win32') + def register_os_chmod_unicode(self): + os_wchmod = self.llexternal(underscore_on_windows+'wchmod', [rffi.CWCHARP, rffi.MODE_T], + rffi.INT) + + def chmod_llimpl(path, mode): + res = rffi.cast(lltype.Signed, os_wchmod(path, rffi.cast(rffi.MODE_T, mode))) + if res < 0: + raise OSError(rposix.get_errno(), "os_chmod failed") + + return extdef([unicode, int], s_None, llimpl=chmod_llimpl, + export_name="ll_os.ll_os_wchmod") + @registering(os.rename) def register_os_rename(self): os_rename = self.llexternal('rename', [rffi.CCHARP, rffi.CCHARP], From antocuni at codespeak.net Thu Jul 15 10:18:07 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 15 Jul 2010 10:18:07 +0200 (CEST) Subject: [pypy-svn] r76230 - pypy/trunk/pypy/jit/tool Message-ID: <20100715081807.8A37B282BDB@codespeak.net> Author: antocuni Date: Thu Jul 15 10:18:06 2010 New Revision: 76230 Modified: pypy/trunk/pypy/jit/tool/pypytrace-mode.el Log: highlight the names of the class and of the field in FieldDescrs Modified: pypy/trunk/pypy/jit/tool/pypytrace-mode.el ============================================================================== --- pypy/trunk/pypy/jit/tool/pypytrace-mode.el (original) +++ pypy/trunk/pypy/jit/tool/pypytrace-mode.el Thu Jul 15 10:18:06 2010 @@ -18,6 +18,12 @@ ("guard_[a-z_]*" . 'widget-button-pressed) ("\\(ptr\\|p\\)[0-9][0-9]*" . 'font-lock-variable-name-face) ("i[0-9][0-9]*" . 'custom-button-pressed-unraised) + ("\\(descr=<.*FieldDescr \\)\\([^ ]*\\.\\)\\([^ ]*\\)\\( .*>\\)" + (1 'font-lock-comment-face) + (2 'font-lock-variable-name-face) + (3 'escape-glyph) + (4 'font-lock-comment-face)) + ("<.*FieldDescr \\([^ ]*\\)" (1 'font-lock-variable-name-face)) ;; comment out debug_merge_point, but then highlight specific part of it ("^debug_merge_point.*" . font-lock-comment-face) ("^\\(debug_merge_point\\).*code object\\(.*\\), file \\('.*'\\), \\(line .*\\)> \\(.*\\)')" @@ -31,6 +37,6 @@ "A mode for pypy traces files") ;; debug helpers -;; (switch-to-buffer-other-window "strslice.trace") +;; (switch-to-buffer-other-window "strslice2.trace") ;; (pypytrace-mode) From afa at codespeak.net Thu Jul 15 11:25:09 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 11:25:09 +0200 (CEST) Subject: [pypy-svn] r76231 - in pypy/branch/unicode_filename-2/pypy: module/posix rlib rlib/test rpython/module Message-ID: <20100715092509.89EB6282BF1@codespeak.net> Author: afa Date: Thu Jul 15 11:25:07 2010 New Revision: 76231 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/rlib/rposix.py pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Unicode version of os.rename() Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Thu Jul 15 11:25:07 2010 @@ -18,12 +18,27 @@ self.space = space self.w_obj = w_obj - def encode(self): + def as_bytes(self): return self.space.path_w(self.w_obj) - def gettext(self): + def as_unicode(self): return self.space.unicode_w(self.w_obj) +class FileDecoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + return self.space.path_w(self.w_obj) + + def as_unicode(self): + space = self.space + filesystemencoding = space.sys.filesystemencoding + w_unicode = space.call_method(self.w_obj, 'decode', + space.wrap(filesystemencoding)) + return space.unicode_w(w_unicode) + @specialize.memo() def dispatch_filename(func): def dispatch(space, w_fname, *args): @@ -35,6 +50,27 @@ return func(fname, *args) return dispatch + at specialize.memo() +def dispatch_filename_2(func): + def dispatch(space, w_fname1, w_fname2, *args): + if space.isinstance_w(w_fname1, space.w_unicode): + fname1 = FileEncoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname1 = FileDecoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + return dispatch + def open(space, w_fname, flag, mode=0777): """Open a file (for low level IO). Return a file descriptor (a small integer).""" @@ -489,13 +525,13 @@ raise wrap_oserror2(space, e, w_path) chmod.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rename(space, old, new): +def rename(space, w_old, w_new): "Rename a file or directory." - try: - os.rename(old, new) - except OSError, e: + try: + dispatch_filename_2(rposix.rename)(space, w_old, w_new) + except OSError, e: raise wrap_oserror(space, e) -rename.unwrap_spec = [ObjSpace, str, str] +rename.unwrap_spec = [ObjSpace, W_Root, W_Root] def umask(space, mask): "Set the current numeric umask and return the previous umask." Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Thu Jul 15 11:25:07 2010 @@ -52,70 +52,74 @@ if isinstance(path, str): return os.open(path, flags, mode) else: - return os.open(path.encode(), flags, mode) + return os.open(path.as_bytes(), flags, mode) @specialize.argtype(0) def stat(path): if isinstance(path, str): return os.stat(path) else: - return os.stat(path.encode()) + return os.stat(path.as_bytes()) @specialize.argtype(0) def lstat(path): if isinstance(path, str): return os.lstat(path) else: - return os.lstat(path.encode()) + return os.lstat(path.as_bytes()) @specialize.argtype(0) def unlink(path): if isinstance(path, str): return os.unlink(path) else: - return os.unlink(path.encode()) + return os.unlink(path.as_bytes()) + + at specialize.argtype(0, 1) +def rename(path1, path2): + return os.rename(path1.as_bytes(), path2.as_bytes()) @specialize.argtype(0) def listdir(dirname): if isinstance(dirname, str): return os.listdir(dirname) else: - return os.listdir(dirname.encode()) + return os.listdir(dirname.as_bytes()) @specialize.argtype(0) def access(path, mode): if isinstance(path, str): return os.access(path, mode) else: - return os.access(path.encode(), mode) + return os.access(path.as_bytes(), mode) @specialize.argtype(0) def chmod(path, mode): if isinstance(path, str): return os.chmod(path, mode) else: - return os.chmod(path.encode(), mode) + return os.chmod(path.as_bytes(), mode) @specialize.argtype(0) def chdir(path): if isinstance(path, str): return os.chdir(path) else: - return os.chdir(path.encode()) + return os.chdir(path.as_bytes()) @specialize.argtype(0) def mkdir(path, mode=0777): if isinstance(path, str): return os.mkdir(path, mode) else: - return os.mkdir(path.encode(), mode) + return os.mkdir(path.as_bytes(), mode) @specialize.argtype(0) def rmdir(path): if isinstance(path, str): return os.rmdir(path) else: - return os.rmdir(path.encode()) + return os.rmdir(path.as_bytes()) if os.name == 'nt': import nt @@ -123,4 +127,4 @@ if isinstance(path, str): return nt._getfullpathname(path) else: - return nt._getfullpathname(path.encode()) + return nt._getfullpathname(path.as_bytes()) Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Thu Jul 15 11:25:07 2010 @@ -11,17 +11,17 @@ self.unistr = unistr if sys.platform == 'win32': - def encode(self): + def as_bytes(self): from pypy.rlib.runicode import unicode_encode_mbcs return unicode_encode_mbcs(self.unistr, len(self.unistr), "strict") else: - def encode(self): + def as_bytes(self): from pypy.rlib.runicode import unicode_encode_utf_8 return unicode_encode_utf_8(self.unistr, len(self.unistr), "strict") - def gettext(self): + def as_unicode(self): return self.unistr class TestPosixUnicode: @@ -32,7 +32,8 @@ f.write("test") f.close() - self.path = UnicodeWithEncoding(self.ufilename) + self.path = UnicodeWithEncoding(self.ufilename) + self.path2 = UnicodeWithEncoding(self.ufilename + ".new") def test_open(self): def f(): @@ -73,6 +74,14 @@ interpret(f, []) assert not os.path.exists(self.ufilename) + def test_rename(self): + def f(): + return rposix.rename(self.path, self.path2) + + interpret(f, []) + assert not os.path.exists(self.ufilename) + assert os.path.exists(self.ufilename + '.new') + def test_listdir(self): udir = UnicodeWithEncoding(os.path.dirname(self.ufilename)) def f(): Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Thu Jul 15 11:25:07 2010 @@ -39,6 +39,9 @@ if not condition: return registering(None, condition=False) + func_name = func.__name__ + + @func_renamer(func_name + "_unicode") def unicodefunc(*args): return func(*args) @@ -47,13 +50,12 @@ arglist = ['arg%d' % (i,) for i in range(nbargs)] transformed_arglist = arglist[:] for i in argnums: - transformed_arglist[i] = transformed_arglist[i] + '.gettext()' + transformed_arglist[i] = transformed_arglist[i] + '.as_unicode()' args = ', '.join(arglist) transformed_args = ', '.join(transformed_arglist) main_arg = 'arg%d' % (argnum,) - func_name = func.__name__ source = py.code.Source(""" def %(func_name)s(%(args)s): if isinstance(%(main_arg)s, str): @@ -1593,6 +1595,19 @@ return extdef([str, str], s_None, llimpl=rename_llimpl, export_name="ll_os.ll_os_rename") + @registering_unicode_version(os.rename, 2, [0, 1], sys.platform=='win32') + def register_os_rename_unicode(self): + os_wrename = self.llexternal(underscore_on_windows+'wrename', + [rffi.CWCHARP, rffi.CWCHARP], rffi.INT) + + def rename_llimpl(oldpath, newpath): + res = rffi.cast(lltype.Signed, os_wrename(oldpath, newpath)) + if res < 0: + raise OSError(rposix.get_errno(), "os_rename failed") + + return extdef([unicode, unicode], s_None, llimpl=rename_llimpl, + export_name="ll_os.ll_os_wrename") + @registering(os.umask) def register_os_umask(self): os_umask = self.llexternal(underscore_on_windows+'umask', [rffi.MODE_T], rffi.MODE_T) From afa at codespeak.net Thu Jul 15 11:30:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 11:30:22 +0200 (CEST) Subject: [pypy-svn] r76232 - pypy/branch/unicode_filename-2/pypy/rlib Message-ID: <20100715093022.58E8D282BF1@codespeak.net> Author: afa Date: Thu Jul 15 11:30:20 2010 New Revision: 76232 Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Log: Keep the path where both args are raw strings Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Thu Jul 15 11:30:20 2010 @@ -77,7 +77,10 @@ @specialize.argtype(0, 1) def rename(path1, path2): - return os.rename(path1.as_bytes(), path2.as_bytes()) + if isinstance(path1, str): + return os.rename(path1, path2) + else: + return os.rename(path1.as_bytes(), path2.as_bytes()) @specialize.argtype(0) def listdir(dirname): From afa at codespeak.net Thu Jul 15 11:58:49 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 11:58:49 +0200 (CEST) Subject: [pypy-svn] r76233 - in pypy/branch/unicode_filename-2/pypy: module/posix rlib rlib/test rpython/module Message-ID: <20100715095849.929A9282BF1@codespeak.net> Author: afa Date: Thu Jul 15 11:58:47 2010 New Revision: 76233 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/rlib/rposix.py pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Unicode version of os.utime() Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Thu Jul 15 11:58:47 2010 @@ -659,7 +659,7 @@ raise wrap_oserror(space, e) execve.unwrap_spec = [ObjSpace, str, W_Root, W_Root] -def utime(space, path, w_tuple): +def utime(space, w_path, w_tuple): """ utime(path, (atime, mtime)) utime(path, None) @@ -668,10 +668,10 @@ """ if space.is_w(w_tuple, space.w_None): try: - os.utime(path, None) + dispatch_filename(rposix.utime)(space, w_path, None) return except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) try: msg = "utime() arg 2 must be a tuple (atime, mtime) or None" args_w = space.fixedview(w_tuple) @@ -679,14 +679,14 @@ raise OperationError(space.w_TypeError, space.wrap(msg)) actime = space.float_w(args_w[0]) modtime = space.float_w(args_w[1]) - os.utime(path, (actime, modtime)) + dispatch_filename(rposix.utime)(space, w_path, (actime, modtime)) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) except OperationError, e: if not e.match(space, space.w_TypeError): raise raise OperationError(space.w_TypeError, space.wrap(msg)) -utime.unwrap_spec = [ObjSpace, str, W_Root] +utime.unwrap_spec = [ObjSpace, W_Root, W_Root] def setsid(space): """setsid() -> pid Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Thu Jul 15 11:58:47 2010 @@ -104,6 +104,13 @@ return os.chmod(path.as_bytes(), mode) @specialize.argtype(0) +def utime(path, times): + if isinstance(path, str): + return os.utime(path, times) + else: + return os.utime(path.as_bytes(), times) + + at specialize.argtype(0) def chdir(path): if isinstance(path, str): return os.chdir(path) Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Thu Jul 15 11:58:47 2010 @@ -61,6 +61,12 @@ assert interpret(f, []) == 1 + def test_utime(self): + def f(): + return rposix.utime(self.path, None) + + interpret(f, []) # does not crash + def test_chmod(self): def f(): return rposix.chmod(self.path, 0777) Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Thu Jul 15 11:58:47 2010 @@ -12,7 +12,8 @@ from pypy.rlib.rarithmetic import r_longlong from pypy.rpython.extfunc import BaseLazyRegistering from pypy.rpython.extfunc import registering, registering_if, extdef -from pypy.annotation.model import SomeInteger, SomeString, SomeTuple, SomeFloat +from pypy.annotation.model import ( + SomeInteger, SomeString, SomeTuple, SomeFloat, SomeUnicodeString) from pypy.annotation.model import s_ImpossibleValue, s_None, s_Bool from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype @@ -498,6 +499,112 @@ "ll_os.ll_os_utime", llimpl=os_utime_llimpl) + # XXX LOT OF DUPLICATED CODE! + @registering_unicode_version(os.utime, 2, [0], sys.platform=='win32') + def register_os_utime_unicode(self): + from pypy.rlib import rwin32 + from pypy.rpython.module.ll_os_stat import time_t_to_FILE_TIME + + class CConfig: + _compilation_info_ = ExternalCompilationInfo( + includes = ['windows.h'], + ) + + FILE_WRITE_ATTRIBUTES = platform.ConstantInteger( + 'FILE_WRITE_ATTRIBUTES') + OPEN_EXISTING = platform.ConstantInteger( + 'OPEN_EXISTING') + FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger( + 'FILE_FLAG_BACKUP_SEMANTICS') + globals().update(platform.configure(CConfig)) + + CreateFile = rffi.llexternal( + 'CreateFileW', + [rwin32.LPCWSTR, rwin32.DWORD, rwin32.DWORD, + rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD, + rwin32.HANDLE], + rwin32.HANDLE, + calling_conv='win') + + GetSystemTime = rffi.llexternal( + 'GetSystemTime', + [lltype.Ptr(rwin32.SYSTEMTIME)], + lltype.Void, + calling_conv='win') + + SystemTimeToFileTime = rffi.llexternal( + 'SystemTimeToFileTime', + [lltype.Ptr(rwin32.SYSTEMTIME), + lltype.Ptr(rwin32.FILETIME)], + rwin32.BOOL, + calling_conv='win') + + SetFileTime = rffi.llexternal( + 'SetFileTime', + [rwin32.HANDLE, + lltype.Ptr(rwin32.FILETIME), + lltype.Ptr(rwin32.FILETIME), + lltype.Ptr(rwin32.FILETIME)], + rwin32.BOOL, + calling_conv = 'win') + + def os_utime_llimpl(path, tp): + hFile = CreateFile(path, + FILE_WRITE_ATTRIBUTES, 0, + None, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, 0) + if hFile == rwin32.INVALID_HANDLE_VALUE: + raise rwin32.lastWindowsError() + ctime = lltype.nullptr(rwin32.FILETIME) + atime = lltype.malloc(rwin32.FILETIME, flavor='raw') + mtime = lltype.malloc(rwin32.FILETIME, flavor='raw') + try: + if tp is None: + now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw') + try: + GetSystemTime(now) + if (not SystemTimeToFileTime(now, atime) or + not SystemTimeToFileTime(now, mtime)): + raise rwin32.lastWindowsError() + finally: + lltype.free(now, flavor='raw') + else: + actime, modtime = tp + time_t_to_FILE_TIME(actime, atime) + time_t_to_FILE_TIME(modtime, mtime) + if not SetFileTime(hFile, ctime, atime, mtime): + raise rwin32.lastWindowsError() + finally: + rwin32.CloseHandle(hFile) + lltype.free(atime, flavor='raw') + lltype.free(mtime, flavor='raw') + os_utime_llimpl._annspecialcase_ = 'specialize:argtype(1)' + + s_string = SomeUnicodeString() + s_tuple_of_2_floats = SomeTuple([SomeFloat(), SomeFloat()]) + + def os_utime_normalize_args(s_path, s_times): + # special handling of the arguments: they can be either + # [str, (float, float)] or [str, s_None], and get normalized + # to exactly one of these two. + if not s_string.contains(s_path): + raise Exception("os.utime() arg 1 must be a string, got %s" % ( + s_path,)) + case1 = s_None.contains(s_times) + case2 = s_tuple_of_2_floats.contains(s_times) + if case1 and case2: + return [s_string, s_ImpossibleValue] #don't know which case yet + elif case1: + return [s_string, s_None] + elif case2: + return [s_string, s_tuple_of_2_floats] + else: + raise Exception("os.utime() arg 2 must be None or a tuple of " + "2 floats, got %s" % (s_times,)) + + return extdef(os_utime_normalize_args, s_None, + "ll_os.ll_os_utime", + llimpl=os_utime_llimpl) @registering(os.times) def register_os_times(self): From wlav at codespeak.net Thu Jul 15 12:28:21 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Thu, 15 Jul 2010 12:28:21 +0200 (CEST) Subject: [pypy-svn] r76234 - pypy/branch/reflex-support/pypy/module/cppyy Message-ID: <20100715102821.C743E282B90@codespeak.net> Author: wlav Date: Thu Jul 15 12:28:20 2010 New Revision: 76234 Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Log: Improved error reporting when an overload fails. Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Thu Jul 15 12:28:20 2010 @@ -13,6 +13,14 @@ lltype.free(arg, flavor='raw') +class VoidConverter(TypeConverter): + def __init__(self, space, name): + self.name = name + + def convert_argument(self, space, w_obj): + raise OperationError(space.w_TypeError, + space.wrap('no converter available for type "%s"' % self.name)) + class BoolConverter(TypeConverter): def convert_argument(self, space, w_obj): arg = space.c_int_w(w_obj) @@ -74,6 +82,7 @@ # 3) accept const ref as by value # 4) accept ref as pointer # 5) generalized cases (covers basically all user classes) + # 6) void converter, which fails on use try: return _converters[name] @@ -85,7 +94,10 @@ if compound == "*": return InstancePtrConverter(space, cpptype) - raise OperationError(space.w_TypeError, space.wrap("no clue what %s is" % name)) + # return a void converter here, so that the class can be build even + # when some types are unknown; this overload will simply fail on use + return VoidConverter(space, name) + _converters["bool"] = BoolConverter() _converters["int"] = IntConverter() Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Thu Jul 15 12:28:20 2010 @@ -189,6 +189,7 @@ @jit.unroll_safe def call(self, cppthis, args_w): space = self.space + errmsg = 'None of the overloads matched:' for i in range(len(self.functions)): cppyyfunc = self.functions[i] try: @@ -196,10 +197,11 @@ except OperationError, e: if not e.match(space, space.w_TypeError): raise + errmsg += '\n\t'+str(e) except KeyError: pass - # XXX better error reporting - raise OperationError(space.w_TypeError, space.wrap("none of the overloads matched")) + + raise OperationError(space.w_TypeError, space.wrap(errmsg)) def __repr__(self): return "W_CPPOverload(%s, %s)" % (self.func_name, self.functions) From afa at codespeak.net Thu Jul 15 15:06:10 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 15:06:10 +0200 (CEST) Subject: [pypy-svn] r76235 - in pypy/branch/unicode_filename-2/pypy: module/posix rlib rpython/module Message-ID: <20100715130610.86B1A282B90@codespeak.net> Author: afa Date: Thu Jul 15 15:06:08 2010 New Revision: 76235 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py pypy/branch/unicode_filename-2/pypy/rlib/rposix.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: os.utime() needs to be specialized on the second argument Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Thu Jul 15 15:06:08 2010 @@ -40,7 +40,7 @@ return space.unicode_w(w_unicode) @specialize.memo() -def dispatch_filename(func): +def dispatch_filename(func, tag=0): def dispatch(space, w_fname, *args): if space.isinstance_w(w_fname, space.w_unicode): fname = FileEncoder(space, w_fname) @@ -668,7 +668,7 @@ """ if space.is_w(w_tuple, space.w_None): try: - dispatch_filename(rposix.utime)(space, w_path, None) + dispatch_filename(rposix.utime, 1)(space, w_path, None) return except OSError, e: raise wrap_oserror2(space, e, w_path) @@ -679,7 +679,7 @@ raise OperationError(space.w_TypeError, space.wrap(msg)) actime = space.float_w(args_w[0]) modtime = space.float_w(args_w[1]) - dispatch_filename(rposix.utime)(space, w_path, (actime, modtime)) + dispatch_filename(rposix.utime, 2)(space, w_path, (actime, modtime)) except OSError, e: raise wrap_oserror2(space, e, w_path) except OperationError, e: Modified: pypy/branch/unicode_filename-2/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/rposix.py Thu Jul 15 15:06:08 2010 @@ -103,7 +103,7 @@ else: return os.chmod(path.as_bytes(), mode) - at specialize.argtype(0) + at specialize.argtype(0, 1) def utime(path, times): if isinstance(path, str): return os.utime(path, times) Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Thu Jul 15 15:06:08 2010 @@ -30,47 +30,48 @@ from pypy.rlib import rgc from pypy.rlib.objectmodel import keepalive_until_here, specialize -def registering_unicode_version(func, nbargs, argnums, condition=True): +def registering_unicode_version(posixfunc, signature, condition=True): """ - Registers an implementation of func() which directly accepts unicode + Registers an implementation of posixfunc() which directly accepts unicode strings. Replaces the corresponding function in pypy.rlib.rposix. - @nbargs is the arity of the function - @argnums is the list of positions of unicode strings + @signature: A list of types. 'unicode' will trigger encoding when needed, + 'None' will use specialize.argtype. """ if not condition: return registering(None, condition=False) - func_name = func.__name__ + func_name = posixfunc.__name__ @func_renamer(func_name + "_unicode") def unicodefunc(*args): return func(*args) - argnum = argnums[0] - - arglist = ['arg%d' % (i,) for i in range(nbargs)] + arglist = ['arg%d' % (i,) for i in range(len(signature))] transformed_arglist = arglist[:] - for i in argnums: - transformed_arglist[i] = transformed_arglist[i] + '.as_unicode()' + for i, arg in enumerate(signature): + if arg is unicode: + transformed_arglist[i] = transformed_arglist[i] + '.as_unicode()' args = ', '.join(arglist) transformed_args = ', '.join(transformed_arglist) - main_arg = 'arg%d' % (argnum,) + main_arg = 'arg%d' % (signature.index(unicode),) source = py.code.Source(""" def %(func_name)s(%(args)s): if isinstance(%(main_arg)s, str): - return func(%(args)s) + return posixfunc(%(args)s) else: return unicodefunc(%(transformed_args)s) """ % locals()) - miniglobals = {'func' : func, + miniglobals = {'posixfunc' : posixfunc, 'unicodefunc': unicodefunc, '__name__': __name__, # for module name propagation } exec source.compile() in miniglobals new_func = miniglobals[func_name] - new_func = specialize.argtype(*argnums)(new_func) + specialized_args = [i for i in range(len(signature)) + if signature[i] in (unicode, None)] + new_func = specialize.argtype(*specialized_args)(new_func) # Monkeypatch the function in pypy.rlib.rposix setattr(rposix, func_name, new_func) @@ -500,7 +501,7 @@ llimpl=os_utime_llimpl) # XXX LOT OF DUPLICATED CODE! - @registering_unicode_version(os.utime, 2, [0], sys.platform=='win32') + @registering_unicode_version(os.utime, [unicode, None], sys.platform=='win32') def register_os_utime_unicode(self): from pypy.rlib import rwin32 from pypy.rpython.module.ll_os_stat import time_t_to_FILE_TIME @@ -860,7 +861,7 @@ return extdef([str, int, int], int, "ll_os.ll_os_open", llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) - @registering_unicode_version(os.open, 3, [0], sys.platform=='win32') + @registering_unicode_version(os.open, [unicode, int, int], sys.platform=='win32') def register_os_open_unicode(self): os_wopen = self.llexternal(underscore_on_windows+'wopen', [rffi.CWCHARP, rffi.INT, rffi.MODE_T], @@ -1055,7 +1056,7 @@ export_name="ll_os.ll_os_access", oofakeimpl=os_access_oofakeimpl) - @registering_unicode_version(os.access, 2, [0], sys.platform=='win32') + @registering_unicode_version(os.access, [unicode, int], sys.platform=='win32') def register_os_access_unicode(self): os_waccess = self.llexternal(underscore_on_windows + 'waccess', [rffi.CWCHARP, rffi.INT], @@ -1105,7 +1106,7 @@ llimpl=_getfullpathname_llimpl) @registering_unicode_version(getattr(posix, '_getfullpathname', None), - 1, [0], sys.platform=='win32') + [unicode], sys.platform=='win32') def register_posix__getfullpathname_unicode(self): from pypy.rlib import rwin32 # this nt function is not exposed via os, but needed @@ -1308,7 +1309,7 @@ "ll_os.ll_os_listdir", llimpl=os_listdir_llimpl) - @registering_unicode_version(os.listdir, 1, [0], sys.platform=='win32') + @registering_unicode_version(os.listdir, [unicode], sys.platform=='win32') def register_os_listdir_unicode(self): from pypy.rlib import rwin32 class CConfig: @@ -1559,7 +1560,7 @@ return extdef([str], s_None, llimpl=unlink_llimpl, export_name="ll_os.ll_os_unlink") - @registering_unicode_version(os.unlink, 1, [0], sys.platform=='win32') + @registering_unicode_version(os.unlink, [unicode], sys.platform=='win32') def register_os_unlink_unicode(self): os_wunlink = self.llexternal(underscore_on_windows+'wunlink', [rffi.CWCHARP], rffi.INT) @@ -1583,7 +1584,7 @@ return extdef([str], s_None, llimpl=chdir_llimpl, export_name="ll_os.ll_os_chdir") - @registering_unicode_version(os.chdir, 1, [0], sys.platform=='win32') + @registering_unicode_version(os.chdir, [unicode], sys.platform=='win32') def register_os_chdir_unicode(self): os_wchdir = self.llexternal(underscore_on_windows+'wchdir', [rffi.CWCHARP], rffi.INT) @@ -1617,7 +1618,7 @@ return extdef([str, int], s_None, llimpl=mkdir_llimpl, export_name="ll_os.ll_os_mkdir") - @registering_unicode_version(os.mkdir, 2, [0], sys.platform=='win32') + @registering_unicode_version(os.mkdir, [unicode, int], sys.platform=='win32') def register_os_mkdir_unicode(self): if os.name == 'nt': ARG2 = [] # no 'mode' argument on Windows - just ignored @@ -1651,7 +1652,7 @@ return extdef([str], s_None, llimpl=rmdir_llimpl, export_name="ll_os.ll_os_rmdir") - @registering_unicode_version(os.rmdir, 1, [0], sys.platform=='win32') + @registering_unicode_version(os.rmdir, [unicode], sys.platform=='win32') def register_os_rmdir_unicode(self): os_wrmdir = self.llexternal(underscore_on_windows+'wrmdir', [rffi.CWCHARP], rffi.INT) @@ -1676,7 +1677,7 @@ return extdef([str, int], s_None, llimpl=chmod_llimpl, export_name="ll_os.ll_os_chmod") - @registering_unicode_version(os.chmod, 2, [0], sys.platform=='win32') + @registering_unicode_version(os.chmod, [unicode, int], sys.platform=='win32') def register_os_chmod_unicode(self): os_wchmod = self.llexternal(underscore_on_windows+'wchmod', [rffi.CWCHARP, rffi.MODE_T], rffi.INT) @@ -1702,7 +1703,7 @@ return extdef([str, str], s_None, llimpl=rename_llimpl, export_name="ll_os.ll_os_rename") - @registering_unicode_version(os.rename, 2, [0, 1], sys.platform=='win32') + @registering_unicode_version(os.rename, [unicode, unicode], sys.platform=='win32') def register_os_rename_unicode(self): os_wrename = self.llexternal(underscore_on_windows+'wrename', [rffi.CWCHARP, rffi.CWCHARP], rffi.INT) @@ -1834,12 +1835,12 @@ from pypy.rpython.module import ll_os_stat return ll_os_stat.register_stat_variant('lstat') - @registering_unicode_version(os.stat, 1, [0], sys.platform=='win32') + @registering_unicode_version(os.stat, [unicode], sys.platform=='win32') def register_os_stat_unicode(self): from pypy.rpython.module import ll_os_stat return ll_os_stat.register_stat_variant_unicode('stat') - @registering_unicode_version(os.lstat, 1, [0], sys.platform=='win32') + @registering_unicode_version(os.lstat, [unicode], sys.platform=='win32') def register_os_lstat_unicode(self): from pypy.rpython.module import ll_os_stat return ll_os_stat.register_stat_variant_unicode('lstat') From antocuni at codespeak.net Thu Jul 15 15:49:48 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 15 Jul 2010 15:49:48 +0200 (CEST) Subject: [pypy-svn] r76236 - in pypy/extradoc/talk: ecoop2009-tutorial/tls ep2010/pypy_demo ep2010/talk icooolps2009 Message-ID: <20100715134948.4364C282B90@codespeak.net> Author: antocuni Date: Thu Jul 15 15:49:46 2010 New Revision: 76236 Added: pypy/extradoc/talk/ecoop2009-tutorial/tls/ pypy/extradoc/talk/ep2010/pypy_demo/ - copied from r76235, pypy/extradoc/talk/pycon-italy-2010/pypy_demo/ pypy/extradoc/talk/ep2010/talk/ - copied from r76235, pypy/extradoc/talk/pycon-italy-2010/talk/ pypy/extradoc/talk/ep2010/talk/talk.rst - copied, changed from r76235, pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Removed: pypy/extradoc/talk/ep2010/talk/talk.txt Modified: pypy/extradoc/talk/ep2010/talk/Makefile pypy/extradoc/talk/ep2010/talk/author.latex pypy/extradoc/talk/ep2010/talk/stylesheet.latex pypy/extradoc/talk/icooolps2009/paper.bib Log: initial commit for ep2010 talk. Mostly copied from the pycon italy one Modified: pypy/extradoc/talk/ep2010/talk/Makefile ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/Makefile (original) +++ pypy/extradoc/talk/ep2010/talk/Makefile Thu Jul 15 15:49:46 2010 @@ -4,8 +4,8 @@ # WARNING: to work, it needs this patch for docutils # https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414 -talk.pdf: talk.txt author.latex title.latex stylesheet.latex - rst2beamer.py --stylesheet=stylesheet.latex --documentoptions=14pt talk.txt talk.latex || exit +talk.pdf: talk.rst author.latex title.latex stylesheet.latex + rst2beamer.py --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit pdflatex talk.latex || exit Modified: pypy/extradoc/talk/ep2010/talk/author.latex ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/author.latex (original) +++ pypy/extradoc/talk/ep2010/talk/author.latex Thu Jul 15 15:49:46 2010 @@ -1,8 +1,8 @@ \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} -\title[PyPy 1.2]{PyPy 1.2: snakes never crawled so fast} -\author[antocuni, arigato] -{Antonio Cuni \\ Armin Rigo} +\title[PyPy 1.3]{PyPy \sout{1.2} 1.3: Status and News} +\author[amaury, antocuni, arigato] +{Amaury Forgeot d'Arc \\ Antonio Cuni \\ Armin Rigo} -\institute{Pycon Italia Qu4ttro} -\date{May 8 2010} +\institute{EuroPython 2010} +\date{July 19 2010} Modified: pypy/extradoc/talk/ep2010/talk/stylesheet.latex ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/stylesheet.latex (original) +++ pypy/extradoc/talk/ep2010/talk/stylesheet.latex Thu Jul 15 15:49:46 2010 @@ -1,3 +1,4 @@ +\usepackage{ulem} \usetheme{Boadilla} \setbeamercovered{transparent} \setbeamertemplate{navigation symbols}{} Copied: pypy/extradoc/talk/ep2010/talk/talk.rst (from r76235, pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt) ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt (original) +++ pypy/extradoc/talk/ep2010/talk/talk.rst Thu Jul 15 15:49:46 2010 @@ -1,14 +1,14 @@ .. include:: beamerdefs.txt ======================================= -PyPy 1.2: snakes never crawled so fast +PyPy 1.3: status and news ======================================= Outline ------- -- PyPy 1.2: what's new and status update +- PyPy 1.3: what's new and status update - Overview of the JIT @@ -40,10 +40,10 @@ - What's new and status update -What's new in PyPy 1.2 +What's new in PyPy 1.3 ----------------------- -- Released on March 12th, 2010 +- Released on March 12th, 2010 XXX - Main theme: speed Modified: pypy/extradoc/talk/icooolps2009/paper.bib ============================================================================== --- pypy/extradoc/talk/icooolps2009/paper.bib (original) +++ pypy/extradoc/talk/icooolps2009/paper.bib Thu Jul 15 15:49:46 2010 @@ -152,16 +152,6 @@ year = {2009}, }, - at article{bala_dynamo:transparent_2000, - title = {Dynamo: a Transparent Dynamic Optimization System}, - volume = {35}, - url = {http://citeseer.ist.psu.edu/bala00dynamo.html}, - number = {5}, - journal = {{ACM} {SIGPLAN} Notices}, - author = {Vasanth Bala and Evelyn Duesterwald and Sanjeev Banerjia}, - year = {2000}, - pages = {1--12} -}, @techreport{andreas_gal_incremental_2006, title = {Incremental Dynamic Code Generation with Trace Trees}, @@ -348,4 +338,3 @@ year = {2009}, note = {to appear} } - From antocuni at codespeak.net Thu Jul 15 16:07:51 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 15 Jul 2010 16:07:51 +0200 (CEST) Subject: [pypy-svn] r76237 - pypy/extradoc/talk/ep2010/talk Message-ID: <20100715140751.98CEE282B90@codespeak.net> Author: antocuni Date: Thu Jul 15 16:07:50 2010 New Revision: 76237 Modified: pypy/extradoc/talk/ep2010/talk/talk.rst Log: update to pypy 1.3. Restructure the talk, bring the old part 3 (the live demo) inside part 1. We need to decided whether we want to keep the demo, though. Modified: pypy/extradoc/talk/ep2010/talk/talk.rst ============================================================================== --- pypy/extradoc/talk/ep2010/talk/talk.rst (original) +++ pypy/extradoc/talk/ep2010/talk/talk.rst Thu Jul 15 16:07:50 2010 @@ -12,7 +12,8 @@ - Overview of the JIT -- Demo: how to use PyPy right now +- ``cpyext``: load CPython extensions in PyPy! + Part 0: What is PyPy? :-) @@ -40,16 +41,26 @@ - What's new and status update -What's new in PyPy 1.3 ------------------------ +What's new in PyPy 1.2, 1.3 +--------------------------- + +- 1.2: released on March 12th, 2010 + + * Main theme: speed + + * JIT compiler + + * speed.pypy.org + +- 1.3: released on June 26th, 2010 -- Released on March 12th, 2010 XXX + * Stability: lot of bugfixes, thanks for the feedback :-) -- Main theme: speed + * More speed! -- JIT compiler + * cpyext -- speed.pypy.org +- Binaries for Linux, Windows, Mac - Ubuntu packages @@ -67,20 +78,6 @@ .. image:: pypy-vs-psyco.png :scale: 35 -Speed: Demo ------------ - -- Django application - -- Mandelbrot fractal - - * fished randomly on the net :-) - -- Run both on CPython and PyPy - - * django trunk! - - What works on PyPy ------------------- @@ -114,7 +111,7 @@ |end_scriptsize| - * **ctypes** + * ctypes What does not work on PyPy @@ -140,61 +137,135 @@ |pause| -- Extension modules |pause| +- Extension modules - * really? + * try cpyext! - * drum roll... -cpyext ------- +Speed: Demo +----------- -- CPython extension modules in PyPy +- Django application -- ``pypy-c setup.py build`` +- Mandelbrot fractal -- still beta + * fished randomly on the net :-) -- not 100% of CPython API is supported +- Run both on CPython and PyPy -- not included in PyPy 1.2 + * django trunk! -- Known to work: - * wxPython (after a patch) +Mandelbrot demo +--------------- - * _sre +- Works purely on PyPy - * PyCrypto +- Not always the case - * PIL + * missing extension modules (cpyext mitigates the problem) + * libraries that rely on CPython details + + * ... -wxPython on PyPy (1) ---------------------- +- clear performance-critical part -.. image:: wxpython1.png - :scale: 30 -wxPython on PyPy (2) ---------------------- +CPython and PyPy side by side +------------------------------ +- CPython: runs the main application -.. image:: wxpython2.png - :scale: 30 +- PyPy: subprocess, runs only the hotspots + +- How do they communicate? + +- execnet + * **The Ring of Python**, Holger Krekel, 9:45 -PyPy 1.2.1 ----------- + * oups, too late :-) + + +Rendering (1) +--------------- + +|example<| Mandelbrot |>| +|small| +:: + + def render(request): + w = int(request.GET.get('w', 320)) + h = int(request.GET.get('h', 240)) + + from py_mandel import mandelbrot + img = mandelbrot(w, h) + + return HttpResponse(img, content_type="image/bmp") + +|end_small| +|end_example| + + +Rendering (2) +------------- + +|example<| Mandelbrot on PyPy |>| +|small| +:: + + def pypy_render(request): + w = int(request.GET.get('w', 320)) + h = int(request.GET.get('h', 240)) + + channel = pypy.remote_exec(""" + from py_mandel import mandelbrot + w, h = channel.receive() + img = mandelbrot(w, h) + channel.send(img) + """) + channel.send((w, h)) + img = channel.receive() + + return HttpResponse(img, content_type="image/bmp") + +|end_small| +|end_example| + +execnet setup +------------- + +|example<| At startup |>| +|small| +:: + + import execnet + mygroup = execnet.Group() + pypy = mygroup.makegateway("popen//python=pypy-c") + pypy.remote_exec(""" + import sys + import os + os.chdir("mandelbrot") + sys.path.insert(0, '') + """) + +|end_small| +|end_example| -- Coming soon -- Many bugfixes +Demo +---- + + +Benchmarks +---------- + +.. image:: demo-graph.pdf + :scale: 45 - * 27 issues reported after release of 1.2 -- beta version of cpyext Part 2: Just-in-Time compilation @@ -389,116 +460,47 @@ Part 3 ------ -How to use PyPy right now - - -Mandelbrot demo ---------------- - -- Works purely on PyPy - -- Not always the case - - * missing extension modules (cpyext mitigates the problem) - - * libraries that rely on CPython details - - * ... - -- clear performance-critical part - - -CPython and PyPy side by side ------------------------------- - -- CPython: runs the main application - -- PyPy: subprocess, runs only the hotspots - -- How do they communicate? - -- execnet - - * **The Ring of Python**, Holger Krekel, 9:00 - - * oups, too late :-) - - -Rendering (1) ---------------- - -|example<| Mandelbrot |>| -|small| -:: +cpyext - def render(request): - w = int(request.GET.get('w', 320)) - h = int(request.GET.get('h', 240)) +cpyext +------ - from py_mandel import mandelbrot - img = mandelbrot(w, h) +- CPython extension modules in PyPy - return HttpResponse(img, content_type="image/bmp") +- ``pypy-c setup.py build`` -|end_small| -|end_example| +- still beta +- not 100% of CPython API is supported -Rendering (2) -------------- +- not included in PyPy 1.2 -|example<| Mandelbrot on PyPy |>| -|small| -:: +- Known to work: - def pypy_render(request): - w = int(request.GET.get('w', 320)) - h = int(request.GET.get('h', 240)) + * wxPython (after a patch) - channel = pypy.remote_exec(""" - from py_mandel import mandelbrot - w, h = channel.receive() - img = mandelbrot(w, h) - channel.send(img) - """) - channel.send((w, h)) - img = channel.receive() + * _sre - return HttpResponse(img, content_type="image/bmp") + * PyCrypto -|end_small| -|end_example| + * PIL -execnet setup -------------- -|example<| At startup |>| -|small| -:: +wxPython on PyPy (1) +--------------------- - import execnet - mygroup = execnet.Group() - pypy = mygroup.makegateway("popen//python=pypy-c") - pypy.remote_exec(""" - import sys - import os - os.chdir("mandelbrot") - sys.path.insert(0, '') - """) +.. image:: wxpython1.png + :scale: 30 -|end_small| -|end_example| +wxPython on PyPy (2) +--------------------- -Demo ----- +.. image:: wxpython2.png + :scale: 30 -Benchmarks ----------- -.. image:: demo-graph.pdf - :scale: 45 Contact / Q&A From antocuni at codespeak.net Thu Jul 15 16:47:31 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 15 Jul 2010 16:47:31 +0200 (CEST) Subject: [pypy-svn] r76239 - pypy/extradoc/talk/ep2010/talk Message-ID: <20100715144731.81BCC282BDB@codespeak.net> Author: antocuni Date: Thu Jul 15 16:47:30 2010 New Revision: 76239 Modified: pypy/extradoc/talk/ep2010/talk/talk.rst Log: add syntax highlighting. Needs the newest version of rst2beamer in my user/antocuni/bin directory, thanks to cfbolz Modified: pypy/extradoc/talk/ep2010/talk/talk.rst ============================================================================== --- pypy/extradoc/talk/ep2010/talk/talk.rst (original) +++ pypy/extradoc/talk/ep2010/talk/talk.rst Thu Jul 15 16:47:30 2010 @@ -194,7 +194,8 @@ |example<| Mandelbrot |>| |small| -:: + +.. sourcecode:: python def render(request): w = int(request.GET.get('w', 320)) @@ -214,7 +215,8 @@ |example<| Mandelbrot on PyPy |>| |small| -:: + +.. sourcecode:: python def pypy_render(request): w = int(request.GET.get('w', 320)) @@ -239,7 +241,8 @@ |example<| At startup |>| |small| -:: + +.. sourcecode:: python import execnet mygroup = execnet.Group() @@ -400,7 +403,7 @@ |example<| |>| -:: +.. sourcecode:: python def g(a, b): if a < 5: # 2 @@ -408,7 +411,7 @@ return a - b # 3 def f(x): - total = 0 + total = 0 # 1 for i in range(x): d = g(i, x) @@ -416,13 +419,11 @@ |end_example| -|pause| - |column2| |example<| |>| -:: +.. sourcecode:: nasm ADD EAX, 1 CMP EAX, EBX From hakanardo at codespeak.net Thu Jul 15 19:09:41 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 15 Jul 2010 19:09:41 +0200 (CEST) Subject: [pypy-svn] r76241 - in pypy/branch/interplevel-array/pypy: jit/metainterp module/array Message-ID: <20100715170941.69E2B282BDB@codespeak.net> Author: hakanardo Date: Thu Jul 15 19:09:39 2010 New Revision: 76241 Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: cleanups Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py Thu Jul 15 19:09:39 2010 @@ -611,8 +611,8 @@ assert oldop.opnum == op.opnum self.make_equal_to(op.result, self.getvalue(oldop.result)) return - elif self.find_rewriteable_constant(op, args): - return + #elif self.find_rewriteable_constant(op, args): + # return else: self.pure_operations[args] = op Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Thu Jul 15 19:09:39 2010 @@ -176,11 +176,8 @@ space = self.space start, stop, step = space.decode_index(w_idx, self.len) if step < 0: - w_lst = space.call_function( - space.getattr(self, space.wrap('tolist'))) - w_lst = space.call_function( - space.getattr(w_lst, space.wrap('__getitem__')), - w_idx) + w_lst = self.descr_tolist() + w_lst = space.getitem(w_lst, w_idx) w_a=mytype.w_class(self.space) w_a.descr_fromsequence(w_lst) else: @@ -271,13 +268,9 @@ size = (stop - start) / step if (stop - start) % step > 0: size += 1 if w_item.len != size or step < 0: - w_lst = space.call_function( - space.getattr(self, space.wrap('tolist'))) - w_item = space.call_function( - space.getattr(w_item, space.wrap('tolist'))) - space.call_function( - space.getattr(w_lst, space.wrap('__setitem__')), - w_idx, w_item) + w_lst = self.descr_tolist() + w_item = space.call_method(w_item, 'tolist') + space.setitem(w_lst, w_idx, w_item) self.setlen(0) self.descr_fromsequence(w_lst) else: @@ -440,11 +433,8 @@ def descr_delitem(self, w_idx): space=self.space - w_lst = space.call_function( - space.getattr(self, space.wrap('tolist'))) - space.call_function( - space.getattr(w_lst, space.wrap('__delitem__')), - w_idx) + w_lst = self.descr_tolist() + space.delitem(w_lst, w_idx) self.setlen(0) self.descr_fromsequence(w_lst) descr_delitem.unwrap_spec = ['self', W_Root] @@ -570,14 +560,12 @@ def descr_pop(self, w_i=-1): space = self.space - return space.call_function(space.getattr(self._array, space.wrap('pop')), w_i) + return space.call_method(self._array, 'pop', w_i) descr_pop.unwrap_spec = ['self', W_Root] def descr_getitem(self, w_i): space = self.space - #w_item = self._array.descr_getitem(w_i) - w_item = space.call_function( - space.getattr(self._array, space.wrap('__getitem__')), w_i) + w_item = space.getitem(self._array, w_i) if isinstance(w_item, W_ArrayBase): return W_WrappedArray(space, w_item) return w_item @@ -585,25 +573,21 @@ def descr_iadd(self, w_i): space = self.space - #self._array.descr_iadd(w_i._array) w_i = space.interp_w(W_WrappedArray, w_i) - w_item = space.call_function( - space.getattr(self._array, space.wrap('__iadd__')), w_i._array) + w_item = space.call_method(self._array, '__iadd__', w_i._array) return self descr_iadd.unwrap_spec = ['self', W_Root] def descr_imul(self, w_i): - space = self.space - #self._array.descr_imul(i) - w_item = space.call_function( - space.getattr(self._array, space.wrap('__imul__')), w_i) + space = self.space + w_item = space.call_method(self._array, '__imul__', w_i) return self descr_imul.unwrap_spec = ['self', W_Root] def descr_reduce(self): space=self.space if space.int_w(space.len(self._array)) > 0: - w_s = space.call_function(space.getattr(self._array, space.wrap('tostring'))) + w_s = space.call_method(self._array, 'tostring') args = [space.wrap(self._array.typecode), w_s] else: args = [space.wrap(self._array.typecode)] @@ -618,31 +602,27 @@ def make_descr(fn, nargs, wrp): if nargs == 0: def descr(space, self): - ret = space.call_function( - space.getattr(self._array, space.wrap(fn))) + ret = space.call_method(self._array, fn) if (wrp): return W_WrappedArray(space, ret) return ret elif nargs == 1: def descr(space, self, w_a): - ret = space.call_function( - space.getattr(self._array, space.wrap(fn)), + ret = space.call_method(self._array, fn, unwrap_array(w_a)) if (wrp): return W_WrappedArray(space, ret) return ret elif nargs == 2: def descr(space, self, w_a, w_b): - ret = space.call_function( - space.getattr(self._array, space.wrap(fn)), + ret = space.call_method(self._array, fn, unwrap_array(w_a), unwrap_array(w_b)) if (wrp): return W_WrappedArray(space, ret) return ret elif nargs == 3: def descr(space, self, w_a, w_b, w_c): - ret = space.call_function( - space.getattr(self._array, space.wrap(fn)), + ret = space.call_method(self._array, fn, unwrap_array(w_a), unwrap_array(w_b), unwrap_array(w_c)) if (wrp): return W_WrappedArray(space, ret) return ret From hakanardo at codespeak.net Thu Jul 15 19:18:59 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 15 Jul 2010 19:18:59 +0200 (CEST) Subject: [pypy-svn] r76242 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100715171859.E9C41282BDB@codespeak.net> Author: hakanardo Date: Thu Jul 15 19:18:58 2010 New Revision: 76242 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: missed one Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Thu Jul 15 19:18:58 2010 @@ -161,7 +161,7 @@ self.allocated = 0 new_buffer = lltype.nullptr(mytype.arraytype) - if self.buffer != lltype.nullptr(mytype.arraytype): + if self.buffer: lltype.free(self.buffer, flavor='raw') self.buffer = new_buffer self.len = size From afa at codespeak.net Thu Jul 15 22:14:05 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 22:14:05 +0200 (CEST) Subject: [pypy-svn] r76243 - pypy/branch/unicode_filename-2/pypy/rlib/test Message-ID: <20100715201405.08976282B90@codespeak.net> Author: afa Date: Thu Jul 15 22:14:03 2010 New Revision: 76243 Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Log: Fix test on Linux Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Thu Jul 15 22:14:03 2010 @@ -90,11 +90,18 @@ def test_listdir(self): udir = UnicodeWithEncoding(os.path.dirname(self.ufilename)) - def f(): - return u', '.join(rposix.listdir(udir)) - result = interpret(f, []) - assert os.path.basename(self.ufilename) in ll_to_string(result) + if sys.platform == 'win32': + def f(): + return u', '.join(rposix.listdir(udir)) + result = interpret(f, []) + assert os.path.basename(self.ufilename) in ll_to_string(result) + else: + def f(): + return ', '.join(rposix.listdir(udir)) + result = interpret(f, []) + assert (os.path.basename(self.ufilename).encode('utf-8') in + ll_to_string(result)) def test_chdir(self): os.unlink(self.ufilename) From afa at codespeak.net Thu Jul 15 22:14:39 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 22:14:39 +0200 (CEST) Subject: [pypy-svn] r76244 - pypy/branch/unicode_filename-2/pypy/rlib/test Message-ID: <20100715201439.47BCB282B90@codespeak.net> Author: afa Date: Thu Jul 15 22:14:37 2010 New Revision: 76244 Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (contents, props changed) Log: fixeol Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Thu Jul 15 22:14:37 2010 @@ -1,129 +1,129 @@ -from pypy.rpython.test.test_llinterp import interpret -from pypy.tool.udir import udir -from pypy.rlib import rposix -import os, sys - -def ll_to_string(s): - return ''.join(s.chars) - -class UnicodeWithEncoding: - def __init__(self, unistr): - self.unistr = unistr - - if sys.platform == 'win32': - def as_bytes(self): - from pypy.rlib.runicode import unicode_encode_mbcs - return unicode_encode_mbcs(self.unistr, len(self.unistr), - "strict") - else: - def as_bytes(self): - from pypy.rlib.runicode import unicode_encode_utf_8 - return unicode_encode_utf_8(self.unistr, len(self.unistr), - "strict") - - def as_unicode(self): - return self.unistr - -class TestPosixUnicode: - def setup_method(self, method): - self.ufilename = (unicode(udir.join('test_open')) + - u'\u65e5\u672c.txt') # "Japan" - f = file(self.ufilename, 'w') - f.write("test") - f.close() - - self.path = UnicodeWithEncoding(self.ufilename) - self.path2 = UnicodeWithEncoding(self.ufilename + ".new") - - def test_open(self): - def f(): - try: - fd = rposix.open(self.path, os.O_RDONLY, 0777) - try: - text = os.read(fd, 50) - return text - finally: - os.close(fd) - except OSError: - return '' - - assert ll_to_string(interpret(f, [])) == "test" - - def test_stat(self): - def f(): - return rposix.stat(self.path).st_mtime - - assert interpret(f, []) == os.stat(self.ufilename).st_mtime - - def test_access(self): - def f(): - return rposix.access(self.path, os.R_OK) - - assert interpret(f, []) == 1 - - def test_utime(self): - def f(): - return rposix.utime(self.path, None) - - interpret(f, []) # does not crash - - def test_chmod(self): - def f(): - return rposix.chmod(self.path, 0777) - - interpret(f, []) # does not crash - - def test_unlink(self): - def f(): - return rposix.unlink(self.path) - - interpret(f, []) - assert not os.path.exists(self.ufilename) - - def test_rename(self): - def f(): - return rposix.rename(self.path, self.path2) - - interpret(f, []) - assert not os.path.exists(self.ufilename) - assert os.path.exists(self.ufilename + '.new') - - def test_listdir(self): - udir = UnicodeWithEncoding(os.path.dirname(self.ufilename)) - - if sys.platform == 'win32': - def f(): - return u', '.join(rposix.listdir(udir)) - result = interpret(f, []) - assert os.path.basename(self.ufilename) in ll_to_string(result) - else: - def f(): - return ', '.join(rposix.listdir(udir)) - result = interpret(f, []) - assert (os.path.basename(self.ufilename).encode('utf-8') in - ll_to_string(result)) - - def test_chdir(self): - os.unlink(self.ufilename) - - def f(): - rposix.mkdir(self.path, 0777) - rposix.chdir(self.path) - - curdir = os.getcwd() - try: - interpret(f, []) - assert os.getcwdu() == self.ufilename - finally: - os.chdir(curdir) - - def g(): - rposix.rmdir(self.path) - - try: - interpret(g, []) - finally: - try: - os.rmdir(self.ufilename) - except Exception: - pass +from pypy.rpython.test.test_llinterp import interpret +from pypy.tool.udir import udir +from pypy.rlib import rposix +import os, sys + +def ll_to_string(s): + return ''.join(s.chars) + +class UnicodeWithEncoding: + def __init__(self, unistr): + self.unistr = unistr + + if sys.platform == 'win32': + def as_bytes(self): + from pypy.rlib.runicode import unicode_encode_mbcs + return unicode_encode_mbcs(self.unistr, len(self.unistr), + "strict") + else: + def as_bytes(self): + from pypy.rlib.runicode import unicode_encode_utf_8 + return unicode_encode_utf_8(self.unistr, len(self.unistr), + "strict") + + def as_unicode(self): + return self.unistr + +class TestPosixUnicode: + def setup_method(self, method): + self.ufilename = (unicode(udir.join('test_open')) + + u'\u65e5\u672c.txt') # "Japan" + f = file(self.ufilename, 'w') + f.write("test") + f.close() + + self.path = UnicodeWithEncoding(self.ufilename) + self.path2 = UnicodeWithEncoding(self.ufilename + ".new") + + def test_open(self): + def f(): + try: + fd = rposix.open(self.path, os.O_RDONLY, 0777) + try: + text = os.read(fd, 50) + return text + finally: + os.close(fd) + except OSError: + return '' + + assert ll_to_string(interpret(f, [])) == "test" + + def test_stat(self): + def f(): + return rposix.stat(self.path).st_mtime + + assert interpret(f, []) == os.stat(self.ufilename).st_mtime + + def test_access(self): + def f(): + return rposix.access(self.path, os.R_OK) + + assert interpret(f, []) == 1 + + def test_utime(self): + def f(): + return rposix.utime(self.path, None) + + interpret(f, []) # does not crash + + def test_chmod(self): + def f(): + return rposix.chmod(self.path, 0777) + + interpret(f, []) # does not crash + + def test_unlink(self): + def f(): + return rposix.unlink(self.path) + + interpret(f, []) + assert not os.path.exists(self.ufilename) + + def test_rename(self): + def f(): + return rposix.rename(self.path, self.path2) + + interpret(f, []) + assert not os.path.exists(self.ufilename) + assert os.path.exists(self.ufilename + '.new') + + def test_listdir(self): + udir = UnicodeWithEncoding(os.path.dirname(self.ufilename)) + + if sys.platform == 'win32': + def f(): + return u', '.join(rposix.listdir(udir)) + result = interpret(f, []) + assert os.path.basename(self.ufilename) in ll_to_string(result) + else: + def f(): + return ', '.join(rposix.listdir(udir)) + result = interpret(f, []) + assert (os.path.basename(self.ufilename).encode('utf-8') in + ll_to_string(result)) + + def test_chdir(self): + os.unlink(self.ufilename) + + def f(): + rposix.mkdir(self.path, 0777) + rposix.chdir(self.path) + + curdir = os.getcwd() + try: + interpret(f, []) + assert os.getcwdu() == self.ufilename + finally: + os.chdir(curdir) + + def g(): + rposix.rmdir(self.path) + + try: + interpret(g, []) + finally: + try: + os.rmdir(self.ufilename) + except Exception: + pass From afa at codespeak.net Thu Jul 15 23:14:29 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 23:14:29 +0200 (CEST) Subject: [pypy-svn] r76245 - pypy/branch/unicode_filename-2/pypy/rpython/module Message-ID: <20100715211429.A69E5282B90@codespeak.net> Author: afa Date: Thu Jul 15 23:14:28 2010 New Revision: 76245 Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Start refactoring: share implementations of the str and unicode versions. Let's see if this translates correctly. Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Thu Jul 15 23:14:28 2010 @@ -10,7 +10,8 @@ from pypy.rpython.module.support import ll_strcpy, OOSupport from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rlib.rarithmetic import r_longlong -from pypy.rpython.extfunc import BaseLazyRegistering +from pypy.rpython.extfunc import ( + BaseLazyRegistering, lazy_register, register_external) from pypy.rpython.extfunc import registering, registering_if, extdef from pypy.annotation.model import ( SomeInteger, SomeString, SomeTuple, SomeFloat, SomeUnicodeString) @@ -30,22 +31,9 @@ from pypy.rlib import rgc from pypy.rlib.objectmodel import keepalive_until_here, specialize -def registering_unicode_version(posixfunc, signature, condition=True): - """ - Registers an implementation of posixfunc() which directly accepts unicode - strings. Replaces the corresponding function in pypy.rlib.rposix. - @signature: A list of types. 'unicode' will trigger encoding when needed, - 'None' will use specialize.argtype. - """ - if not condition: - return registering(None, condition=False) - +def monkeypatch_rlib(posixfunc, unicodefunc, signature): func_name = posixfunc.__name__ - @func_renamer(func_name + "_unicode") - def unicodefunc(*args): - return func(*args) - arglist = ['arg%d' % (i,) for i in range(len(signature))] transformed_arglist = arglist[:] for i, arg in enumerate(signature): @@ -76,8 +64,71 @@ # Monkeypatch the function in pypy.rlib.rposix setattr(rposix, func_name, new_func) +def registering_unicode_version(posixfunc, signature, condition=True): + """ + Registers an implementation of posixfunc() which directly accepts unicode + strings. Replaces the corresponding function in pypy.rlib.rposix. + @signature: A list of types. 'unicode' will trigger encoding when needed, + 'None' will use specialize.argtype. + """ + if not condition: + return registering(None, condition=False) + + func_name = posixfunc.__name__ + + @func_renamer(func_name + "_unicode") + def unicodefunc(*args): + return posixfunc(*args) + + monkeypatch_rlib(posixfunc, unicodefunc, signature) + return registering(unicodefunc, condition=condition) +class StringTypes: + str = str + CCHARP = rffi.CCHARP + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_' + name + +class UnicodeTypes: + str = unicode + CCHARP = rffi.CWCHARP + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + 'w' + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_w' + name + +def registering_str_unicode(posixfunc, signature): + func_name = posixfunc.__name__ + + def register_posixfunc(self, method): + val = method(self, StringTypes()) + register_external(posixfunc, *val.def_args, **val.def_kwds) + + if sys.platform == 'win32': + val = method(self, UnicodeTypes()) + @func_renamer(func_name + "_unicode") + def unicodefunc(*args): + return posixfunc(*args) + register_external(unicodefunc, *val.def_args, **val.def_kwds) + monkeypatch_rlib(posixfunc, unicodefunc, signature) + + def decorator(method): + decorated = lambda self: register_posixfunc(self, method) + decorated._registering_func = posixfunc + return decorated + return decorator + posix = __import__(os.name) if sys.platform.startswith('win'): @@ -843,12 +894,11 @@ def register_os_setsid(self): return self.extdef_for_os_function_returning_int('setsid') - @registering(os.open) - def register_os_open(self): - os_open = self.llexternal(underscore_on_windows+'open', - [rffi.CCHARP, rffi.INT, rffi.MODE_T], + @registering_str_unicode(os.open, [unicode, int, int]) + def register_os_open(self, ttypes): + os_open = self.llexternal(ttypes.posix_function_name('open'), + [ttypes.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT) - def os_open_llimpl(path, flags, mode): result = rffi.cast(rffi.LONG, os_open(path, flags, mode)) if result == -1: @@ -856,25 +906,11 @@ return result def os_open_oofakeimpl(o_path, flags, mode): - return os.open(o_path._str, flags, mode) + return os.open(OOSupport.from_rstr(path), flags, mode) - return extdef([str, int, int], int, "ll_os.ll_os_open", + return extdef([ttypes.str, int, int], int, ttypes.ll_os_name('open'), llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) - @registering_unicode_version(os.open, [unicode, int, int], sys.platform=='win32') - def register_os_open_unicode(self): - os_wopen = self.llexternal(underscore_on_windows+'wopen', - [rffi.CWCHARP, rffi.INT, rffi.MODE_T], - rffi.INT) - def os_wopen_llimpl(path, flags, mode): - result = rffi.cast(rffi.LONG, os_wopen(path, flags, mode)) - if result == -1: - raise OSError(rposix.get_errno(), "os_open failed") - return result - - return extdef([unicode, int, int], int, "ll_os.ll_os_wopen", - llimpl=os_wopen_llimpl) - # ------------------------------- os.read ------------------------------- @registering(os.read) From afa at codespeak.net Thu Jul 15 23:29:31 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Jul 2010 23:29:31 +0200 (CEST) Subject: [pypy-svn] r76246 - pypy/branch/unicode_filename-2/pypy/rpython/module Message-ID: <20100715212931.F1FE9282BFC@codespeak.net> Author: afa Date: Thu Jul 15 23:29:29 2010 New Revision: 76246 Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Simplify more; apply to os.access Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Thu Jul 15 23:29:29 2010 @@ -31,7 +31,7 @@ from pypy.rlib import rgc from pypy.rlib.objectmodel import keepalive_until_here, specialize -def monkeypatch_rlib(posixfunc, unicodefunc, signature): +def monkeypatch_rposix(posixfunc, unicodefunc, signature): func_name = posixfunc.__name__ arglist = ['arg%d' % (i,) for i in range(len(signature))] @@ -80,7 +80,7 @@ def unicodefunc(*args): return posixfunc(*args) - monkeypatch_rlib(posixfunc, unicodefunc, signature) + monkeypatch_rposix(posixfunc, unicodefunc, signature) return registering(unicodefunc, condition=condition) @@ -108,7 +108,10 @@ def ll_os_name(name): return 'll_os.ll_os_w' + name -def registering_str_unicode(posixfunc, signature): +def registering_str_unicode(posixfunc, condition=True): + if not condition: + return registering(None, condition=False) + func_name = posixfunc.__name__ def register_posixfunc(self, method): @@ -121,7 +124,8 @@ def unicodefunc(*args): return posixfunc(*args) register_external(unicodefunc, *val.def_args, **val.def_kwds) - monkeypatch_rlib(posixfunc, unicodefunc, signature) + signature = val.def_args[0] + monkeypatch_rposix(posixfunc, unicodefunc, signature) def decorator(method): decorated = lambda self: register_posixfunc(self, method) @@ -894,7 +898,7 @@ def register_os_setsid(self): return self.extdef_for_os_function_returning_int('setsid') - @registering_str_unicode(os.open, [unicode, int, int]) + @registering_str_unicode(os.open) def register_os_open(self, ttypes): os_open = self.llexternal(ttypes.posix_function_name('open'), [ttypes.CCHARP, rffi.INT, rffi.MODE_T], @@ -1068,10 +1072,10 @@ llimpl=fdatasync_llimpl, export_name="ll_os.ll_os_fdatasync") - @registering(os.access) - def register_os_access(self): - os_access = self.llexternal(underscore_on_windows + 'access', - [rffi.CCHARP, rffi.INT], + @registering_str_unicode(os.access) + def register_os_access(self, ttypes): + os_access = self.llexternal(ttypes.posix_function_name('access'), + [ttypes.CCHARP, rffi.INT], rffi.INT) if sys.platform.startswith('win'): @@ -1088,25 +1092,10 @@ def os_access_oofakeimpl(path, mode): return os.access(OOSupport.from_rstr(path), mode) - return extdef([str, int], s_Bool, llimpl=access_llimpl, - export_name="ll_os.ll_os_access", + return extdef([ttypes.str, int], s_Bool, llimpl=access_llimpl, + export_name=ttypes.ll_os_name("access"), oofakeimpl=os_access_oofakeimpl) - @registering_unicode_version(os.access, [unicode, int], sys.platform=='win32') - def register_os_access_unicode(self): - os_waccess = self.llexternal(underscore_on_windows + 'waccess', - [rffi.CWCHARP, rffi.INT], - rffi.INT) - - def access_llimpl(path, mode): - # All files are executable on Windows - mode = mode & ~os.X_OK - error = rffi.cast(lltype.Signed, os_waccess(path, mode)) - return error == 0 - - return extdef([unicode, int], s_Bool, llimpl=access_llimpl, - export_name="ll_os.ll_os_waccess") - @registering_if(posix, '_getfullpathname') def register_posix__getfullpathname(self): from pypy.rlib import rwin32 From afa at codespeak.net Fri Jul 16 00:20:02 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Jul 2010 00:20:02 +0200 (CEST) Subject: [pypy-svn] r76247 - pypy/branch/unicode_filename-2/pypy/rpython/module Message-ID: <20100715222002.46925282B90@codespeak.net> Author: afa Date: Fri Jul 16 00:20:00 2010 New Revision: 76247 Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: Typo Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Fri Jul 16 00:20:00 2010 @@ -909,7 +909,7 @@ raise OSError(rposix.get_errno(), "os_open failed") return result - def os_open_oofakeimpl(o_path, flags, mode): + def os_open_oofakeimpl(path, flags, mode): return os.open(OOSupport.from_rstr(path), flags, mode) return extdef([ttypes.str, int, int], int, ttypes.ll_os_name('open'), From agaynor at codespeak.net Fri Jul 16 01:27:34 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Fri, 16 Jul 2010 01:27:34 +0200 (CEST) Subject: [pypy-svn] r76248 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20100715232734.DAF3F282BDB@codespeak.net> Author: agaynor Date: Fri Jul 16 01:27:33 2010 New Revision: 76248 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: Optimize int_sub(x, 0). Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Fri Jul 16 01:27:33 2010 @@ -18,6 +18,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.metainterp.history import AbstractDescr, make_hashable_int + def optimize_loop_1(metainterp_sd, loop): """Optimize loop.operations to make it match the input of loop.specnodes and to remove internal overheadish operations. Note that loop.specnodes @@ -992,6 +993,14 @@ self.make_equal_to(op.result, v1) else: self.optimize_default(op) + + def optimize_INT_SUB(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v2.is_constant() and v2.box.getint() == 0: + self.make_equal_to(op.result, v1) + else: + return self.optimize_default(op) optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Fri Jul 16 01:27:33 2010 @@ -2051,8 +2051,19 @@ jump(i1, i0) """ self.optimize_loop(ops, 'Not, Not', expected) - - + + def test_fold_partially_constant_ops(self): + ops = """ + [i0] + i1 = int_sub(i0, 0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + # ---------- def make_fail_descr(self): From hakanardo at codespeak.net Fri Jul 16 08:22:49 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Fri, 16 Jul 2010 08:22:49 +0200 (CEST) Subject: [pypy-svn] r76249 - pypy/branch/interplevel-array/pypy/module/pypyjit/test Message-ID: <20100716062249.C8309282BDB@codespeak.net> Author: hakanardo Date: Fri Jul 16 08:22:47 2010 New Revision: 76249 Modified: pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py Log: boolen rewriting tests Modified: pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py Fri Jul 16 08:22:47 2010 @@ -83,6 +83,7 @@ assert isinstance(expected_max_ops, int) source = py.code.Source(source) filepath = self.tmpdir.join('case%d.py' % self.counter) + print filepath logfilepath = filepath.new(ext='.log') self.__class__.counter += 1 f = filepath.open('w') @@ -615,16 +616,103 @@ return total ''', 170, ([], 4999450000L)) - def test_boolrewrite(self): - self.run_source(''' - def main(a,b,c): - sa = 0 - for i in range(100000): - if b < a: sa += 1 - else: sa += 2 - if b >= c: sa += 3 - else: sa += 4 - ''', 0, ([0, 1, 0], 7)) + def test_boolrewrite_invers(self): + for a, b, res, ops in (('2000', '2000', 20001000, 53), + ( '500', '500', 15001500, 83), + ( '300', '600', 16001700, 89), + ( 'a', 'b', 16001700, 89), + ( 'a', 'a', 13001700, 87)): + + self.run_source(''' + def main(): + sa = 0 + a = 300 + b = 600 + for i in range(1000): + if i < %s: sa += 1 + else: sa += 2 + if i >= %s: sa += 10000 + else: sa += 20000 + return sa + '''%(a, b), ops, ([], res)) + + def test_boolrewrite_reflex(self): + for a, b, res, ops in (('2000', '2000', 10001000, 53), + ( '500', '500', 15001500, 83), + ( '300', '600', 14001700, 89), + ( 'a', 'b', 14001700, 89), + ( 'a', 'a', 17001700, 87)): + + self.run_source(''' + def main(): + sa = 0 + a = 300 + b = 600 + for i in range(1000): + if i < %s: sa += 1 + else: sa += 2 + if %s > i: sa += 10000 + else: sa += 20000 + return sa + '''%(a, b), ops, ([], res)) + + + def test_boolrewrite_int_correct_invers(self): + def opval(i, op, a): + if eval('%d %s %d' % (i, op, a)): return 1 + return 2 + + ops = ('<', '>', '<=', '>=', '==', '!=') + for op1 in ops: + for op2 in ops: + for a,b in ((500, 500), (300, 600)): + res = 0 + res += opval(a-1, op1, a) * (a) + res += opval( a, op1, a) + res += opval(a+1, op1, a) * (1000 - a - 1) + res += opval(b-1, op2, b) * 10000 * (b) + res += opval( b, op2, b) * 10000 + res += opval(b+1, op2, b) * 10000 * (1000 - b - 1) + + self.run_source(''' + def main(): + sa = 0 + for i in range(1000): + if i %s %d: sa += 1 + else: sa += 2 + if i %s %d: sa += 10000 + else: sa += 20000 + return sa + '''%(op1, a, op2, b), 100, ([], res)) + + def test_boolrewrite_int_correct_reflex(self): + def opval(i, op, a): + if eval('%d %s %d' % (i, op, a)): return 1 + return 2 + + ops = ('<', '>', '<=', '>=', '==', '!=') + for op1 in ops: + for op2 in ops: + for a,b in ((500, 500), (300, 600)): + res = 0 + res += opval(a-1, op1, a) * (a) + res += opval( a, op1, a) + res += opval(a+1, op1, a) * (1000 - a - 1) + res += opval(b, op2, b-1) * 10000 * (b) + res += opval(b, op2, b) * 10000 + res += opval(b, op2, b+1) * 10000 * (1000 - b - 1) + + self.run_source(''' + def main(): + sa = 0 + for i in range(1000): + if i %s %d: sa += 1 + else: sa += 2 + if %d %s i: sa += 10000 + else: sa += 20000 + return sa + '''%(op1, a, b, op2), 100, ([], res)) + class AppTestJIT(PyPyCJITTests): def setup_class(cls): From afa at codespeak.net Fri Jul 16 09:42:01 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Jul 2010 09:42:01 +0200 (CEST) Subject: [pypy-svn] r76250 - pypy/branch/unicode_filename-2/pypy/rpython/module Message-ID: <20100716074201.6D54C282BDB@codespeak.net> Author: afa Date: Fri Jul 16 09:41:59 2010 New Revision: 76250 Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: More refactoring, less duplication Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Fri Jul 16 09:41:59 2010 @@ -1573,62 +1573,40 @@ return extdef([str], int, llimpl=system_llimpl, export_name="ll_os.ll_os_system") - @registering(os.unlink) - def register_os_unlink(self): - os_unlink = self.llexternal(underscore_on_windows+'unlink', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.unlink) + def register_os_unlink(self, ttypes): + os_unlink = self.llexternal(ttypes.posix_function_name('unlink'), + [ttypes.CCHARP], rffi.INT) def unlink_llimpl(pathname): res = rffi.cast(lltype.Signed, os_unlink(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_unlink failed") - return extdef([str], s_None, llimpl=unlink_llimpl, - export_name="ll_os.ll_os_unlink") + return extdef([ttypes.str], s_None, llimpl=unlink_llimpl, + export_name=ttypes.ll_os_name('unlink')) - @registering_unicode_version(os.unlink, [unicode], sys.platform=='win32') - def register_os_unlink_unicode(self): - os_wunlink = self.llexternal(underscore_on_windows+'wunlink', [rffi.CWCHARP], rffi.INT) - - def wunlink_llimpl(pathname): - res = rffi.cast(lltype.Signed, os_wunlink(pathname)) - if res < 0: - raise OSError(rposix.get_errno(), "os_unlink failed") - - return extdef([unicode], s_None, llimpl=wunlink_llimpl, - export_name="ll_os.ll_os_wunlink") - - @registering(os.chdir) - def register_os_chdir(self): - os_chdir = self.llexternal(underscore_on_windows+'chdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.chdir) + def register_os_chdir(self, ttypes): + os_chdir = self.llexternal(ttypes.posix_function_name('chdir'), + [ttypes.CCHARP], rffi.INT) def chdir_llimpl(path): res = rffi.cast(lltype.Signed, os_chdir(path)) if res < 0: raise OSError(rposix.get_errno(), "os_chdir failed") - return extdef([str], s_None, llimpl=chdir_llimpl, - export_name="ll_os.ll_os_chdir") - - @registering_unicode_version(os.chdir, [unicode], sys.platform=='win32') - def register_os_chdir_unicode(self): - os_wchdir = self.llexternal(underscore_on_windows+'wchdir', [rffi.CWCHARP], rffi.INT) - - def chdir_llimpl(path): - res = rffi.cast(lltype.Signed, os_wchdir(path)) - if res < 0: - raise OSError(rposix.get_errno(), "os_chdir failed") - - return extdef([unicode], s_None, llimpl=chdir_llimpl, - export_name="ll_os.ll_os_wchdir") + return extdef([ttypes.str], s_None, llimpl=chdir_llimpl, + export_name=ttypes.ll_os_name('chdir')) - @registering(os.mkdir) - def register_os_mkdir(self): + @registering_str_unicode(os.mkdir) + def register_os_mkdir(self, ttypes): if os.name == 'nt': ARG2 = [] # no 'mode' argument on Windows - just ignored else: ARG2 = [rffi.MODE_T] - os_mkdir = self.llexternal(underscore_on_windows+'mkdir', - [rffi.CCHARP]+ARG2, rffi.INT) + os_mkdir = self.llexternal(ttypes.posix_function_name('mkdir'), + [ttypes.CCHARP] + ARG2, rffi.INT) IGNORE_MODE = len(ARG2) == 0 def mkdir_llimpl(pathname, mode): @@ -1640,106 +1618,47 @@ if res < 0: raise OSError(rposix.get_errno(), "os_mkdir failed") - return extdef([str, int], s_None, llimpl=mkdir_llimpl, - export_name="ll_os.ll_os_mkdir") + return extdef([ttypes.str, int], s_None, llimpl=mkdir_llimpl, + export_name=ttypes.ll_os_name('mkdir')) - @registering_unicode_version(os.mkdir, [unicode, int], sys.platform=='win32') - def register_os_mkdir_unicode(self): - if os.name == 'nt': - ARG2 = [] # no 'mode' argument on Windows - just ignored - else: - ARG2 = [rffi.MODE_T] - os_wmkdir = self.llexternal(underscore_on_windows+'wmkdir', - [rffi.CWCHARP]+ARG2, rffi.INT) - IGNORE_MODE = len(ARG2) == 0 - - def mkdir_llimpl(pathname, mode): - if IGNORE_MODE: - res = os_wmkdir(pathname) - else: - res = os_wmkdir(pathname, mode) - res = rffi.cast(lltype.Signed, res) - if res < 0: - raise OSError(rposix.get_errno(), "os_mkdir failed") - - return extdef([unicode, int], s_None, llimpl=mkdir_llimpl, - export_name="ll_os.ll_os_wmkdir") - - @registering(os.rmdir) - def register_os_rmdir(self): - os_rmdir = self.llexternal(underscore_on_windows+'rmdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.rmdir) + def register_os_rmdir(self, ttypes): + os_rmdir = self.llexternal(ttypes.posix_function_name('rmdir'), + [ttypes.CCHARP], rffi.INT) def rmdir_llimpl(pathname): res = rffi.cast(lltype.Signed, os_rmdir(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_rmdir failed") - return extdef([str], s_None, llimpl=rmdir_llimpl, - export_name="ll_os.ll_os_rmdir") + return extdef([ttypes.str], s_None, llimpl=rmdir_llimpl, + export_name=ttypes.ll_os_name('rmdir')) - @registering_unicode_version(os.rmdir, [unicode], sys.platform=='win32') - def register_os_rmdir_unicode(self): - os_wrmdir = self.llexternal(underscore_on_windows+'wrmdir', [rffi.CWCHARP], rffi.INT) - - def rmdir_llimpl(pathname): - res = rffi.cast(lltype.Signed, os_wrmdir(pathname)) - if res < 0: - raise OSError(rposix.get_errno(), "os_rmdir failed") - - return extdef([unicode], s_None, llimpl=rmdir_llimpl, - export_name="ll_os.ll_os_wrmdir") - - @registering(os.chmod) - def register_os_chmod(self): - os_chmod = self.llexternal(underscore_on_windows+'chmod', [rffi.CCHARP, rffi.MODE_T], - rffi.INT) + @registering_str_unicode(os.chmod) + def register_os_chmod(self, ttypes): + os_chmod = self.llexternal(ttypes.posix_function_name('chmod'), + [ttypes.CCHARP, rffi.MODE_T], rffi.INT) def chmod_llimpl(path, mode): res = rffi.cast(lltype.Signed, os_chmod(path, rffi.cast(rffi.MODE_T, mode))) if res < 0: raise OSError(rposix.get_errno(), "os_chmod failed") - return extdef([str, int], s_None, llimpl=chmod_llimpl, - export_name="ll_os.ll_os_chmod") + return extdef([ttypes.str, int], s_None, llimpl=chmod_llimpl, + export_name=ttypes.ll_os_name('chmod')) - @registering_unicode_version(os.chmod, [unicode, int], sys.platform=='win32') - def register_os_chmod_unicode(self): - os_wchmod = self.llexternal(underscore_on_windows+'wchmod', [rffi.CWCHARP, rffi.MODE_T], - rffi.INT) - - def chmod_llimpl(path, mode): - res = rffi.cast(lltype.Signed, os_wchmod(path, rffi.cast(rffi.MODE_T, mode))) - if res < 0: - raise OSError(rposix.get_errno(), "os_chmod failed") - - return extdef([unicode, int], s_None, llimpl=chmod_llimpl, - export_name="ll_os.ll_os_wchmod") - - @registering(os.rename) - def register_os_rename(self): - os_rename = self.llexternal('rename', [rffi.CCHARP, rffi.CCHARP], - rffi.INT) + @registering_str_unicode(os.rename) + def register_os_rename(self, ttypes): + os_rename = self.llexternal(ttypes.posix_function_name('rename'), + [ttypes.CCHARP, ttypes.CCHARP], rffi.INT) def rename_llimpl(oldpath, newpath): res = rffi.cast(lltype.Signed, os_rename(oldpath, newpath)) if res < 0: raise OSError(rposix.get_errno(), "os_rename failed") - return extdef([str, str], s_None, llimpl=rename_llimpl, - export_name="ll_os.ll_os_rename") - - @registering_unicode_version(os.rename, [unicode, unicode], sys.platform=='win32') - def register_os_rename_unicode(self): - os_wrename = self.llexternal(underscore_on_windows+'wrename', - [rffi.CWCHARP, rffi.CWCHARP], rffi.INT) - - def rename_llimpl(oldpath, newpath): - res = rffi.cast(lltype.Signed, os_wrename(oldpath, newpath)) - if res < 0: - raise OSError(rposix.get_errno(), "os_rename failed") - - return extdef([unicode, unicode], s_None, llimpl=rename_llimpl, - export_name="ll_os.ll_os_wrename") + return extdef([ttypes.str, ttypes.str], s_None, llimpl=rename_llimpl, + export_name=ttypes.ll_os_name('rename')) @registering(os.umask) def register_os_umask(self): From afa at codespeak.net Fri Jul 16 09:44:02 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Jul 2010 09:44:02 +0200 (CEST) Subject: [pypy-svn] r76251 - pypy/branch/unicode_filename-2/pypy/rpython/module Message-ID: <20100716074402.3381B282BDB@codespeak.net> Author: afa Date: Fri Jul 16 09:44:00 2010 New Revision: 76251 Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Log: rename: types->traits Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Fri Jul 16 09:44:00 2010 @@ -84,7 +84,7 @@ return registering(unicodefunc, condition=condition) -class StringTypes: +class StringTraits: str = str CCHARP = rffi.CCHARP @@ -96,7 +96,7 @@ def ll_os_name(name): return 'll_os.ll_os_' + name -class UnicodeTypes: +class UnicodeTraits: str = unicode CCHARP = rffi.CWCHARP @@ -115,11 +115,11 @@ func_name = posixfunc.__name__ def register_posixfunc(self, method): - val = method(self, StringTypes()) + val = method(self, StringTraits()) register_external(posixfunc, *val.def_args, **val.def_kwds) if sys.platform == 'win32': - val = method(self, UnicodeTypes()) + val = method(self, UnicodeTraits()) @func_renamer(func_name + "_unicode") def unicodefunc(*args): return posixfunc(*args) @@ -899,9 +899,9 @@ return self.extdef_for_os_function_returning_int('setsid') @registering_str_unicode(os.open) - def register_os_open(self, ttypes): - os_open = self.llexternal(ttypes.posix_function_name('open'), - [ttypes.CCHARP, rffi.INT, rffi.MODE_T], + def register_os_open(self, traits): + os_open = self.llexternal(traits.posix_function_name('open'), + [traits.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT) def os_open_llimpl(path, flags, mode): result = rffi.cast(rffi.LONG, os_open(path, flags, mode)) @@ -912,7 +912,7 @@ def os_open_oofakeimpl(path, flags, mode): return os.open(OOSupport.from_rstr(path), flags, mode) - return extdef([ttypes.str, int, int], int, ttypes.ll_os_name('open'), + return extdef([traits.str, int, int], int, traits.ll_os_name('open'), llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) # ------------------------------- os.read ------------------------------- @@ -1073,9 +1073,9 @@ export_name="ll_os.ll_os_fdatasync") @registering_str_unicode(os.access) - def register_os_access(self, ttypes): - os_access = self.llexternal(ttypes.posix_function_name('access'), - [ttypes.CCHARP, rffi.INT], + def register_os_access(self, traits): + os_access = self.llexternal(traits.posix_function_name('access'), + [traits.CCHARP, rffi.INT], rffi.INT) if sys.platform.startswith('win'): @@ -1092,8 +1092,8 @@ def os_access_oofakeimpl(path, mode): return os.access(OOSupport.from_rstr(path), mode) - return extdef([ttypes.str, int], s_Bool, llimpl=access_llimpl, - export_name=ttypes.ll_os_name("access"), + return extdef([traits.str, int], s_Bool, llimpl=access_llimpl, + export_name=traits.ll_os_name("access"), oofakeimpl=os_access_oofakeimpl) @registering_if(posix, '_getfullpathname') @@ -1574,39 +1574,39 @@ export_name="ll_os.ll_os_system") @registering_str_unicode(os.unlink) - def register_os_unlink(self, ttypes): - os_unlink = self.llexternal(ttypes.posix_function_name('unlink'), - [ttypes.CCHARP], rffi.INT) + def register_os_unlink(self, traits): + os_unlink = self.llexternal(traits.posix_function_name('unlink'), + [traits.CCHARP], rffi.INT) def unlink_llimpl(pathname): res = rffi.cast(lltype.Signed, os_unlink(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_unlink failed") - return extdef([ttypes.str], s_None, llimpl=unlink_llimpl, - export_name=ttypes.ll_os_name('unlink')) + return extdef([traits.str], s_None, llimpl=unlink_llimpl, + export_name=traits.ll_os_name('unlink')) @registering_str_unicode(os.chdir) - def register_os_chdir(self, ttypes): - os_chdir = self.llexternal(ttypes.posix_function_name('chdir'), - [ttypes.CCHARP], rffi.INT) + def register_os_chdir(self, traits): + os_chdir = self.llexternal(traits.posix_function_name('chdir'), + [traits.CCHARP], rffi.INT) def chdir_llimpl(path): res = rffi.cast(lltype.Signed, os_chdir(path)) if res < 0: raise OSError(rposix.get_errno(), "os_chdir failed") - return extdef([ttypes.str], s_None, llimpl=chdir_llimpl, - export_name=ttypes.ll_os_name('chdir')) + return extdef([traits.str], s_None, llimpl=chdir_llimpl, + export_name=traits.ll_os_name('chdir')) @registering_str_unicode(os.mkdir) - def register_os_mkdir(self, ttypes): + def register_os_mkdir(self, traits): if os.name == 'nt': ARG2 = [] # no 'mode' argument on Windows - just ignored else: ARG2 = [rffi.MODE_T] - os_mkdir = self.llexternal(ttypes.posix_function_name('mkdir'), - [ttypes.CCHARP] + ARG2, rffi.INT) + os_mkdir = self.llexternal(traits.posix_function_name('mkdir'), + [traits.CCHARP] + ARG2, rffi.INT) IGNORE_MODE = len(ARG2) == 0 def mkdir_llimpl(pathname, mode): @@ -1618,47 +1618,47 @@ if res < 0: raise OSError(rposix.get_errno(), "os_mkdir failed") - return extdef([ttypes.str, int], s_None, llimpl=mkdir_llimpl, - export_name=ttypes.ll_os_name('mkdir')) + return extdef([traits.str, int], s_None, llimpl=mkdir_llimpl, + export_name=traits.ll_os_name('mkdir')) @registering_str_unicode(os.rmdir) - def register_os_rmdir(self, ttypes): - os_rmdir = self.llexternal(ttypes.posix_function_name('rmdir'), - [ttypes.CCHARP], rffi.INT) + def register_os_rmdir(self, traits): + os_rmdir = self.llexternal(traits.posix_function_name('rmdir'), + [traits.CCHARP], rffi.INT) def rmdir_llimpl(pathname): res = rffi.cast(lltype.Signed, os_rmdir(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_rmdir failed") - return extdef([ttypes.str], s_None, llimpl=rmdir_llimpl, - export_name=ttypes.ll_os_name('rmdir')) + return extdef([traits.str], s_None, llimpl=rmdir_llimpl, + export_name=traits.ll_os_name('rmdir')) @registering_str_unicode(os.chmod) - def register_os_chmod(self, ttypes): - os_chmod = self.llexternal(ttypes.posix_function_name('chmod'), - [ttypes.CCHARP, rffi.MODE_T], rffi.INT) + def register_os_chmod(self, traits): + os_chmod = self.llexternal(traits.posix_function_name('chmod'), + [traits.CCHARP, rffi.MODE_T], rffi.INT) def chmod_llimpl(path, mode): res = rffi.cast(lltype.Signed, os_chmod(path, rffi.cast(rffi.MODE_T, mode))) if res < 0: raise OSError(rposix.get_errno(), "os_chmod failed") - return extdef([ttypes.str, int], s_None, llimpl=chmod_llimpl, - export_name=ttypes.ll_os_name('chmod')) + return extdef([traits.str, int], s_None, llimpl=chmod_llimpl, + export_name=traits.ll_os_name('chmod')) @registering_str_unicode(os.rename) - def register_os_rename(self, ttypes): - os_rename = self.llexternal(ttypes.posix_function_name('rename'), - [ttypes.CCHARP, ttypes.CCHARP], rffi.INT) + def register_os_rename(self, traits): + os_rename = self.llexternal(traits.posix_function_name('rename'), + [traits.CCHARP, traits.CCHARP], rffi.INT) def rename_llimpl(oldpath, newpath): res = rffi.cast(lltype.Signed, os_rename(oldpath, newpath)) if res < 0: raise OSError(rposix.get_errno(), "os_rename failed") - return extdef([ttypes.str, ttypes.str], s_None, llimpl=rename_llimpl, - export_name=ttypes.ll_os_name('rename')) + return extdef([traits.str, traits.str], s_None, llimpl=rename_llimpl, + export_name=traits.ll_os_name('rename')) @registering(os.umask) def register_os_umask(self): From dan at codespeak.net Fri Jul 16 12:32:27 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Fri, 16 Jul 2010 12:32:27 +0200 (CEST) Subject: [pypy-svn] r76252 - in pypy/branch/micronumpy/pypy: module/micronumpy tool Message-ID: <20100716103227.02593282BD4@codespeak.net> Author: dan Date: Fri Jul 16 12:32:24 2010 New Revision: 76252 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py pypy/branch/micronumpy/pypy/tool/numpybench.py Log: Worst. Merge. Ever. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py Fri Jul 16 12:32:24 2010 @@ -41,7 +41,7 @@ ) storage_type = lltype.Ptr(lltype.Array(lltype.Char)) -null_storage = lltype.nullptr(storage_type.TO) +null_data = lltype.nullptr(storage_type.TO) class DescrBase(object): pass @@ -62,7 +62,7 @@ return space.wrap(self.getitem(data, index)) def w_setitem(self, space, data, index, w_value): - value = self.unwrap(space, w_value) + value = self.coerce_w(space, w_value) self.setitem(data, index, value) def itemsize(self): @@ -123,12 +123,14 @@ int_descr = descriptor('i', 'int32', lltype.Signed) type(int_descr).unwrap = lambda self, space, value: space.int_w(value) +type(int_descr).coerce_w = lambda self, space, value: space.int_w(space.int(value)) _int_index = _typeindex['i'] _typestring['int32'] = _int_index w_int_descr = _w_descriptors[_int_index] float_descr = descriptor('d', 'float64', lltype.Float) type(float_descr).unwrap = lambda self, space, value: space.float_w(value) +type(float_descr).coerce_w = lambda self, space, value: space.float_w(space.float(value)) _float_index = _typeindex['d'] _typestring['float64'] = _float_index w_float_descr = _w_descriptors[_float_index] Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Fri Jul 16 12:32:24 2010 @@ -12,7 +12,7 @@ from pypy.module import micronumpy from pypy.module.micronumpy.array import BaseNumArray from pypy.module.micronumpy.array import construct_array, infer_shape -from pypy.module.micronumpy.dtype import null_storage +from pypy.module.micronumpy.dtype import null_data def size_from_shape(shape): size = 1 @@ -84,7 +84,7 @@ elif parent is not None: self.data = parent.data else: - self.data = null_storage + self.data = null_data def descr_len(self, space): return space.wrap(self.shape[0]) @@ -209,7 +209,7 @@ descr_iter.unwrap_spec = ['self', ObjSpace] def __del__(self): - if self.parent is None and self.data is not None: + if self.parent is None and self.data != null_data: dtype = self.dtype.dtype dtype.free(self.data) Modified: pypy/branch/micronumpy/pypy/tool/numpybench.py ============================================================================== --- pypy/branch/micronumpy/pypy/tool/numpybench.py (original) +++ pypy/branch/micronumpy/pypy/tool/numpybench.py Fri Jul 16 12:32:24 2010 @@ -1,54 +1,33 @@ -from __future__ import division try: - import numpy as np + import numpy as numpy except ImportError, e: - import micronumpy as np + import micronumpy as numpy -def naive_convolve(f, g): - # f is an image and is indexed by (v, w) - # g is a filter kernel and is indexed by (s, t), - # it needs odd dimensions - # h is the output image and is indexed by (x, y), - # it is not cropped - if g.shape[0] % 2 != 1 or g.shape[1] % 2 != 1: - raise ValueError("Only odd dimensions on filter supported") - # smid and tmid are number of pixels between the center pixel - # and the edge, ie for a 5x5 filter they will be 2. - # - # The output size is calculated by adding smid, tmid to each - # side of the dimensions of the input image. - vmax = f.shape[0] - wmax = f.shape[1] - smax = g.shape[0] - tmax = g.shape[1] - smid = smax // 2 - tmid = tmax // 2 - xmax = vmax + 2*smid - ymax = wmax + 2*tmid - # Allocate result image. - h = np.zeros([xmax, ymax], dtype=f.dtype) - # Do convolution - for x in range(xmax): - for y in range(ymax): - print "(%d, %d)" % (x, y) - # Calculate pixel value for h at (x,y). Sum one component - # for each pixel (s, t) of the filter g. - s_from = max(smid - x, -smid) - s_to = min((xmax - x) - smid, smid + 1) - t_from = max(tmid - y, -tmid) - t_to = min((ymax - y) - tmid, tmid + 1) - value = 0 - for s in range(s_from, s_to): - for t in range(t_from, t_to): - v = x - smid + s - w = y - tmid + t - value += g[smid - s, tmid - t] * f[v, w] - h[x, y] = value - return h +def generate_image(width, height): + return numpy.array([[x + y for x in range(width)] for y in range(height)]) + +def generate_kernel(width, height): + from math import sin, pi + #kernel = numpy.zeros((width, height), dtype=int) # FIXME: micronumpy.zeros can't handle missing dtype + kernel = [[0] * width] * height + + for i in range(width): + for j in range(height): + u = i / float(width) + v = j / float(height) + kernel[j][i] = int((0.5 + sin(u * pi)) * (0.5 + sin(v * pi))) # DOUBLE FIXME: setitem doesn't coerce to array type + + #return kernel + return numpy.array(kernel) if __name__ == '__main__': - image = np.array([[x + y for x in range(200)] for y in range(200)]) - kernel = np.array([[0, 1, 0], - [1, 1, 1], - [0, 1, 0]]) - result = naive_convolve(image, kernel) + from sys import argv as args + width, height, kwidth, kheight = [int(x) for x in args[1:]] + + image = generate_image(width, height) + kernel = generate_kernel(kwidth, kheight) + + from timeit import Timer + convolve_timer = Timer('naive_convolve(image, kernel)', 'from convolve import naive_convolve; from __main__ import image, kernel; gc.enable()') + count = 100 + print "%.5f sec/pass" % (convolve_timer.timeit(number=count)/count) From afa at codespeak.net Fri Jul 16 13:45:42 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Jul 2010 13:45:42 +0200 (CEST) Subject: [pypy-svn] r76253 - pypy/branch/unicode_filename-2/pypy/rpython/module Message-ID: <20100716114542.F21F2282BD4@codespeak.net> Author: afa Date: Fri Jul 16 13:45:41 2010 New Revision: 76253 Added: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py (contents, props changed) Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os_stat.py Log: Move win32 functions to another file, share the str/unicode implementations. Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Fri Jul 16 13:45:41 2010 @@ -86,6 +86,7 @@ class StringTraits: str = str + CHAR = rffi.CHAR CCHARP = rffi.CCHARP @staticmethod @@ -98,6 +99,7 @@ class UnicodeTraits: str = unicode + CHAR = rffi.WCHAR_T CCHARP = rffi.CWCHARP @staticmethod @@ -1229,66 +1231,8 @@ def register_os_listdir(self): # we need a different approach on Windows and on Posix if sys.platform.startswith('win'): - from pypy.rlib import rwin32 - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'] - ) - WIN32_FIND_DATA = platform.Struct('struct _WIN32_FIND_DATAA', - [('cFileName', lltype.FixedSizeArray(rffi.CHAR, 1))]) - ERROR_FILE_NOT_FOUND = platform.ConstantInteger( - 'ERROR_FILE_NOT_FOUND') - ERROR_NO_MORE_FILES = platform.ConstantInteger( - 'ERROR_NO_MORE_FILES') - - config = platform.configure(CConfig) - WIN32_FIND_DATA = config['WIN32_FIND_DATA'] - ERROR_FILE_NOT_FOUND = config['ERROR_FILE_NOT_FOUND'] - ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] - LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) - - FindFirstFile = self.llexternal('FindFirstFileA', - [rwin32.LPCSTR, LPWIN32_FIND_DATA], - rwin32.HANDLE) - FindNextFile = self.llexternal('FindNextFileA', - [rwin32.HANDLE, LPWIN32_FIND_DATA], - rwin32.BOOL) - FindClose = self.llexternal('FindClose', - [rwin32.HANDLE], - rwin32.BOOL) - - def os_listdir_llimpl(path): - if path and path[-1] not in ('/', '\\', ':'): - path += '/' - path += '*.*' - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - try: - result = [] - hFindFile = FindFirstFile(path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - error = rwin32.GetLastError() - if error == ERROR_FILE_NOT_FOUND: - return result - else: - raise WindowsError(error, "FindFirstFile failed") - while True: - name = rffi.charp2str(rffi.cast(rffi.CCHARP, - filedata.c_cFileName)) - if name != "." and name != "..": # skip these - result.append(name) - if not FindNextFile(hFindFile, filedata): - break - # FindNextFile sets error to ERROR_NO_MORE_FILES if - # it got to the end of the directory - error = rwin32.GetLastError() - FindClose(hFindFile) - if error == ERROR_NO_MORE_FILES: - return result - else: - raise WindowsError(error, "FindNextFile failed") - finally: - lltype.free(filedata, flavor='raw') - + from pypy.rpython.module.win32file import make_listdir_impl + os_listdir_llimpl = make_listdir_impl(StringTraits()) else: compilation_info = ExternalCompilationInfo( includes = ['sys/types.h', 'dirent.h'] @@ -1336,70 +1280,12 @@ @registering_unicode_version(os.listdir, [unicode], sys.platform=='win32') def register_os_listdir_unicode(self): - from pypy.rlib import rwin32 - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'] - ) - WIN32_FIND_DATA = platform.Struct('struct _WIN32_FIND_DATAW', - [('cFileName', lltype.FixedSizeArray(rwin32.WCHAR, 250))]) - ERROR_FILE_NOT_FOUND = platform.ConstantInteger( - 'ERROR_FILE_NOT_FOUND') - ERROR_NO_MORE_FILES = platform.ConstantInteger( - 'ERROR_NO_MORE_FILES') - - config = platform.configure(CConfig) - WIN32_FIND_DATA = config['WIN32_FIND_DATA'] - ERROR_FILE_NOT_FOUND = config['ERROR_FILE_NOT_FOUND'] - ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] - LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) - - FindFirstFile = self.llexternal('FindFirstFileW', - [rwin32.LPCWSTR, LPWIN32_FIND_DATA], - rwin32.HANDLE) - FindNextFile = self.llexternal('FindNextFileW', - [rwin32.HANDLE, LPWIN32_FIND_DATA], - rwin32.BOOL) - FindClose = self.llexternal('FindClose', - [rwin32.HANDLE], - rwin32.BOOL) - - def os_listdir_llimpl(path): - if path and path[-1] not in (u'/', u'\\', u':'): - path += u'/' - path += u'*.*' - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - try: - result = [] - hFindFile = FindFirstFile(path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - error = rwin32.GetLastError() - if error == ERROR_FILE_NOT_FOUND: - return result - else: - raise WindowsError(error, "FindFirstFile failed") - while True: - name = rffi.wcharp2unicode(rffi.cast(rffi.CWCHARP, - filedata.c_cFileName)) - if name != u"." and name != u"..": # skip these - result.append(name) - if not FindNextFile(hFindFile, filedata): - break - # FindNextFile sets error to ERROR_NO_MORE_FILES if - # it got to the end of the directory - error = rwin32.GetLastError() - FindClose(hFindFile) - if error == ERROR_NO_MORE_FILES: - return result - else: - raise WindowsError(error, "FindNextFile failed") - finally: - lltype.free(filedata, flavor='raw') + from pypy.rpython.module.win32file import make_listdir_impl return extdef([unicode], # a single argument which is a unicode [unicode], # returns a list of unicodes "ll_os.ll_os_wlistdir", - llimpl=os_listdir_llimpl) + llimpl=make_listdir_impl(UnicodeTraits())) @registering(os.pipe) def register_os_pipe(self): Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os_stat.py Fri Jul 16 13:45:41 2010 @@ -5,7 +5,7 @@ import os, sys from pypy.annotation import model as annmodel from pypy.tool.pairtype import pairtype -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rpython import extregistry from pypy.rpython.extfunc import register_external, extdef from pypy.rpython.lltypesystem import rffi, lltype @@ -229,6 +229,7 @@ arg_is_path = (name != 'fstat') + @func_renamer('os_%s_llimpl' % (name,)) def posix_stat_llimpl(arg): stresult = lltype.malloc(STAT_STRUCT.TO, flavor='raw') try: @@ -243,6 +244,7 @@ finally: lltype.free(stresult, flavor='raw') + @func_renamer('os_%s_fake' % (name,)) def posix_fakeimpl(arg): if s_arg == str: arg = hlstr(arg) @@ -273,18 +275,12 @@ compilation_info=compilation_info) return extdef( - [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(posix_stat_llimpl, - 'os_%s_llimpl' % (name,)), - llfakeimpl=func_with_new_name(posix_fakeimpl, - 'os_%s_fake' % (name,)), - ) + [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), + llimpl=posix_stat_llimpl, llfakeimpl=posix_fakeimpl) else: # See Win32 implementation below return extdef( - [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), + [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), llimpl=func_with_new_name(globals()['win32_%s_llimpl' % (name,)], 'os_%s_llimpl' % (name,)), ) @@ -303,8 +299,7 @@ # ____________________________________________________________ if sys.platform == 'win32': # The CRT of Windows has a number of flaws wrt. its stat() implementation: - # - for when we implement subsecond resolution in RPython, time stamps - # would be restricted to second resolution + # - time stamps are restricted to second resolution # - file modification times suffer from forth-and-back conversions between # UTC and local time # Therefore, we implement our own stat, based on the Win32 API directly. Added: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py ============================================================================== --- (empty file) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py Fri Jul 16 13:45:41 2010 @@ -0,0 +1,112 @@ +""" +The Windows implementation of some posix modules, +based on the Win32 API. +""" + +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rpython.tool import rffi_platform as platform + +def make_win32_traits(traits): + from pypy.rlib import rwin32 + + if traits.str is unicode: + suffix = 'W' + else: + suffix = 'A' + + class CConfig: + _compilation_info_ = ExternalCompilationInfo( + includes = ['windows.h'] + ) + WIN32_FIND_DATA = platform.Struct( + 'struct _WIN32_FIND_DATA' + suffix, + # Only interesting fields + [('dwFileAttributes', rwin32.DWORD), + ('nFileSizeHigh', rwin32.DWORD), + ('nFileSizeLow', rwin32.DWORD), + ('ftCreationTime', rwin32.FILETIME), + ('ftLastAccessTime', rwin32.FILETIME), + ('ftLastWriteTime', rwin32.FILETIME), + ('cFileName', lltype.FixedSizeArray(traits.CHAR, 250))]) + ERROR_FILE_NOT_FOUND = platform.ConstantInteger( + 'ERROR_FILE_NOT_FOUND') + ERROR_NO_MORE_FILES = platform.ConstantInteger( + 'ERROR_NO_MORE_FILES') + + def external(*args, **kwargs): + kwargs['compilation_info'] = CConfig._compilation_info_ + return rffi.llexternal(*args, **kwargs) + + config = platform.configure(CConfig) + + class Win32Traits: + WIN32_FIND_DATA = config['WIN32_FIND_DATA'] + ERROR_FILE_NOT_FOUND = config['ERROR_FILE_NOT_FOUND'] + ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] + LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) + + FindFirstFile = external('FindFirstFile' + suffix, + [traits.CCHARP, LPWIN32_FIND_DATA], + rwin32.HANDLE) + FindNextFile = external('FindNextFile' + suffix, + [rwin32.HANDLE, LPWIN32_FIND_DATA], + rwin32.BOOL) + FindClose = external('FindClose', + [rwin32.HANDLE], + rwin32.BOOL) + + return Win32Traits + +def make_listdir_impl(traits): + from pypy.rlib import rwin32 + win32traits = make_win32_traits(traits) + + if traits.str is unicode: + def make_listdir_mask(path): + if path and path[-1] not in (u'/', u'\\', u':'): + path += u'/' + return path + u'*.*' + + def skip_listdir(path): + return name == u"." or name == u"..": + else: + def make_listdir_mask(path): + if path and path[-1] not in ('/', '\\', ':'): + path += '/' + return path + '*.*' + + def skip_listdir(path): + return name == "." or name == "..": + + def listdir_llimpl(path): + mask = make_listdir_mask(path) + filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') + try: + result = [] + hFindFile = win32traits.FindFirstFile(mask, filedata) + if hFindFile == rwin32.INVALID_HANDLE_VALUE: + error = rwin32.GetLastError() + if error == win32traits.ERROR_FILE_NOT_FOUND: + return result + else: + raise WindowsError(error, "FindFirstFile failed") + while True: + name = rffi.wcharp2unicode(rffi.cast(rffi.CWCHARP, + filedata.c_cFileName)) + if not skip_listdir(name): + result.append(name) + if not win32traits.FindNextFile(hFindFile, filedata): + break + # FindNextFile sets error to ERROR_NO_MORE_FILES if + # it got to the end of the directory + error = rwin32.GetLastError() + win32traits.FindClose(hFindFile) + if error == win32traits.ERROR_NO_MORE_FILES: + return result + else: + raise WindowsError(error, "FindNextFile failed") + finally: + lltype.free(filedata, flavor='raw') + + return listdir_llimpl From afa at codespeak.net Fri Jul 16 13:57:18 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Jul 2010 13:57:18 +0200 (CEST) Subject: [pypy-svn] r76254 - pypy/branch/unicode_filename-2/pypy/rpython/module Message-ID: <20100716115718.4E4DE282BD4@codespeak.net> Author: afa Date: Fri Jul 16 13:57:16 2010 New Revision: 76254 Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py Log: Fix translation, and use the new "registering" decorator Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Fri Jul 16 13:57:16 2010 @@ -88,6 +88,7 @@ str = str CHAR = rffi.CHAR CCHARP = rffi.CCHARP + str2charp = rffi.str2charp @staticmethod def posix_function_name(name): @@ -101,6 +102,7 @@ str = unicode CHAR = rffi.WCHAR_T CCHARP = rffi.CWCHARP + str2charp = rffi.wcharp2unicode @staticmethod def posix_function_name(name): @@ -1227,13 +1229,14 @@ return extdef([], unicode, "ll_os.ll_os_wgetcwd", llimpl=os_getcwd_llimpl) - @registering(os.listdir) - def register_os_listdir(self): + @registering_str_unicode(os.listdir) + def register_os_listdir(self, traits): # we need a different approach on Windows and on Posix if sys.platform.startswith('win'): from pypy.rpython.module.win32file import make_listdir_impl - os_listdir_llimpl = make_listdir_impl(StringTraits()) + os_listdir_llimpl = make_listdir_impl(traits) else: + assert traits.str is str compilation_info = ExternalCompilationInfo( includes = ['sys/types.h', 'dirent.h'] ) @@ -1273,20 +1276,11 @@ raise OSError(error, "os_readdir failed") return result - return extdef([str], # a single argument which is a str - [str], # returns a list of strings - "ll_os.ll_os_listdir", + return extdef([traits.str], # a single argument which is a str + [traits.str], # returns a list of strings + traits.ll_os_name('listdir'), llimpl=os_listdir_llimpl) - @registering_unicode_version(os.listdir, [unicode], sys.platform=='win32') - def register_os_listdir_unicode(self): - from pypy.rpython.module.win32file import make_listdir_impl - - return extdef([unicode], # a single argument which is a unicode - [unicode], # returns a list of unicodes - "ll_os.ll_os_wlistdir", - llimpl=make_listdir_impl(UnicodeTraits())) - @registering(os.pipe) def register_os_pipe(self): # we need a different approach on Windows and on Posix Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py Fri Jul 16 13:57:16 2010 @@ -79,7 +79,8 @@ def skip_listdir(path): return name == "." or name == "..": - def listdir_llimpl(path): + @func_renamer('listdir_llimpl_%s' % traits.str.__name__) + def os_listdir_llimpl(path): mask = make_listdir_mask(path) filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') try: @@ -92,8 +93,8 @@ else: raise WindowsError(error, "FindFirstFile failed") while True: - name = rffi.wcharp2unicode(rffi.cast(rffi.CWCHARP, - filedata.c_cFileName)) + name = traits.str2charp(rffi.cast(traits.CCHARP, + filedata.c_cFileName)) if not skip_listdir(name): result.append(name) if not win32traits.FindNextFile(hFindFile, filedata): From afa at codespeak.net Fri Jul 16 14:12:33 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Jul 2010 14:12:33 +0200 (CEST) Subject: [pypy-svn] r76255 - pypy/branch/unicode_filename-2/pypy/rpython/module Message-ID: <20100716121233.99B3D282BFC@codespeak.net> Author: afa Date: Fri Jul 16 14:12:31 2010 New Revision: 76255 Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py Log: more progress Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Fri Jul 16 14:12:31 2010 @@ -88,7 +88,8 @@ str = str CHAR = rffi.CHAR CCHARP = rffi.CCHARP - str2charp = rffi.str2charp + charp2str = staticmethod(rffi.charp2str) + str2charp = staticmethod(rffi.str2charp) @staticmethod def posix_function_name(name): @@ -102,7 +103,8 @@ str = unicode CHAR = rffi.WCHAR_T CCHARP = rffi.CWCHARP - str2charp = rffi.wcharp2unicode + charp2str = staticmethod(rffi.wcharp2unicode) + str2charp = staticmethod(rffi.unicode2wcharp) @staticmethod def posix_function_name(name): @@ -1100,73 +1102,17 @@ export_name=traits.ll_os_name("access"), oofakeimpl=os_access_oofakeimpl) - @registering_if(posix, '_getfullpathname') - def register_posix__getfullpathname(self): - from pypy.rlib import rwin32 - # this nt function is not exposed via os, but needed - # to get a correct implementation of os.abspath - # XXX why do we ignore WINAPI conventions everywhere? - LPSTRP = rffi.CArrayPtr(rwin32.LPSTR) - GetFullPathName = self.llexternal( - 'GetFullPathNameA', - [rwin32.LPCSTR, - rwin32.DWORD, - rwin32.LPSTR, - rffi.CArrayPtr(rwin32.LPSTR)], - rwin32.DWORD) - - def _getfullpathname_llimpl(lpFileName): - nBufferLength = rwin32.MAX_PATH + 1 - lpBuffer = lltype.malloc(rwin32.LPSTR.TO, nBufferLength, flavor='raw') - try: - res = GetFullPathName( - lpFileName, rffi.cast(rwin32.DWORD, nBufferLength), - lpBuffer, lltype.nullptr(LPSTRP.TO)) - if res == 0: - raise rwin32.lastWindowsError("_getfullpathname failed") - result = rffi.charp2str(lpBuffer) - return result - finally: - lltype.free(lpBuffer, flavor='raw') - - return extdef([str], # a single argument which is a str - str, # returns a string - "ll_os.posix__getfullpathname", - llimpl=_getfullpathname_llimpl) - - @registering_unicode_version(getattr(posix, '_getfullpathname', None), - [unicode], sys.platform=='win32') - def register_posix__getfullpathname_unicode(self): - from pypy.rlib import rwin32 + @registering_str_unicode(getattr(posix, '_getfullpathname', None), + condition=sys.platform=='win32') + def register_posix__getfullpathname(self, traits): # this nt function is not exposed via os, but needed # to get a correct implementation of os.abspath - # XXX why do we ignore WINAPI conventions everywhere? - LPWSTRP = rffi.CArrayPtr(rwin32.LPWSTR) - GetFullPathName = self.llexternal( - 'GetFullPathNameW', - [rwin32.LPCWSTR, - rwin32.DWORD, - rwin32.LPWSTR, - rffi.CArrayPtr(rwin32.LPWSTR)], - rwin32.DWORD) - - def _getfullpathname_llimpl(lpFileName): - nBufferLength = rwin32.MAX_PATH + 1 - lpBuffer = lltype.malloc(rwin32.LPWSTR.TO, nBufferLength, flavor='raw') - try: - res = GetFullPathName( - lpFileName, rffi.cast(rwin32.DWORD, nBufferLength), - lpBuffer, lltype.nullptr(LPWSTRP.TO)) - if res == 0: - raise rwin32.lastWindowsError("_getfullpathname failed") - result = rffi.wcharp2unicode(lpBuffer) - return result - finally: - lltype.free(lpBuffer, flavor='raw') + from pypy.rpython.module.ll_win32file import make_getfullpathname_impl + _getfullpathname_llimpl = make_getfullpathname_impl(traits) - return extdef([unicode], # a single argument which is a str - unicode, # returns a string - "ll_os.posix__wgetfullpathname", + return extdef([traits.str], # a single argument which is a str + traits.str, # returns a string + traits.ll_os_function('_getfullpathname'), llimpl=_getfullpathname_llimpl) @registering(os.getcwd) @@ -1233,7 +1179,7 @@ def register_os_listdir(self, traits): # we need a different approach on Windows and on Posix if sys.platform.startswith('win'): - from pypy.rpython.module.win32file import make_listdir_impl + from pypy.rpython.module.ll_win32file import make_listdir_impl os_listdir_llimpl = make_listdir_impl(traits) else: assert traits.str is str Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py Fri Jul 16 14:12:31 2010 @@ -6,6 +6,7 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.tool import rffi_platform as platform +from pypy.tool.sourcetools import func_renamer def make_win32_traits(traits): from pypy.rlib import rwin32 @@ -68,19 +69,19 @@ path += u'/' return path + u'*.*' - def skip_listdir(path): - return name == u"." or name == u"..": + def skip_listdir(name): + return name == u"." or name == u".." else: def make_listdir_mask(path): if path and path[-1] not in ('/', '\\', ':'): path += '/' return path + '*.*' - def skip_listdir(path): - return name == "." or name == "..": + def skip_listdir(name): + return name == "." or name == ".." @func_renamer('listdir_llimpl_%s' % traits.str.__name__) - def os_listdir_llimpl(path): + def listdir_llimpl(path): mask = make_listdir_mask(path) filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') try: @@ -93,7 +94,7 @@ else: raise WindowsError(error, "FindFirstFile failed") while True: - name = traits.str2charp(rffi.cast(traits.CCHARP, + name = traits.charp2str(rffi.cast(traits.CCHARP, filedata.c_cFileName)) if not skip_listdir(name): result.append(name) @@ -111,3 +112,26 @@ lltype.free(filedata, flavor='raw') return listdir_llimpl + +def make_getfullpathname_impl(traits): + win32traits = make_win32_traits(traits) + + LPSTRP = rffi.CArrayPtr(traits.CCHARP) + + @func_renamer('getfullpathname_llimpl_%s' % traits.str.__name__) + def getfullpathname_llimpl(path): + # XXX why do we ignore WINAPI conventions everywhere? + nBufferLength = rwin32.MAX_PATH + 1 + lpBuffer = lltype.malloc(traits.CCHARP.TO, nBufferLength, flavor='raw') + try: + res = win32traits.GetFullPathName( + lpFileName, rffi.cast(rwin32.DWORD, nBufferLength), + lpBuffer, lltype.nullptr(LPSTRP.TO)) + if res == 0: + raise rwin32.lastWindowsError("_getfullpathname failed") + result = traits.charp2str(lpBuffer) + return result + finally: + lltype.free(lpBuffer, flavor='raw') + + return getfullpathname_llimpl From getxsick at codespeak.net Fri Jul 16 15:04:52 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 16 Jul 2010 15:04:52 +0200 (CEST) Subject: [pypy-svn] r76256 - in pypy/branch/fast-ctypes/pypy: module/jitffi rlib Message-ID: <20100716130452.D7875282BD4@codespeak.net> Author: getxsick Date: Fri Jul 16 15:04:50 2010 New Revision: 76256 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: fix translation and revert wrong r76213 which was intended to fix the error Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Fri Jul 16 15:04:50 2010 @@ -17,7 +17,7 @@ args_type_w = [ space.str_w(w_x) for w_x in space.listview(w_args_type) ] try: - ret = W_Get(space, self.rcdll.cpu, space.wrap(self.lib_w), + ret = W_Get(space, self.rcdll.cpu, self.lib_w, func, args_type_w, res_type) except ValueError, e: raise OperationError(space.w_ValueError, space.wrap(str(e))) Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Fri Jul 16 15:04:50 2010 @@ -25,12 +25,11 @@ def __init__(self, name): name_ptr = rffi.str2charp(name) try: - handler = rdynload.dlopen(name_ptr) + self.handler = rdynload.dlopen(name_ptr) except rdynload.DLOpenError, e: raise OSError('%s: %s', name, e.msg or 'unspecified error') finally: rffi.free_charp(name_ptr) - self.handler = handler class _Get(object): def __init__(self, cpu, lib, func, args_type, res_type='v'): From hakanardo at codespeak.net Fri Jul 16 16:30:23 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Fri, 16 Jul 2010 16:30:23 +0200 (CEST) Subject: [pypy-svn] r76259 - in pypy/branch/interplevel-array/pypy: jit/metainterp module/pypyjit/test Message-ID: <20100716143023.E2ADF282BD4@codespeak.net> Author: hakanardo Date: Fri Jul 16 16:30:22 2010 New Revision: 76259 Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py Log: boolean rewriting tests Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py Fri Jul 16 16:30:22 2010 @@ -611,8 +611,8 @@ assert oldop.opnum == op.opnum self.make_equal_to(op.result, self.getvalue(oldop.result)) return - #elif self.find_rewriteable_constant(op, args): - # return + elif self.find_rewriteable_bool(op, args): + return else: self.pure_operations[args] = op @@ -620,7 +620,7 @@ self.emit_operation(op) - def find_rewriteable_constant(self, op, args): + def find_rewriteable_bool(self, op, args): try: oldopnum = opboolinvers[op.opnum] targ = [args[0], args[1], ConstInt(oldopnum)] @@ -642,12 +642,12 @@ targ = [args[1], args[0], ConstInt(oldopnum)] oldop = self.pure_operations.get(targ, None) if oldop is not None and oldop.descr is op.descr: - value = self.getvalue(oldop.result) - if value.is_constant(): - self.make_constant(op.result, value.box) - return True + self.make_equal_to(op.result, self.getvalue(oldop.result)) + return True except KeyError: pass + + # FIXME: oldopnum = opboolinvers[opboolreflex[op.opnum]]? return False Modified: pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py Fri Jul 16 16:30:22 2010 @@ -617,11 +617,11 @@ ''', 170, ([], 4999450000L)) def test_boolrewrite_invers(self): - for a, b, res, ops in (('2000', '2000', 20001000, 53), - ( '500', '500', 15001500, 83), - ( '300', '600', 16001700, 89), + for a, b, res, ops in (('2000', '2000', 20001000, 51), + ( '500', '500', 15001500, 81), + ( '300', '600', 16001700, 83), ( 'a', 'b', 16001700, 89), - ( 'a', 'a', 13001700, 87)): + ( 'a', 'a', 13001700, 85)): self.run_source(''' def main(): @@ -637,11 +637,11 @@ '''%(a, b), ops, ([], res)) def test_boolrewrite_reflex(self): - for a, b, res, ops in (('2000', '2000', 10001000, 53), - ( '500', '500', 15001500, 83), - ( '300', '600', 14001700, 89), + for a, b, res, ops in (('2000', '2000', 10001000, 51), + ( '500', '500', 15001500, 81), + ( '300', '600', 14001700, 83), ( 'a', 'b', 14001700, 89), - ( 'a', 'a', 17001700, 87)): + ( 'a', 'a', 17001700, 85)): self.run_source(''' def main(): @@ -657,7 +657,7 @@ '''%(a, b), ops, ([], res)) - def test_boolrewrite_int_correct_invers(self): + def test_boolrewrite_correct_invers(self): def opval(i, op, a): if eval('%d %s %d' % (i, op, a)): return 1 return 2 @@ -683,9 +683,23 @@ if i %s %d: sa += 10000 else: sa += 20000 return sa - '''%(op1, a, op2, b), 100, ([], res)) + '''%(op1, a, op2, b), 83, ([], res)) - def test_boolrewrite_int_correct_reflex(self): + self.run_source(''' + def main(): + sa = 0 + i = 0.0 + while i < 250.0: + if i %s %f: sa += 1 + else: sa += 2 + if i %s %f: sa += 10000 + else: sa += 20000 + i += 0.25 + return sa + '''%(op1, float(a)/4.0, op2, float(b)/4.0), 109, ([], res)) + + + def test_boolrewrite_correct_reflex(self): def opval(i, op, a): if eval('%d %s %d' % (i, op, a)): return 1 return 2 @@ -711,8 +725,58 @@ if %d %s i: sa += 10000 else: sa += 20000 return sa - '''%(op1, a, b, op2), 100, ([], res)) + '''%(op1, a, b, op2), 83, ([], res)) + + self.run_source(''' + def main(): + sa = 0 + i = 0.0 + while i < 250.0: + if i %s %f: sa += 1 + else: sa += 2 + if %f %s i: sa += 10000 + else: sa += 20000 + i += 0.25 + return sa + '''%(op1, float(a)/4.0, float(b)/4.0, op2), 109, ([], res)) + def test_boolrewrite_ptr(self): + compares = ('a == b', 'b == a', 'a != b', 'a == c') + for e1 in compares: + for e2 in compares: + a, b, c = 1, 2, 3 + if eval(e1): res = 752 * 1 + else: res = 752 * 2 + if eval(e2): res += 752 * 10000 + else: res += 752 * 20000 + a = b + if eval(e1): res += 248 * 1 + else: res += 248 * 2 + if eval(e2): res += 248 * 10000 + else: res += 248 * 20000 + + + if 'c' in e1 or 'c' in e2: + n = 245 + else: + n = 215 + + self.run_source(''' + class tst: + pass + def main(): + a = tst() + b = tst() + c = tst() + sa = 0 + for i in range(1000): + if %s: sa += 1 + else: sa += 2 + if %s: sa += 10000 + else: sa += 20000 + if i > 750: a = b + return sa + '''%(e1, e2), n, ([], res)) class AppTestJIT(PyPyCJITTests): def setup_class(cls): From getxsick at codespeak.net Fri Jul 16 16:39:29 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 16 Jul 2010 16:39:29 +0200 (CEST) Subject: [pypy-svn] r76260 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100716143929.94667282BD4@codespeak.net> Author: getxsick Date: Fri Jul 16 16:39:27 2010 New Revision: 76260 Modified: pypy/branch/fast-ctypes/pypy/rlib/rdynload.py Log: merge r74994 from trunk Modified: pypy/branch/fast-ctypes/pypy/rlib/rdynload.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rdynload.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rdynload.py Fri Jul 16 16:39:27 2010 @@ -18,8 +18,6 @@ if _WIN32: from pypy.rlib import rwin32 - -if _WIN32: includes = ['windows.h'] else: includes = ['dlfcn.h'] @@ -138,7 +136,7 @@ intresource = rffi.cast(rffi.CCHARP, r_uint(index) & 0xFFFF) res = rwin32.GetProcAddress(handle, intresource) if not res: - raise KeyError(name) + raise KeyError(index) # XXX rffi.cast here... return res From getxsick at codespeak.net Fri Jul 16 17:26:49 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 16 Jul 2010 17:26:49 +0200 (CEST) Subject: [pypy-svn] r76261 - pypy/trunk/pypy/jit/backend/llsupport/test Message-ID: <20100716152649.1CA06282BD4@codespeak.net> Author: getxsick Date: Fri Jul 16 17:26:47 2010 New Revision: 76261 Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py Log: fix tests Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py Fri Jul 16 17:26:47 2010 @@ -232,11 +232,11 @@ # descr2 = get_field_descr(c0, S, 'y') o, _ = symbolic.get_field_token(S, 'y', False) - assert descr2.repr_of_descr() == '' % o + assert descr2.repr_of_descr() == '' % o # descr2i = get_field_descr(c0, S, 'x') o, _ = symbolic.get_field_token(S, 'x', False) - assert descr2i.repr_of_descr() == '' % o + assert descr2i.repr_of_descr() == '' % o # descr3 = get_array_descr(c0, lltype.GcArray(lltype.Ptr(S))) assert descr3.repr_of_descr() == '' From afa at codespeak.net Fri Jul 16 17:31:41 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Jul 2010 17:31:41 +0200 (CEST) Subject: [pypy-svn] r76262 - pypy/branch/unicode_filename-2/pypy/rpython/module Message-ID: <20100716153141.3C7DD282BD4@codespeak.net> Author: afa Date: Fri Jul 16 17:31:39 2010 New Revision: 76262 Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os_stat.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py Log: more progress around the win32 implementations of os.stat() Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Fri Jul 16 17:31:39 2010 @@ -90,6 +90,7 @@ CCHARP = rffi.CCHARP charp2str = staticmethod(rffi.charp2str) str2charp = staticmethod(rffi.str2charp) + free_charp = staticmethod(rffi.free_charp) @staticmethod def posix_function_name(name): @@ -105,6 +106,7 @@ CCHARP = rffi.CWCHARP charp2str = staticmethod(rffi.wcharp2unicode) str2charp = staticmethod(rffi.unicode2wcharp) + free_charp = staticmethod(rffi.free_wcharp) @staticmethod def posix_function_name(name): @@ -1108,12 +1110,12 @@ # this nt function is not exposed via os, but needed # to get a correct implementation of os.abspath from pypy.rpython.module.ll_win32file import make_getfullpathname_impl - _getfullpathname_llimpl = make_getfullpathname_impl(traits) + getfullpathname_llimpl = make_getfullpathname_impl(traits) return extdef([traits.str], # a single argument which is a str traits.str, # returns a string - traits.ll_os_function('_getfullpathname'), - llimpl=_getfullpathname_llimpl) + traits.ll_os_name('_getfullpathname'), + llimpl=getfullpathname_llimpl) @registering(os.getcwd) def register_os_getcwd(self): @@ -1593,27 +1595,17 @@ @registering(os.fstat) def register_os_fstat(self): from pypy.rpython.module import ll_os_stat - return ll_os_stat.register_stat_variant('fstat') + return ll_os_stat.register_stat_variant('fstat', StringTraits()) - @registering(os.stat) - def register_os_stat(self): + @registering_str_unicode(os.stat) + def register_os_stat(self, traits): from pypy.rpython.module import ll_os_stat - return ll_os_stat.register_stat_variant('stat') + return ll_os_stat.register_stat_variant('stat', traits) - @registering(os.lstat) - def register_os_lstat(self): + @registering_str_unicode(os.lstat) + def register_os_lstat(self, traits): from pypy.rpython.module import ll_os_stat - return ll_os_stat.register_stat_variant('lstat') - - @registering_unicode_version(os.stat, [unicode], sys.platform=='win32') - def register_os_stat_unicode(self): - from pypy.rpython.module import ll_os_stat - return ll_os_stat.register_stat_variant_unicode('stat') - - @registering_unicode_version(os.lstat, [unicode], sys.platform=='win32') - def register_os_lstat_unicode(self): - from pypy.rpython.module import ll_os_stat - return ll_os_stat.register_stat_variant_unicode('lstat') + return ll_os_stat.register_stat_variant('lstat', traits) # ------------------------------- os.W* --------------------------------- Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os_stat.py Fri Jul 16 17:31:39 2010 @@ -212,13 +212,27 @@ return make_stat_result(result) -def register_stat_variant(name): - if sys.platform.startswith('win'): - _functions = {'stat': '_stati64', - 'fstat': '_fstati64', - 'lstat': '_stati64'} # no lstat on Windows - c_func_name = _functions[name] - elif sys.platform.startswith('linux'): +def register_stat_variant(name, traits): + if name != 'fstat': + arg_is_path = True + s_arg = traits.str + ARG1 = traits.CCHARP + else: + arg_is_path = False + s_arg = int + ARG1 = rffi.INT + + if sys.platform == 'win32': + # See Win32 implementation below + posix_stat_llimpl = make_stat_impl(name, traits) + + return extdef( + [s_arg], s_StatResult, traits.ll_os_name(name), + llimpl=posix_stat_llimpl) + + assert traits.str is str + + if sys.platform.startswith('linux'): # because we always use _FILE_OFFSET_BITS 64 - this helps things work that are not a c compiler _functions = {'stat': 'stat64', 'fstat': 'fstat64', @@ -227,17 +241,19 @@ else: c_func_name = name - arg_is_path = (name != 'fstat') + posix_mystat = rffi.llexternal(c_func_name, + [ARG1, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info) @func_renamer('os_%s_llimpl' % (name,)) def posix_stat_llimpl(arg): stresult = lltype.malloc(STAT_STRUCT.TO, flavor='raw') try: if arg_is_path: - arg = rffi.str2charp(arg) + arg = traits.str2charp(arg) error = rffi.cast(rffi.LONG, posix_mystat(arg, stresult)) if arg_is_path: - rffi.free_charp(arg) + traits.free_charp(arg) if error != 0: raise OSError(rposix.get_errno(), "os_?stat failed") return build_stat_result(stresult) @@ -262,42 +278,15 @@ setattr(ll_tup, 'item%d' % i, val) return ll_tup - if arg_is_path: - s_arg = str - ARG1 = rffi.CCHARP - else: - s_arg = int - ARG1 = rffi.INT - - if sys.platform != 'win32': - posix_mystat = rffi.llexternal(c_func_name, - [ARG1, STAT_STRUCT], rffi.INT, - compilation_info=compilation_info) - - return extdef( - [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), - llimpl=posix_stat_llimpl, llfakeimpl=posix_fakeimpl) - else: - # See Win32 implementation below - return extdef( - [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(globals()['win32_%s_llimpl' % (name,)], - 'os_%s_llimpl' % (name,)), - ) - -def register_stat_variant_unicode(name): - assert name in ('stat', 'lstat') - - # See Win32 implementation below return extdef( - [unicode], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(globals()['win32_wstat_llimpl'], - 'os_wstat_llimpl') - ) + [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), + llimpl=posix_stat_llimpl, llfakeimpl=posix_fakeimpl) + +def make_stat_impl(name, traits): + from pypy.rlib import rwin32 + from pypy.rpython.module.ll_win32file import make_win32_traits + win32traits = make_win32_traits(traits) -# ____________________________________________________________ -if sys.platform == 'win32': # The CRT of Windows has a number of flaws wrt. its stat() implementation: # - time stamps are restricted to second resolution # - file modification times suffer from forth-and-back conversions between @@ -309,144 +298,18 @@ assert len(STAT_FIELDS) == 10 # no extra fields on Windows - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h', 'winbase.h', 'sys/stat.h'], - ) - - GetFileExInfoStandard = platform.ConstantInteger( - 'GetFileExInfoStandard') - FILE_ATTRIBUTE_DIRECTORY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_DIRECTORY') - FILE_ATTRIBUTE_READONLY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_READONLY') - ERROR_SHARING_VIOLATION = platform.ConstantInteger( - 'ERROR_SHARING_VIOLATION') - _S_IFDIR = platform.ConstantInteger('_S_IFDIR') - _S_IFREG = platform.ConstantInteger('_S_IFREG') - _S_IFCHR = platform.ConstantInteger('_S_IFCHR') - _S_IFIFO = platform.ConstantInteger('_S_IFIFO') - FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN') - FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') - FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') - - WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( - 'WIN32_FILE_ATTRIBUTE_DATA', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - BY_HANDLE_FILE_INFORMATION = platform.Struct( - 'BY_HANDLE_FILE_INFORMATION', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('nNumberOfLinks', rwin32.DWORD), - ('nFileIndexHigh', rwin32.DWORD), - ('nFileIndexLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - WIN32_FIND_DATAA = platform.Struct( - 'WIN32_FIND_DATAA', - # Only interesting fields - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - WIN32_FIND_DATAW = platform.Struct( - 'WIN32_FIND_DATAW', - # Only interesting fields - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - globals().update(platform.configure(CConfig)) - GET_FILEEX_INFO_LEVELS = rffi.ULONG # an enumeration - - GetFileAttributesExA = rffi.llexternal( - 'GetFileAttributesExA', - [rffi.CCHARP, GET_FILEEX_INFO_LEVELS, - lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)], - rwin32.BOOL, - calling_conv='win') - - GetFileAttributesExW = rffi.llexternal( - 'GetFileAttributesExW', - [rffi.CWCHARP, GET_FILEEX_INFO_LEVELS, - lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)], - rwin32.BOOL, - calling_conv='win') - - GetFileInformationByHandle = rffi.llexternal( - 'GetFileInformationByHandle', - [rwin32.HANDLE, lltype.Ptr(BY_HANDLE_FILE_INFORMATION)], - rwin32.BOOL, - calling_conv='win') - - GetFileType = rffi.llexternal( - 'GetFileType', - [rwin32.HANDLE], - rwin32.DWORD, - calling_conv='win') - - FindFirstFileA = rffi.llexternal( - 'FindFirstFileA', - [rffi.CCHARP, lltype.Ptr(WIN32_FIND_DATAA)], - rwin32.HANDLE, - calling_conv='win') - - FindFirstFileW = rffi.llexternal( - 'FindFirstFileW', - [rffi.CWCHARP, lltype.Ptr(WIN32_FIND_DATAW)], - rwin32.HANDLE, - calling_conv='win') - - FindClose = rffi.llexternal( - 'FindClose', - [rwin32.HANDLE], - rwin32.BOOL, - calling_conv='win') - def attributes_to_mode(attributes): m = 0 - if attributes & FILE_ATTRIBUTE_DIRECTORY: - m |= _S_IFDIR | 0111 # IFEXEC for user,group,other + if attributes & win32traits.FILE_ATTRIBUTE_DIRECTORY: + m |= win32traits._S_IFDIR | 0111 # IFEXEC for user,group,other else: - m |= _S_IFREG - if attributes & FILE_ATTRIBUTE_READONLY: + m |= win32traits._S_IFREG + if attributes & win32traits.FILE_ATTRIBUTE_READONLY: m |= 0444 else: m |= 0666 return m - def make_longlong(high, low): - return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) - - # Seconds between 1.1.1601 and 1.1.1970 - secs_between_epochs = lltype.r_longlong(11644473600) - - def FILE_TIME_to_time_t_nsec(filetime): - ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) - # FILETIME is in units of 100 nsec - nsec = (ft % 10000000) * 100 - time = (ft / 10000000) - secs_between_epochs - return time, nsec - - def time_t_to_FILE_TIME(time, filetime): - ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) - filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) - filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) - def attribute_data_to_stat(info): st_mode = attributes_to_mode(info.c_dwFileAttributes) st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) @@ -484,22 +347,13 @@ return make_stat_result(result) - @specialize.ll() - def findfirstfile(l_path): - if lltype.typeOf(l_path) == rffi.CWCHARP: - filedata = lltype.malloc(WIN32_FIND_DATAW, flavor='raw') - return FindFirstFileW(l_path, filedata), filedata - else: - filedata = lltype.malloc(WIN32_FIND_DATAA, flavor='raw') - return FindFirstFileA(l_path, filedata), filedata - - @specialize.ll() def attributes_from_dir(l_path, data): - hFindFile, filedata = findfirstfile(l_path) + filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') try: + hFindFile = win32traits.FindFirstFile(l_path, filedata) if hFindFile == rwin32.INVALID_HANDLE_VALUE: return 0 - FindClose(hFindFile) + win32traits.FindClose(hFindFile) data.c_dwFileAttributes = filedata.c_dwFileAttributes rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) @@ -511,34 +365,16 @@ lltype.free(filedata, flavor='raw') def win32_stat_llimpl(path): - data = lltype.malloc(WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') + data = lltype.malloc(win32traits.WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') try: - l_path = rffi.str2charp(path) - res = GetFileAttributesExA(l_path, GetFileExInfoStandard, data) + l_path = traits.str2charp(path) + res = win32traits.GetFileAttributesEx(l_path, win32traits.GetFileExInfoStandard, data) errcode = rwin32.GetLastError() if res == 0: - if errcode == ERROR_SHARING_VIOLATION: + if errcode == win32traits.ERROR_SHARING_VIOLATION: res = attributes_from_dir(l_path, data) errcode = rwin32.GetLastError() - rffi.free_charp(l_path) - if res == 0: - raise WindowsError(errcode, "os_stat failed") - return attribute_data_to_stat(data) - finally: - lltype.free(data, flavor='raw') - win32_lstat_llimpl = win32_stat_llimpl - - def win32_wstat_llimpl(path): - data = lltype.malloc(WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') - try: - l_path = rffi.unicode2wcharp(path) - res = GetFileAttributesExW(l_path, GetFileExInfoStandard, data) - errcode = rwin32.GetLastError() - if res == 0: - if errcode == ERROR_SHARING_VIOLATION: - res = attributes_from_dir(l_path, data) - errcode = rwin32.GetLastError() - rffi.free_wcharp(l_path) + traits.free_charp(l_path) if res == 0: raise WindowsError(errcode, "os_stat failed") return attribute_data_to_stat(data) @@ -548,31 +384,58 @@ def win32_fstat_llimpl(fd): handle = rwin32._get_osfhandle(fd) - filetype = GetFileType(handle) - if filetype == FILE_TYPE_CHAR: + filetype = win32traits.GetFileType(handle) + if filetype == win32traits.FILE_TYPE_CHAR: # console or LPT device - return make_stat_result((_S_IFCHR, + return make_stat_result((win32traits._S_IFCHR, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_PIPE: + elif filetype == win32traits.FILE_TYPE_PIPE: # socket or named pipe - return make_stat_result((_S_IFIFO, + return make_stat_result((win32traits._S_IFIFO, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_UNKNOWN: + elif filetype == win32traits.FILE_TYPE_UNKNOWN: error = rwin32.GetLastError() if error != 0: raise WindowsError(error, "os_fstat failed") # else: unknown but valid file # normal disk file (FILE_TYPE_DISK) - info = lltype.malloc(BY_HANDLE_FILE_INFORMATION, flavor='raw', - zero=True) + info = lltype.malloc(win32traits.BY_HANDLE_FILE_INFORMATION, + flavor='raw', zero=True) try: - res = GetFileInformationByHandle(handle, info) + res = win32traits.GetFileInformationByHandle(handle, info) if res == 0: raise WindowsError(rwin32.GetLastError(), "os_fstat failed") return by_handle_info_to_stat(info) finally: lltype.free(info, flavor='raw') + if name == 'fstat': + return win32_fstat_llimpl + else: + return win32_stat_llimpl + + +#__________________________________________________ +# Helper functions for win32 + +def make_longlong(high, low): + return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) + +# Seconds between 1.1.1601 and 1.1.1970 +secs_between_epochs = lltype.r_longlong(11644473600) + +def FILE_TIME_to_time_t_nsec(filetime): + ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) + # FILETIME is in units of 100 nsec + nsec = (ft % 10000000) * 100 + time = (ft / 10000000) - secs_between_epochs + return time, nsec + +def time_t_to_FILE_TIME(time, filetime): + ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) + filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) + filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) + Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py Fri Jul 16 17:31:39 2010 @@ -18,7 +18,7 @@ class CConfig: _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'] + includes = ['windows.h', 'winbase.h', 'sys/stat.h'], ) WIN32_FIND_DATA = platform.Struct( 'struct _WIN32_FIND_DATA' + suffix, @@ -35,17 +35,63 @@ ERROR_NO_MORE_FILES = platform.ConstantInteger( 'ERROR_NO_MORE_FILES') - def external(*args, **kwargs): - kwargs['compilation_info'] = CConfig._compilation_info_ - return rffi.llexternal(*args, **kwargs) + GetFileExInfoStandard = platform.ConstantInteger( + 'GetFileExInfoStandard') + FILE_ATTRIBUTE_DIRECTORY = platform.ConstantInteger( + 'FILE_ATTRIBUTE_DIRECTORY') + FILE_ATTRIBUTE_READONLY = platform.ConstantInteger( + 'FILE_ATTRIBUTE_READONLY') + ERROR_SHARING_VIOLATION = platform.ConstantInteger( + 'ERROR_SHARING_VIOLATION') + _S_IFDIR = platform.ConstantInteger('_S_IFDIR') + _S_IFREG = platform.ConstantInteger('_S_IFREG') + _S_IFCHR = platform.ConstantInteger('_S_IFCHR') + _S_IFIFO = platform.ConstantInteger('_S_IFIFO') + FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN') + FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') + FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') + + WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( + 'WIN32_FILE_ATTRIBUTE_DATA', + [('dwFileAttributes', rwin32.DWORD), + ('nFileSizeHigh', rwin32.DWORD), + ('nFileSizeLow', rwin32.DWORD), + ('ftCreationTime', rwin32.FILETIME), + ('ftLastAccessTime', rwin32.FILETIME), + ('ftLastWriteTime', rwin32.FILETIME)]) + + BY_HANDLE_FILE_INFORMATION = platform.Struct( + 'BY_HANDLE_FILE_INFORMATION', + [('dwFileAttributes', rwin32.DWORD), + ('nFileSizeHigh', rwin32.DWORD), + ('nFileSizeLow', rwin32.DWORD), + ('nNumberOfLinks', rwin32.DWORD), + ('nFileIndexHigh', rwin32.DWORD), + ('nFileIndexLow', rwin32.DWORD), + ('ftCreationTime', rwin32.FILETIME), + ('ftLastAccessTime', rwin32.FILETIME), + ('ftLastWriteTime', rwin32.FILETIME)]) config = platform.configure(CConfig) + def external(*args, **kwargs): + kwargs['compilation_info'] = CConfig._compilation_info_ + return rffi.llexternal(calling_conv='win', *args, **kwargs) + class Win32Traits: - WIN32_FIND_DATA = config['WIN32_FIND_DATA'] - ERROR_FILE_NOT_FOUND = config['ERROR_FILE_NOT_FOUND'] - ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] + apisuffix = suffix + + for name in '''WIN32_FIND_DATA WIN32_FILE_ATTRIBUTE_DATA BY_HANDLE_FILE_INFORMATION + GetFileExInfoStandard + FILE_ATTRIBUTE_DIRECTORY FILE_ATTRIBUTE_READONLY + _S_IFDIR _S_IFREG _S_IFCHR _S_IFIFO + FILE_TYPE_UNKNOWN FILE_TYPE_CHAR FILE_TYPE_PIPE + ERROR_FILE_NOT_FOUND ERROR_NO_MORE_FILES + ERROR_SHARING_VIOLATION + '''.split(): + locals()[name] = config[name] LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) + GET_FILEEX_INFO_LEVELS = rffi.ULONG # an enumeration FindFirstFile = external('FindFirstFile' + suffix, [traits.CCHARP, LPWIN32_FIND_DATA], @@ -57,8 +103,35 @@ [rwin32.HANDLE], rwin32.BOOL) + GetFileAttributesEx = external( + 'GetFileAttributesEx' + suffix, + [traits.CCHARP, GET_FILEEX_INFO_LEVELS, + lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)], + rwin32.BOOL) + + GetFileInformationByHandle = external( + 'GetFileInformationByHandle', + [rwin32.HANDLE, lltype.Ptr(BY_HANDLE_FILE_INFORMATION)], + rwin32.BOOL) + + GetFileType = external( + 'GetFileType', + [rwin32.HANDLE], + rwin32.DWORD) + + LPSTRP = rffi.CArrayPtr(traits.CCHARP) + + GetFullPathName = external( + 'GetFullPathName' + suffix, + [traits.CCHARP, rwin32.DWORD, + traits.CCHARP, LPSTRP], + rwin32.DWORD) + return Win32Traits +#_______________________________________________________________ +# listdir + def make_listdir_impl(traits): from pypy.rlib import rwin32 win32traits = make_win32_traits(traits) @@ -113,20 +186,21 @@ return listdir_llimpl +#_______________________________________________________________ +# getfullpathname + def make_getfullpathname_impl(traits): + from pypy.rlib import rwin32 win32traits = make_win32_traits(traits) - LPSTRP = rffi.CArrayPtr(traits.CCHARP) - @func_renamer('getfullpathname_llimpl_%s' % traits.str.__name__) def getfullpathname_llimpl(path): - # XXX why do we ignore WINAPI conventions everywhere? nBufferLength = rwin32.MAX_PATH + 1 lpBuffer = lltype.malloc(traits.CCHARP.TO, nBufferLength, flavor='raw') try: res = win32traits.GetFullPathName( - lpFileName, rffi.cast(rwin32.DWORD, nBufferLength), - lpBuffer, lltype.nullptr(LPSTRP.TO)) + path, rffi.cast(rwin32.DWORD, nBufferLength), + lpBuffer, lltype.nullptr(win32traits.LPSTRP.TO)) if res == 0: raise rwin32.lastWindowsError("_getfullpathname failed") result = traits.charp2str(lpBuffer) From afa at codespeak.net Fri Jul 16 17:52:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Jul 2010 17:52:43 +0200 (CEST) Subject: [pypy-svn] r76263 - pypy/branch/unicode_filename-2/pypy/rpython/module Message-ID: <20100716155243.42954282BD4@codespeak.net> Author: afa Date: Fri Jul 16 17:52:41 2010 New Revision: 76263 Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py Log: Dramatically reduce code duplication for os.utime() Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_os.py Fri Jul 16 17:52:41 2010 @@ -34,6 +34,8 @@ def monkeypatch_rposix(posixfunc, unicodefunc, signature): func_name = posixfunc.__name__ + if hasattr(signature, '_default_signature_'): + signature = signature._default_signature_ arglist = ['arg%d' % (i,) for i in range(len(signature))] transformed_arglist = arglist[:] for i, arg in enumerate(signature): @@ -64,26 +66,6 @@ # Monkeypatch the function in pypy.rlib.rposix setattr(rposix, func_name, new_func) -def registering_unicode_version(posixfunc, signature, condition=True): - """ - Registers an implementation of posixfunc() which directly accepts unicode - strings. Replaces the corresponding function in pypy.rlib.rposix. - @signature: A list of types. 'unicode' will trigger encoding when needed, - 'None' will use specialize.argtype. - """ - if not condition: - return registering(None, condition=False) - - func_name = posixfunc.__name__ - - @func_renamer(func_name + "_unicode") - def unicodefunc(*args): - return posixfunc(*args) - - monkeypatch_rposix(posixfunc, unicodefunc, signature) - - return registering(unicodefunc, condition=condition) - class StringTraits: str = str CHAR = rffi.CHAR @@ -395,8 +377,8 @@ return extdef([int, int], s_None, llimpl=dup2_llimpl, export_name="ll_os.ll_os_dup2") - @registering(os.utime) - def register_os_utime(self): + @registering_str_unicode(os.utime) + def register_os_utime(self, traits): UTIMBUFP = lltype.Ptr(self.UTIMBUF) os_utime = self.llexternal('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT) @@ -449,6 +431,9 @@ # tp is known to be None, and one version where it is known # to be a tuple of 2 floats. if not _WIN32: + assert traits.str is str + + @specialize.argtype(1) def os_utime_llimpl(path, tp): if tp is None: error = os_utime(path, lltype.nullptr(UTIMBUFP.TO)) @@ -459,192 +444,13 @@ if error == -1: raise OSError(rposix.get_errno(), "os_utime failed") else: - from pypy.rlib import rwin32 - from pypy.rpython.module.ll_os_stat import time_t_to_FILE_TIME - - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'], - ) - - FILE_WRITE_ATTRIBUTES = platform.ConstantInteger( - 'FILE_WRITE_ATTRIBUTES') - OPEN_EXISTING = platform.ConstantInteger( - 'OPEN_EXISTING') - FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger( - 'FILE_FLAG_BACKUP_SEMANTICS') - globals().update(platform.configure(CConfig)) - - CreateFile = rffi.llexternal( - 'CreateFileA', - [rwin32.LPCSTR, rwin32.DWORD, rwin32.DWORD, - rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD, - rwin32.HANDLE], - rwin32.HANDLE, - calling_conv='win') - - GetSystemTime = rffi.llexternal( - 'GetSystemTime', - [lltype.Ptr(rwin32.SYSTEMTIME)], - lltype.Void, - calling_conv='win') - - SystemTimeToFileTime = rffi.llexternal( - 'SystemTimeToFileTime', - [lltype.Ptr(rwin32.SYSTEMTIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv='win') - - SetFileTime = rffi.llexternal( - 'SetFileTime', - [rwin32.HANDLE, - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv = 'win') - - def os_utime_llimpl(path, tp): - hFile = CreateFile(path, - FILE_WRITE_ATTRIBUTES, 0, - None, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, 0) - if hFile == rwin32.INVALID_HANDLE_VALUE: - raise rwin32.lastWindowsError() - ctime = lltype.nullptr(rwin32.FILETIME) - atime = lltype.malloc(rwin32.FILETIME, flavor='raw') - mtime = lltype.malloc(rwin32.FILETIME, flavor='raw') - try: - if tp is None: - now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw') - try: - GetSystemTime(now) - if (not SystemTimeToFileTime(now, atime) or - not SystemTimeToFileTime(now, mtime)): - raise rwin32.lastWindowsError() - finally: - lltype.free(now, flavor='raw') - else: - actime, modtime = tp - time_t_to_FILE_TIME(actime, atime) - time_t_to_FILE_TIME(modtime, mtime) - if not SetFileTime(hFile, ctime, atime, mtime): - raise rwin32.lastWindowsError() - finally: - rwin32.CloseHandle(hFile) - lltype.free(atime, flavor='raw') - lltype.free(mtime, flavor='raw') - os_utime_llimpl._annspecialcase_ = 'specialize:argtype(1)' - - s_string = SomeString() - s_tuple_of_2_floats = SomeTuple([SomeFloat(), SomeFloat()]) - - def os_utime_normalize_args(s_path, s_times): - # special handling of the arguments: they can be either - # [str, (float, float)] or [str, s_None], and get normalized - # to exactly one of these two. - if not s_string.contains(s_path): - raise Exception("os.utime() arg 1 must be a string, got %s" % ( - s_path,)) - case1 = s_None.contains(s_times) - case2 = s_tuple_of_2_floats.contains(s_times) - if case1 and case2: - return [s_string, s_ImpossibleValue] #don't know which case yet - elif case1: - return [s_string, s_None] - elif case2: - return [s_string, s_tuple_of_2_floats] - else: - raise Exception("os.utime() arg 2 must be None or a tuple of " - "2 floats, got %s" % (s_times,)) - - return extdef(os_utime_normalize_args, s_None, - "ll_os.ll_os_utime", - llimpl=os_utime_llimpl) - - # XXX LOT OF DUPLICATED CODE! - @registering_unicode_version(os.utime, [unicode, None], sys.platform=='win32') - def register_os_utime_unicode(self): - from pypy.rlib import rwin32 - from pypy.rpython.module.ll_os_stat import time_t_to_FILE_TIME - - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'], - ) + from pypy.rpython.module.ll_win32file import make_utime_impl + os_utime_llimpl = make_utime_impl(traits) - FILE_WRITE_ATTRIBUTES = platform.ConstantInteger( - 'FILE_WRITE_ATTRIBUTES') - OPEN_EXISTING = platform.ConstantInteger( - 'OPEN_EXISTING') - FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger( - 'FILE_FLAG_BACKUP_SEMANTICS') - globals().update(platform.configure(CConfig)) - - CreateFile = rffi.llexternal( - 'CreateFileW', - [rwin32.LPCWSTR, rwin32.DWORD, rwin32.DWORD, - rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD, - rwin32.HANDLE], - rwin32.HANDLE, - calling_conv='win') - - GetSystemTime = rffi.llexternal( - 'GetSystemTime', - [lltype.Ptr(rwin32.SYSTEMTIME)], - lltype.Void, - calling_conv='win') - - SystemTimeToFileTime = rffi.llexternal( - 'SystemTimeToFileTime', - [lltype.Ptr(rwin32.SYSTEMTIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv='win') - - SetFileTime = rffi.llexternal( - 'SetFileTime', - [rwin32.HANDLE, - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv = 'win') - - def os_utime_llimpl(path, tp): - hFile = CreateFile(path, - FILE_WRITE_ATTRIBUTES, 0, - None, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, 0) - if hFile == rwin32.INVALID_HANDLE_VALUE: - raise rwin32.lastWindowsError() - ctime = lltype.nullptr(rwin32.FILETIME) - atime = lltype.malloc(rwin32.FILETIME, flavor='raw') - mtime = lltype.malloc(rwin32.FILETIME, flavor='raw') - try: - if tp is None: - now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw') - try: - GetSystemTime(now) - if (not SystemTimeToFileTime(now, atime) or - not SystemTimeToFileTime(now, mtime)): - raise rwin32.lastWindowsError() - finally: - lltype.free(now, flavor='raw') - else: - actime, modtime = tp - time_t_to_FILE_TIME(actime, atime) - time_t_to_FILE_TIME(modtime, mtime) - if not SetFileTime(hFile, ctime, atime, mtime): - raise rwin32.lastWindowsError() - finally: - rwin32.CloseHandle(hFile) - lltype.free(atime, flavor='raw') - lltype.free(mtime, flavor='raw') - os_utime_llimpl._annspecialcase_ = 'specialize:argtype(1)' - - s_string = SomeUnicodeString() + if traits.str is str: + s_string = SomeString() + else: + s_string = SomeUnicodeString() s_tuple_of_2_floats = SomeTuple([SomeFloat(), SomeFloat()]) def os_utime_normalize_args(s_path, s_times): @@ -665,6 +471,7 @@ else: raise Exception("os.utime() arg 2 must be None or a tuple of " "2 floats, got %s" % (s_times,)) + os_utime_normalize_args._default_signature_ = [traits.str, None] return extdef(os_utime_normalize_args, s_None, "ll_os.ll_os_utime", Modified: pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py (original) +++ pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py Fri Jul 16 17:52:41 2010 @@ -7,6 +7,7 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.tool import rffi_platform as platform from pypy.tool.sourcetools import func_renamer +from pypy.rlib.objectmodel import specialize def make_win32_traits(traits): from pypy.rlib import rwin32 @@ -209,3 +210,85 @@ lltype.free(lpBuffer, flavor='raw') return getfullpathname_llimpl + +def make_utime_impl(traits): + from pypy.rlib import rwin32 + win32traits = make_win32_traits(traits) + from pypy.rpython.module.ll_os_stat import time_t_to_FILE_TIME + + class CConfig: + _compilation_info_ = ExternalCompilationInfo( + includes = ['windows.h'], + ) + + FILE_WRITE_ATTRIBUTES = platform.ConstantInteger( + 'FILE_WRITE_ATTRIBUTES') + OPEN_EXISTING = platform.ConstantInteger( + 'OPEN_EXISTING') + FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger( + 'FILE_FLAG_BACKUP_SEMANTICS') + globals().update(platform.configure(CConfig)) + + CreateFile = rffi.llexternal( + 'CreateFile' + win32traits.apisuffix, + [traits.CCHARP, rwin32.DWORD, rwin32.DWORD, + rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD, + rwin32.HANDLE], + rwin32.HANDLE, + calling_conv='win') + + GetSystemTime = rffi.llexternal( + 'GetSystemTime', + [lltype.Ptr(rwin32.SYSTEMTIME)], + lltype.Void, + calling_conv='win') + + SystemTimeToFileTime = rffi.llexternal( + 'SystemTimeToFileTime', + [lltype.Ptr(rwin32.SYSTEMTIME), + lltype.Ptr(rwin32.FILETIME)], + rwin32.BOOL, + calling_conv='win') + + SetFileTime = rffi.llexternal( + 'SetFileTime', + [rwin32.HANDLE, + lltype.Ptr(rwin32.FILETIME), + lltype.Ptr(rwin32.FILETIME), + lltype.Ptr(rwin32.FILETIME)], + rwin32.BOOL, + calling_conv = 'win') + + @specialize.argtype(1) + def os_utime_llimpl(path, tp): + hFile = CreateFile(path, + FILE_WRITE_ATTRIBUTES, 0, + None, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, 0) + if hFile == rwin32.INVALID_HANDLE_VALUE: + raise rwin32.lastWindowsError() + ctime = lltype.nullptr(rwin32.FILETIME) + atime = lltype.malloc(rwin32.FILETIME, flavor='raw') + mtime = lltype.malloc(rwin32.FILETIME, flavor='raw') + try: + if tp is None: + now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw') + try: + GetSystemTime(now) + if (not SystemTimeToFileTime(now, atime) or + not SystemTimeToFileTime(now, mtime)): + raise rwin32.lastWindowsError() + finally: + lltype.free(now, flavor='raw') + else: + actime, modtime = tp + time_t_to_FILE_TIME(actime, atime) + time_t_to_FILE_TIME(modtime, mtime) + if not SetFileTime(hFile, ctime, atime, mtime): + raise rwin32.lastWindowsError() + finally: + rwin32.CloseHandle(hFile) + lltype.free(atime, flavor='raw') + lltype.free(mtime, flavor='raw') + + return os_utime_llimpl From afa at codespeak.net Fri Jul 16 21:42:11 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Jul 2010 21:42:11 +0200 (CEST) Subject: [pypy-svn] r76264 - pypy/branch/unicode_filename-2/pypy/module/posix Message-ID: <20100716194211.31916282BD4@codespeak.net> Author: afa Date: Fri Jul 16 21:42:07 2010 New Revision: 76264 Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Log: space.sys.filesystemencoding is None on startup; always call the function that may compute it. Modified: pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/interp_posix.py Fri Jul 16 21:42:07 2010 @@ -19,7 +19,11 @@ self.w_obj = w_obj def as_bytes(self): - return self.space.path_w(self.w_obj) + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_bytes = space.call_method(self.w_obj, 'encode', + getfilesystemencoding(space)) + return space.str_w(w_bytes) def as_unicode(self): return self.space.unicode_w(self.w_obj) @@ -30,13 +34,13 @@ self.w_obj = w_obj def as_bytes(self): - return self.space.path_w(self.w_obj) + return self.space.str_w(self.w_obj) def as_unicode(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding space = self.space - filesystemencoding = space.sys.filesystemencoding w_unicode = space.call_method(self.w_obj, 'decode', - space.wrap(filesystemencoding)) + getfilesystemencoding(space)) return space.unicode_w(w_unicode) @specialize.memo() From afa at codespeak.net Fri Jul 16 21:51:23 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Jul 2010 21:51:23 +0200 (CEST) Subject: [pypy-svn] r76265 - in pypy/branch/unicode_filename-2/pypy/interpreter: . test Message-ID: <20100716195123.7FD47282BD4@codespeak.net> Author: afa Date: Fri Jul 16 21:51:22 2010 New Revision: 76265 Modified: pypy/branch/unicode_filename-2/pypy/interpreter/baseobjspace.py pypy/branch/unicode_filename-2/pypy/interpreter/gateway.py pypy/branch/unicode_filename-2/pypy/interpreter/test/test_gateway.py Log: Remove space.path_w() and the 'path' unwrap_spec: it's wrong to unconditionally encode file names (the conversion loses characters on Windows) a better logic is implemented by interp_posix. Modified: pypy/branch/unicode_filename-2/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/unicode_filename-2/pypy/interpreter/baseobjspace.py Fri Jul 16 21:51:22 2010 @@ -1102,17 +1102,6 @@ self.wrap('argument must be a unicode')) return self.unicode_w(w_obj) - def path_w(self, w_obj): - """ Like str_w, but if the object is unicode, encode it using - filesystemencoding - """ - filesystemencoding = self.sys.filesystemencoding - if (filesystemencoding and - self.is_true(self.isinstance(w_obj, self.w_unicode))): - w_obj = self.call_method(w_obj, "encode", - self.wrap(filesystemencoding)) - return self.str_w(w_obj) - def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. # This is here mostly just for gateway.int_unwrapping_space_method(). Modified: pypy/branch/unicode_filename-2/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/interpreter/gateway.py (original) +++ pypy/branch/unicode_filename-2/pypy/interpreter/gateway.py Fri Jul 16 21:51:22 2010 @@ -137,9 +137,6 @@ def visit_c_nonnegint(self, el, app_sig): self.checked_space_method(el, app_sig) - def visit_path(self, el, app_sig): - self.checked_space_method(el, app_sig) - def visit__Wrappable(self, el, app_sig): name = el.__name__ argname = self.orig_arg() @@ -241,9 +238,6 @@ def visit_bufferstr(self, typ): self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),)) - def visit_path(self, typ): - self.run_args.append("space.path_w(%s)" % (self.scopenext(),)) - def visit_nonnegint(self, typ): self.run_args.append("space.nonnegint_w(%s)" % (self.scopenext(),)) @@ -371,9 +365,6 @@ def visit_bufferstr(self, typ): self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),)) - def visit_path(self, typ): - self.unwrap.append("space.path_w(%s)" % (self.nextarg(),)) - def visit_nonnegint(self, typ): self.unwrap.append("space.nonnegint_w(%s)" % (self.nextarg(),)) Modified: pypy/branch/unicode_filename-2/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/unicode_filename-2/pypy/interpreter/test/test_gateway.py Fri Jul 16 21:51:22 2010 @@ -454,16 +454,6 @@ assert len(l) == 1 assert space.eq_w(l[0], w("foo")) - def test_interp2app_unwrap_spec_path(self, monkeypatch): - space = self.space - def g(space, p): - return p - - app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'path']) - w_app_g = space.wrap(app_g) - monkeypatch.setattr(space.sys, "filesystemencoding", "utf-8") - w_res = space.call_function(w_app_g, space.wrap(u"?")) - def test_interp2app_classmethod(self): space = self.space w = space.wrap From afa at codespeak.net Fri Jul 16 22:39:41 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Jul 2010 22:39:41 +0200 (CEST) Subject: [pypy-svn] r76266 - in pypy/branch/unicode_filename-2/pypy: module/_file/test module/posix/test rlib/test Message-ID: <20100716203941.EF1EF282BD4@codespeak.net> Author: afa Date: Fri Jul 16 22:39:39 2010 New Revision: 76266 Modified: pypy/branch/unicode_filename-2/pypy/module/_file/test/test_file.py pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Log: on tannit32, sys.getfilesystemencoding() is 'ascii' :-( Skip tests when the filename cannot be encoded. Modified: pypy/branch/unicode_filename-2/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/_file/test/test_file.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/_file/test/test_file.py Fri Jul 16 22:39:39 2010 @@ -126,6 +126,11 @@ f.close() def test_unicode_filename(self): + import sys + try: + u'\xe9'.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + skip("encoding not good enough") f = self.file(self.temppath + u'\xe9', "w") f.close() Modified: pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/unicode_filename-2/pypy/module/posix/test/test_posix2.py Fri Jul 16 22:39:39 2010 @@ -713,7 +713,10 @@ def setup_class(cls): ufilename = (unicode(udir.join('test_unicode_filename_')) + u'\u65e5\u672c.txt') # "Japan" - f = file(ufilename, 'w') + try: + f = file(ufilename, 'w') + except UnicodeEncodeError: + py.test.skip("encoding not good enough") f.write("test") f.close() cls.space = space Modified: pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py Fri Jul 16 22:39:39 2010 @@ -2,6 +2,7 @@ from pypy.tool.udir import udir from pypy.rlib import rposix import os, sys +import py def ll_to_string(s): return ''.join(s.chars) @@ -28,7 +29,10 @@ def setup_method(self, method): self.ufilename = (unicode(udir.join('test_open')) + u'\u65e5\u672c.txt') # "Japan" - f = file(self.ufilename, 'w') + try: + f = file(self.ufilename, 'w') + except UnicodeEncodeError: + py.test.skip("encoding not good enough") f.write("test") f.close() From afa at codespeak.net Fri Jul 16 23:46:25 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Jul 2010 23:46:25 +0200 (CEST) Subject: [pypy-svn] r76267 - in pypy/trunk/pypy: interpreter interpreter/test module/_file module/_file/test module/posix module/posix/test rlib rlib/test rpython rpython/lltypesystem rpython/module rpython/module/test rpython/test Message-ID: <20100716214625.8DC91282BD4@codespeak.net> Author: afa Date: Fri Jul 16 23:46:23 2010 New Revision: 76267 Added: pypy/trunk/pypy/rlib/test/test_rposix.py - copied, changed from r76265, pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py pypy/trunk/pypy/rpython/module/ll_win32file.py - copied, changed from r76265, pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py Modified: pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/error.py pypy/trunk/pypy/interpreter/gateway.py pypy/trunk/pypy/interpreter/test/test_gateway.py pypy/trunk/pypy/module/_file/interp_file.py pypy/trunk/pypy/module/_file/test/test_file.py pypy/trunk/pypy/module/posix/interp_posix.py pypy/trunk/pypy/module/posix/test/test_posix2.py pypy/trunk/pypy/rlib/rposix.py pypy/trunk/pypy/rlib/rwin32.py pypy/trunk/pypy/rlib/streamio.py pypy/trunk/pypy/rpython/extfunc.py pypy/trunk/pypy/rpython/lltypesystem/rffi.py pypy/trunk/pypy/rpython/module/ll_os.py pypy/trunk/pypy/rpython/module/ll_os_stat.py pypy/trunk/pypy/rpython/module/test/test_ll_os_stat.py pypy/trunk/pypy/rpython/test/test_extfunc.py Log: Merge branch/unicode_filename-2 In general, it's wrong to encode filenames with sys.filesystemencoding; on Windows for example, the 'mbcs' codec silently loose characters. This change lets the backends provide unicode versions of posix functions; when they do, the unicode string is passed untranslated. When they don't, the encoding passed by the posix module is used and the string function is called. Use this extensively on Windows, which provides a "wide" version for every function (stat -> wstat, FindFirstFile -> FindFirstFileW). This fixes test_pep277 and test_unicode_file. This could also be used by ootype backends (jvm and cli typically handle unicode better than byte strings) but they seem completely broken at the moment. Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Fri Jul 16 23:46:23 2010 @@ -1102,17 +1102,6 @@ self.wrap('argument must be a unicode')) return self.unicode_w(w_obj) - def path_w(self, w_obj): - """ Like str_w, but if the object is unicode, encode it using - filesystemencoding - """ - filesystemencoding = self.sys.filesystemencoding - if (filesystemencoding and - self.is_true(self.isinstance(w_obj, self.w_unicode))): - w_obj = self.call_method(w_obj, "encode", - self.wrap(filesystemencoding)) - return self.str_w(w_obj) - def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. # This is here mostly just for gateway.int_unwrapping_space_method(). Modified: pypy/trunk/pypy/interpreter/error.py ============================================================================== --- pypy/trunk/pypy/interpreter/error.py (original) +++ pypy/trunk/pypy/interpreter/error.py Fri Jul 16 23:46:23 2010 @@ -344,7 +344,7 @@ else: _WINDOWS = True - def wrap_windowserror(space, e, filename=None): + def wrap_windowserror(space, e, w_filename=None): from pypy.rlib import rwin32 winerror = e.winerror @@ -353,19 +353,19 @@ except ValueError: msg = 'Windows Error %d' % winerror exc = space.w_WindowsError - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(winerror), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: w_error = space.call_function(exc, space.wrap(winerror), space.wrap(msg)) return OperationError(exc, w_error) -def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): +def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError'): assert isinstance(e, OSError) if _WINDOWS and isinstance(e, WindowsError): - return wrap_windowserror(space, e, filename) + return wrap_windowserror(space, e, w_filename) errno = e.errno try: @@ -373,10 +373,21 @@ except ValueError: msg = 'error %d' % errno exc = getattr(space, exception_name) - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(errno), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: - w_error = space.call_function(exc, space.wrap(errno), space.wrap(msg)) + w_error = space.call_function(exc, space.wrap(errno), + space.wrap(msg)) return OperationError(exc, w_error) +wrap_oserror2._annspecialcase_ = 'specialize:arg(3)' + +def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): + if filename is not None: + return wrap_oserror2(space, e, space.wrap(filename), + exception_name=exception_name) + else: + return wrap_oserror2(space, e, None, + exception_name=exception_name) wrap_oserror._annspecialcase_ = 'specialize:arg(3)' + Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Fri Jul 16 23:46:23 2010 @@ -137,9 +137,6 @@ def visit_c_nonnegint(self, el, app_sig): self.checked_space_method(el, app_sig) - def visit_path(self, el, app_sig): - self.checked_space_method(el, app_sig) - def visit__Wrappable(self, el, app_sig): name = el.__name__ argname = self.orig_arg() @@ -241,9 +238,6 @@ def visit_bufferstr(self, typ): self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),)) - def visit_path(self, typ): - self.run_args.append("space.path_w(%s)" % (self.scopenext(),)) - def visit_nonnegint(self, typ): self.run_args.append("space.nonnegint_w(%s)" % (self.scopenext(),)) @@ -371,9 +365,6 @@ def visit_bufferstr(self, typ): self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),)) - def visit_path(self, typ): - self.unwrap.append("space.path_w(%s)" % (self.nextarg(),)) - def visit_nonnegint(self, typ): self.unwrap.append("space.nonnegint_w(%s)" % (self.nextarg(),)) Modified: pypy/trunk/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_gateway.py (original) +++ pypy/trunk/pypy/interpreter/test/test_gateway.py Fri Jul 16 23:46:23 2010 @@ -454,16 +454,6 @@ assert len(l) == 1 assert space.eq_w(l[0], w("foo")) - def test_interp2app_unwrap_spec_path(self, monkeypatch): - space = self.space - def g(space, p): - return p - - app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'path']) - w_app_g = space.wrap(app_g) - monkeypatch.setattr(space.sys, "filesystemencoding", "utf-8") - w_res = space.call_function(w_app_g, space.wrap(u"?")) - def test_interp2app_classmethod(self): space = self.space w = space.wrap Modified: pypy/trunk/pypy/module/_file/interp_file.py ============================================================================== --- pypy/trunk/pypy/module/_file/interp_file.py (original) +++ pypy/trunk/pypy/module/_file/interp_file.py Fri Jul 16 23:46:23 2010 @@ -4,6 +4,7 @@ from pypy.rlib.rarithmetic import r_longlong from pypy.module._file.interp_stream import W_AbstractStream from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror +from pypy.module.posix.interp_posix import dispatch_filename from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.typedef import TypeDef, GetSetProperty @@ -81,11 +82,11 @@ # file lock. They don't convert StreamErrors to OperationErrors, too. def direct___init__(self, w_name, mode='r', buffering=-1): - name = self.space.str_w(w_name) self.direct_close() self.w_name = w_name self.check_mode_ok(mode) - stream = streamio.open_file_as_stream(name, mode, buffering) + stream = dispatch_filename(streamio.open_file_as_stream)( + self.space, w_name, mode, buffering) fd = stream.try_to_find_file_descriptor() self.fdopenstream(stream, fd, mode) Modified: pypy/trunk/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/trunk/pypy/module/_file/test/test_file.py (original) +++ pypy/trunk/pypy/module/_file/test/test_file.py Fri Jul 16 23:46:23 2010 @@ -125,6 +125,15 @@ assert type(res) is str f.close() + def test_unicode_filename(self): + import sys + try: + u'\xe9'.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + skip("encoding not good enough") + f = self.file(self.temppath + u'\xe9', "w") + f.close() + def test_oserror_has_filename(self): try: f = self.file("file that is clearly not there") Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Fri Jul 16 23:46:23 2010 @@ -1,8 +1,9 @@ from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong from pypy.rlib.unroll import unrolling_iterable -from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 from pypy.rpython.module.ll_os import RegisterOs from pypy.rpython.module import ll_os_stat from pypy.rpython.lltypesystem import rffi, lltype @@ -12,15 +13,78 @@ import os, sys _WIN = sys.platform == 'win32' -def open(space, fname, flag, mode=0777): +class FileEncoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_bytes = space.call_method(self.w_obj, 'encode', + getfilesystemencoding(space)) + return space.str_w(w_bytes) + + def as_unicode(self): + return self.space.unicode_w(self.w_obj) + +class FileDecoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + return self.space.str_w(self.w_obj) + + def as_unicode(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_unicode = space.call_method(self.w_obj, 'decode', + getfilesystemencoding(space)) + return space.unicode_w(w_unicode) + + at specialize.memo() +def dispatch_filename(func, tag=0): + def dispatch(space, w_fname, *args): + if space.isinstance_w(w_fname, space.w_unicode): + fname = FileEncoder(space, w_fname) + return func(fname, *args) + else: + fname = space.str_w(w_fname) + return func(fname, *args) + return dispatch + + at specialize.memo() +def dispatch_filename_2(func): + def dispatch(space, w_fname1, w_fname2, *args): + if space.isinstance_w(w_fname1, space.w_unicode): + fname1 = FileEncoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname1 = FileDecoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + return dispatch + +def open(space, w_fname, flag, mode=0777): """Open a file (for low level IO). Return a file descriptor (a small integer).""" - try: - fd = os.open(fname, flag, mode) + try: + fd = dispatch_filename(rposix.open)( + space, w_fname, flag, mode) except OSError, e: - raise wrap_oserror(space, e, fname) + raise wrap_oserror2(space, e, w_fname) return space.wrap(fd) -open.unwrap_spec = [ObjSpace, 'path', "c_int", "c_int"] +open.unwrap_spec = [ObjSpace, W_Root, "c_int", "c_int"] def lseek(space, fd, pos, how): """Set the current position of a file descriptor. Return the new position. @@ -159,7 +223,7 @@ return build_stat_result(space, st) fstat.unwrap_spec = [ObjSpace, "c_int"] -def stat(space, path): +def stat(space, w_path): """Perform a stat system call on the given path. Return an object with (at least) the following attributes: st_mode @@ -175,22 +239,22 @@ """ try: - st = os.stat(path) + st = dispatch_filename(rposix.stat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -stat.unwrap_spec = [ObjSpace, 'path'] +stat.unwrap_spec = [ObjSpace, W_Root] -def lstat(space, path): +def lstat(space, w_path): "Like stat(path), but do no follow symbolic links." try: - st = os.lstat(path) + st = dispatch_filename(rposix.lstat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -lstat.unwrap_spec = [ObjSpace, 'path'] +lstat.unwrap_spec = [ObjSpace, W_Root] class StatState(object): def __init__(self, space): @@ -231,7 +295,7 @@ raise wrap_oserror(space, e) dup2.unwrap_spec = [ObjSpace, "c_int", "c_int"] -def access(space, path, mode): +def access(space, w_path, mode): """ access(path, mode) -> 1 if granted, 0 otherwise @@ -242,12 +306,12 @@ existence, or the inclusive-OR of R_OK, W_OK, and X_OK. """ try: - ok = os.access(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) + ok = dispatch_filename(rposix.access)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) else: return space.wrap(ok) -access.unwrap_spec = [ObjSpace, str, "c_int"] +access.unwrap_spec = [ObjSpace, W_Root, "c_int"] def times(space): @@ -278,32 +342,38 @@ return space.wrap(rc) system.unwrap_spec = [ObjSpace, str] -def unlink(space, path): +def unlink(space, w_path): """Remove a file (same as remove(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -unlink.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +unlink.unwrap_spec = [ObjSpace, W_Root] -def remove(space, path): +def remove(space, w_path): """Remove a file (same as unlink(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -remove.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +remove.unwrap_spec = [ObjSpace, W_Root] -def _getfullpathname(space, path): +def _getfullpathname(space, w_path): """helper for ntpath.abspath """ - posix = __import__(os.name) # nt specific try: - fullpath = posix._getfullpathname(path) + if space.isinstance_w(w_path, space.w_unicode): + path = FileEncoder(space, w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) + else: + path = space.str_w(w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) except OSError, e: - raise wrap_oserror(space, e, path) - else: - return space.wrap(fullpath) -_getfullpathname.unwrap_spec = [ObjSpace, str] + raise wrap_oserror2(space, e, w_path) + else: + return w_fullpath +_getfullpathname.unwrap_spec = [ObjSpace, W_Root] def getcwd(space): """Return the current working directory.""" @@ -315,35 +385,46 @@ return space.wrap(cur) getcwd.unwrap_spec = [ObjSpace] -def getcwdu(space): - """Return the current working directory as a unicode string.""" - # XXX ascii encoding for now - return space.call_method(getcwd(space), 'decode') +if sys.platform == 'win32': + def getcwdu(space): + """Return the current working directory as a unicode string.""" + try: + cur = os.getcwdu() + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(cur) +else: + def getcwdu(space): + """Return the current working directory as a unicode string.""" + filesystemencoding = space.sys.filesystemencoding + return space.call_method(getcwd(space), 'decode', + space.wrap(filesystemencoding)) getcwdu.unwrap_spec = [ObjSpace] -def chdir(space, path): +def chdir(space, w_path): """Change the current working directory to the specified path.""" try: - os.chdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -chdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.chdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chdir.unwrap_spec = [ObjSpace, W_Root] -def mkdir(space, path, mode=0777): +def mkdir(space, w_path, mode=0777): """Create a directory.""" try: - os.mkdir(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -mkdir.unwrap_spec = [ObjSpace, str, "c_int"] + dispatch_filename(rposix.mkdir)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +mkdir.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rmdir(space, path): +def rmdir(space, w_path): """Remove a directory.""" try: - os.rmdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -rmdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.rmdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +rmdir.unwrap_spec = [ObjSpace, W_Root] def strerror(space, errno): """Translate an error code to a message string.""" @@ -410,7 +491,7 @@ unsetenv.unwrap_spec = [ObjSpace, str] -def listdir(space, dirname): +def listdir(space, w_dirname): """Return a list containing the names of the entries in the directory. \tpath: path of directory to list @@ -418,12 +499,18 @@ The list is in arbitrary order. It does not include the special entries '.' and '..' even if they are present in the directory.""" try: - result = os.listdir(dirname) + if space.isinstance_w(w_dirname, space.w_unicode): + dirname = FileEncoder(space, w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] + else: + dirname = space.str_w(w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] except OSError, e: - raise wrap_oserror(space, e, dirname) - result_w = [space.wrap(s) for s in result] + raise wrap_oserror2(space, e, w_dirname) return space.newlist(result_w) -listdir.unwrap_spec = [ObjSpace, str] +listdir.unwrap_spec = [ObjSpace, W_Root] def pipe(space): "Create a pipe. Returns (read_end, write_end)." @@ -434,21 +521,21 @@ return space.newtuple([space.wrap(fd1), space.wrap(fd2)]) pipe.unwrap_spec = [ObjSpace] -def chmod(space, path, mode): +def chmod(space, w_path, mode): "Change the access permissions of a file." - try: - os.chmod(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -chmod.unwrap_spec = [ObjSpace, str, "c_int"] + try: + dispatch_filename(rposix.chmod)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chmod.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rename(space, old, new): +def rename(space, w_old, w_new): "Rename a file or directory." - try: - os.rename(old, new) - except OSError, e: + try: + dispatch_filename_2(rposix.rename)(space, w_old, w_new) + except OSError, e: raise wrap_oserror(space, e) -rename.unwrap_spec = [ObjSpace, str, str] +rename.unwrap_spec = [ObjSpace, W_Root, W_Root] def umask(space, mask): "Set the current numeric umask and return the previous umask." @@ -576,7 +663,7 @@ raise wrap_oserror(space, e) execve.unwrap_spec = [ObjSpace, str, W_Root, W_Root] -def utime(space, path, w_tuple): +def utime(space, w_path, w_tuple): """ utime(path, (atime, mtime)) utime(path, None) @@ -585,10 +672,10 @@ """ if space.is_w(w_tuple, space.w_None): try: - os.utime(path, None) + dispatch_filename(rposix.utime, 1)(space, w_path, None) return except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) try: msg = "utime() arg 2 must be a tuple (atime, mtime) or None" args_w = space.fixedview(w_tuple) @@ -596,14 +683,14 @@ raise OperationError(space.w_TypeError, space.wrap(msg)) actime = space.float_w(args_w[0]) modtime = space.float_w(args_w[1]) - os.utime(path, (actime, modtime)) + dispatch_filename(rposix.utime, 2)(space, w_path, (actime, modtime)) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) except OperationError, e: if not e.match(space, space.w_TypeError): raise raise OperationError(space.w_TypeError, space.wrap(msg)) -utime.unwrap_spec = [ObjSpace, str, W_Root] +utime.unwrap_spec = [ObjSpace, W_Root, W_Root] def setsid(space): """setsid() -> pid Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Fri Jul 16 23:46:23 2010 @@ -32,6 +32,9 @@ # even when running on top of CPython 2.4. os.stat_float_times(True) + # Initialize sys.filesystemencoding + space.call_method(space.getbuiltinmodule('sys'), 'getfilesystemencoding') + def need_sparse_files(): if sys.platform == 'darwin': py.test.skip("no sparse files on default Mac OS X file system") @@ -706,6 +709,28 @@ except OSError: pass +class AppTestUnicodeFilename: + def setup_class(cls): + ufilename = (unicode(udir.join('test_unicode_filename_')) + + u'\u65e5\u672c.txt') # "Japan" + try: + f = file(ufilename, 'w') + except UnicodeEncodeError: + py.test.skip("encoding not good enough") + f.write("test") + f.close() + cls.space = space + cls.w_filename = space.wrap(ufilename) + cls.w_posix = space.appexec([], GET_POSIX) + + def test_open(self): + fd = self.posix.open(self.filename, self.posix.O_RDONLY) + try: + content = self.posix.read(fd, 50) + finally: + self.posix.close(fd) + assert content == "test" + class TestPexpect(object): # XXX replace with AppExpectTest class as soon as possible Modified: pypy/trunk/pypy/rlib/rposix.py ============================================================================== --- pypy/trunk/pypy/rlib/rposix.py (original) +++ pypy/trunk/pypy/rlib/rposix.py Fri Jul 16 23:46:23 2010 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rlib.rarithmetic import intmask +from pypy.rlib.objectmodel import specialize class CConstantErrno(CConstant): # these accessors are used when calling get_errno() or set_errno() @@ -42,3 +43,102 @@ os.close(fd) except OSError: pass + +#___________________________________________________________________ +# Wrappers around posix functions, that accept either strings, or +# instances with a "as_bytes()" method. +# - pypy.modules.posix.interp_posix passes an object containing a unicode path +# which can encode itself with sys.filesystemencoding. +# - but pypy.rpython.module.ll_os.py on Windows will replace these functions +# with other wrappers that directly handle unicode strings. + at specialize.argtype(0) +def open(path, flags, mode): + if isinstance(path, str): + return os.open(path, flags, mode) + else: + return os.open(path.as_bytes(), flags, mode) + + at specialize.argtype(0) +def stat(path): + if isinstance(path, str): + return os.stat(path) + else: + return os.stat(path.as_bytes()) + + at specialize.argtype(0) +def lstat(path): + if isinstance(path, str): + return os.lstat(path) + else: + return os.lstat(path.as_bytes()) + + at specialize.argtype(0) +def unlink(path): + if isinstance(path, str): + return os.unlink(path) + else: + return os.unlink(path.as_bytes()) + + at specialize.argtype(0, 1) +def rename(path1, path2): + if isinstance(path1, str): + return os.rename(path1, path2) + else: + return os.rename(path1.as_bytes(), path2.as_bytes()) + + at specialize.argtype(0) +def listdir(dirname): + if isinstance(dirname, str): + return os.listdir(dirname) + else: + return os.listdir(dirname.as_bytes()) + + at specialize.argtype(0) +def access(path, mode): + if isinstance(path, str): + return os.access(path, mode) + else: + return os.access(path.as_bytes(), mode) + + at specialize.argtype(0) +def chmod(path, mode): + if isinstance(path, str): + return os.chmod(path, mode) + else: + return os.chmod(path.as_bytes(), mode) + + at specialize.argtype(0, 1) +def utime(path, times): + if isinstance(path, str): + return os.utime(path, times) + else: + return os.utime(path.as_bytes(), times) + + at specialize.argtype(0) +def chdir(path): + if isinstance(path, str): + return os.chdir(path) + else: + return os.chdir(path.as_bytes()) + + at specialize.argtype(0) +def mkdir(path, mode=0777): + if isinstance(path, str): + return os.mkdir(path, mode) + else: + return os.mkdir(path.as_bytes(), mode) + + at specialize.argtype(0) +def rmdir(path): + if isinstance(path, str): + return os.rmdir(path) + else: + return os.rmdir(path.as_bytes()) + +if os.name == 'nt': + import nt + def _getfullpathname(path): + if isinstance(path, str): + return nt._getfullpathname(path) + else: + return nt._getfullpathname(path.as_bytes()) Modified: pypy/trunk/pypy/rlib/rwin32.py ============================================================================== --- pypy/trunk/pypy/rlib/rwin32.py (original) +++ pypy/trunk/pypy/rlib/rwin32.py Fri Jul 16 23:46:23 2010 @@ -31,6 +31,7 @@ DWORD = rffi_platform.SimpleType("DWORD", rffi.UINT) BOOL = rffi_platform.SimpleType("BOOL", rffi.LONG) BYTE = rffi_platform.SimpleType("BYTE", rffi.UCHAR) + WCHAR = rffi_platform.SimpleType("WCHAR", rffi.UCHAR) INT = rffi_platform.SimpleType("INT", rffi.INT) LONG = rffi_platform.SimpleType("LONG", rffi.LONG) PLONG = rffi_platform.SimpleType("PLONG", rffi.LONGP) @@ -38,6 +39,8 @@ LPCVOID = rffi_platform.SimpleType("LPCVOID", rffi.VOIDP) LPSTR = rffi_platform.SimpleType("LPSTR", rffi.CCHARP) LPCSTR = rffi_platform.SimpleType("LPCSTR", rffi.CCHARP) + LPWSTR = rffi_platform.SimpleType("LPWSTR", rffi.CWCHARP) + LPCWSTR = rffi_platform.SimpleType("LPCWSTR", rffi.CWCHARP) LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.INTP) SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T) ULONG_PTR = rffi_platform.SimpleType("ULONG_PTR", rffi.ULONG) @@ -87,6 +90,10 @@ GetLastError = winexternal('GetLastError', [], DWORD) SetLastError = winexternal('SetLastError', [DWORD], lltype.Void) + # In tests, the first call to GetLastError is always wrong, because error + # is hidden by operations in ll2ctypes. Call it now. + GetLastError() + LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], rffi.VOIDP) GetProcAddress = winexternal('GetProcAddress', [rffi.VOIDP, rffi.CCHARP], Modified: pypy/trunk/pypy/rlib/streamio.py ============================================================================== --- pypy/trunk/pypy/rlib/streamio.py (original) +++ pypy/trunk/pypy/rlib/streamio.py Fri Jul 16 23:46:23 2010 @@ -38,7 +38,9 @@ # import os, sys +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong, intmask +from pypy.rlib import rposix from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC O_BINARY = getattr(os, "O_BINARY", 0) @@ -71,6 +73,7 @@ return s.join(string.split(c)) + at specialize.argtype(0) def open_file_as_stream(path, mode="r", buffering=-1): os_flags, universal, reading, writing, basemode, binary = decode_mode(mode) stream = open_path_helper(path, os_flags, basemode == "a") @@ -89,9 +92,10 @@ return construct_stream_tower(stream, buffering, universal, reading, writing, binary) + at specialize.argtype(0) def open_path_helper(path, os_flags, append): # XXX for now always return DiskFile - fd = os.open(path, os_flags, 0666) + fd = rposix.open(path, os_flags, 0666) if append: try: os.lseek(fd, 0, 2) Copied: pypy/trunk/pypy/rlib/test/test_rposix.py (from r76265, pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py) ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rlib/test/test_rposix.py (original) +++ pypy/trunk/pypy/rlib/test/test_rposix.py Fri Jul 16 23:46:23 2010 @@ -2,6 +2,7 @@ from pypy.tool.udir import udir from pypy.rlib import rposix import os, sys +import py def ll_to_string(s): return ''.join(s.chars) @@ -28,7 +29,10 @@ def setup_method(self, method): self.ufilename = (unicode(udir.join('test_open')) + u'\u65e5\u672c.txt') # "Japan" - f = file(self.ufilename, 'w') + try: + f = file(self.ufilename, 'w') + except UnicodeEncodeError: + py.test.skip("encoding not good enough") f.write("test") f.close() Modified: pypy/trunk/pypy/rpython/extfunc.py ============================================================================== --- pypy/trunk/pypy/rpython/extfunc.py (original) +++ pypy/trunk/pypy/rpython/extfunc.py Fri Jul 16 23:46:23 2010 @@ -52,7 +52,10 @@ return super(ExtRegistryEntry, self).__getattr__(attr) raise exc, exc_inst, tb -def registering(func): +def registering(func, condition=True): + if not condition: + return lambda method: None + def decorator(method): method._registering_func = func return method @@ -63,11 +66,9 @@ func = getattr(ns, name) except AttributeError: condition = False + func = None - if condition: - return registering(func) - else: - return lambda method: None + return registering(func, condition=condition) class LazyRegisteringMeta(type): def __new__(self, _name, _type, _vars): @@ -167,8 +168,6 @@ return signature_args def compute_result_annotation(self, *args_s): - if hasattr(self, 'ann_hook'): - self.ann_hook() self.normalize_args(*args_s) # check arguments return self.signature_result @@ -235,7 +234,6 @@ def register_external(function, args, result=None, export_name=None, llimpl=None, ooimpl=None, llfakeimpl=None, oofakeimpl=None, - annotation_hook=None, sandboxsafe=False): """ function: the RPython function that will be rendered as an external function (e.g.: math.floor) @@ -244,7 +242,6 @@ export_name: the name of the function as it will be seen by the backends llimpl, ooimpl: optional; if provided, these RPython functions are called instead of the target function llfakeimpl, oofakeimpl: optional; if provided, they are called by the llinterpreter - annotationhook: optional; a callable that is called during annotation, useful for genc hacks sandboxsafe: use True if the function performs no I/O (safe for --sandbox) """ @@ -271,8 +268,6 @@ lltypefakeimpl = staticmethod(llfakeimpl) if oofakeimpl: ootypefakeimpl = staticmethod(oofakeimpl) - if annotation_hook: - ann_hook = staticmethod(annotation_hook) if export_name: FunEntry.__name__ = export_name Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rffi.py Fri Jul 16 23:46:23 2010 @@ -176,6 +176,15 @@ # XXX leaks if a str2charp() fails with MemoryError # and was not the first in this function freeme = arg + elif TARGET == CWCHARP: + if arg is None: + arg = lltype.nullptr(CWCHARP.TO) # None => (wchar_t*)NULL + freeme = arg + elif isinstance(arg, unicode): + arg = unicode2wcharp(arg) + # XXX leaks if a unicode2wcharp() fails with MemoryError + # and was not the first in this function + freeme = arg elif _isfunctype(TARGET) and not _isllptr(arg): # XXX pass additional arguments if invoke_around_handlers: Modified: pypy/trunk/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/trunk/pypy/rpython/module/ll_os.py (original) +++ pypy/trunk/pypy/rpython/module/ll_os.py Fri Jul 16 23:46:23 2010 @@ -6,12 +6,15 @@ # might be found in doc/rffi.txt import os, sys, errno +import py from pypy.rpython.module.support import ll_strcpy, OOSupport -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rlib.rarithmetic import r_longlong -from pypy.rpython.extfunc import BaseLazyRegistering +from pypy.rpython.extfunc import ( + BaseLazyRegistering, lazy_register, register_external) from pypy.rpython.extfunc import registering, registering_if, extdef -from pypy.annotation.model import SomeInteger, SomeString, SomeTuple, SomeFloat +from pypy.annotation.model import ( + SomeInteger, SomeString, SomeTuple, SomeFloat, SomeUnicodeString) from pypy.annotation.model import s_ImpossibleValue, s_None, s_Bool from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype @@ -26,7 +29,99 @@ from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.annlowlevel import llstr from pypy.rlib import rgc -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import keepalive_until_here, specialize + +def monkeypatch_rposix(posixfunc, unicodefunc, signature): + func_name = posixfunc.__name__ + + if hasattr(signature, '_default_signature_'): + signature = signature._default_signature_ + arglist = ['arg%d' % (i,) for i in range(len(signature))] + transformed_arglist = arglist[:] + for i, arg in enumerate(signature): + if arg is unicode: + transformed_arglist[i] = transformed_arglist[i] + '.as_unicode()' + + args = ', '.join(arglist) + transformed_args = ', '.join(transformed_arglist) + main_arg = 'arg%d' % (signature.index(unicode),) + + source = py.code.Source(""" + def %(func_name)s(%(args)s): + if isinstance(%(main_arg)s, str): + return posixfunc(%(args)s) + else: + return unicodefunc(%(transformed_args)s) + """ % locals()) + miniglobals = {'posixfunc' : posixfunc, + 'unicodefunc': unicodefunc, + '__name__': __name__, # for module name propagation + } + exec source.compile() in miniglobals + new_func = miniglobals[func_name] + specialized_args = [i for i in range(len(signature)) + if signature[i] in (unicode, None)] + new_func = specialize.argtype(*specialized_args)(new_func) + + # Monkeypatch the function in pypy.rlib.rposix + setattr(rposix, func_name, new_func) + +class StringTraits: + str = str + CHAR = rffi.CHAR + CCHARP = rffi.CCHARP + charp2str = staticmethod(rffi.charp2str) + str2charp = staticmethod(rffi.str2charp) + free_charp = staticmethod(rffi.free_charp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_' + name + +class UnicodeTraits: + str = unicode + CHAR = rffi.WCHAR_T + CCHARP = rffi.CWCHARP + charp2str = staticmethod(rffi.wcharp2unicode) + str2charp = staticmethod(rffi.unicode2wcharp) + free_charp = staticmethod(rffi.free_wcharp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + 'w' + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_w' + name + +def registering_str_unicode(posixfunc, condition=True): + if not condition: + return registering(None, condition=False) + + func_name = posixfunc.__name__ + + def register_posixfunc(self, method): + val = method(self, StringTraits()) + register_external(posixfunc, *val.def_args, **val.def_kwds) + + if sys.platform == 'win32': + val = method(self, UnicodeTraits()) + @func_renamer(func_name + "_unicode") + def unicodefunc(*args): + return posixfunc(*args) + register_external(unicodefunc, *val.def_args, **val.def_kwds) + signature = val.def_args[0] + monkeypatch_rposix(posixfunc, unicodefunc, signature) + + def decorator(method): + decorated = lambda self: register_posixfunc(self, method) + decorated._registering_func = posixfunc + return decorated + return decorator posix = __import__(os.name) @@ -282,8 +377,8 @@ return extdef([int, int], s_None, llimpl=dup2_llimpl, export_name="ll_os.ll_os_dup2") - @registering(os.utime) - def register_os_utime(self): + @registering_str_unicode(os.utime) + def register_os_utime(self, traits): UTIMBUFP = lltype.Ptr(self.UTIMBUF) os_utime = self.llexternal('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT) @@ -336,6 +431,9 @@ # tp is known to be None, and one version where it is known # to be a tuple of 2 floats. if not _WIN32: + assert traits.str is str + + @specialize.argtype(1) def os_utime_llimpl(path, tp): if tp is None: error = os_utime(path, lltype.nullptr(UTIMBUFP.TO)) @@ -346,85 +444,13 @@ if error == -1: raise OSError(rposix.get_errno(), "os_utime failed") else: - from pypy.rlib import rwin32 - from pypy.rpython.module.ll_os_stat import time_t_to_FILE_TIME - - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'], - ) - - FILE_WRITE_ATTRIBUTES = platform.ConstantInteger( - 'FILE_WRITE_ATTRIBUTES') - OPEN_EXISTING = platform.ConstantInteger( - 'OPEN_EXISTING') - FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger( - 'FILE_FLAG_BACKUP_SEMANTICS') - globals().update(platform.configure(CConfig)) - - CreateFile = rffi.llexternal( - 'CreateFileA', - [rwin32.LPCSTR, rwin32.DWORD, rwin32.DWORD, - rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD, - rwin32.HANDLE], - rwin32.HANDLE, - calling_conv='win') - - GetSystemTime = rffi.llexternal( - 'GetSystemTime', - [lltype.Ptr(rwin32.SYSTEMTIME)], - lltype.Void, - calling_conv='win') - - SystemTimeToFileTime = rffi.llexternal( - 'SystemTimeToFileTime', - [lltype.Ptr(rwin32.SYSTEMTIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv='win') - - SetFileTime = rffi.llexternal( - 'SetFileTime', - [rwin32.HANDLE, - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv = 'win') + from pypy.rpython.module.ll_win32file import make_utime_impl + os_utime_llimpl = make_utime_impl(traits) - def os_utime_llimpl(path, tp): - hFile = CreateFile(path, - FILE_WRITE_ATTRIBUTES, 0, - None, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, 0) - if hFile == rwin32.INVALID_HANDLE_VALUE: - raise rwin32.lastWindowsError() - ctime = lltype.nullptr(rwin32.FILETIME) - atime = lltype.malloc(rwin32.FILETIME, flavor='raw') - mtime = lltype.malloc(rwin32.FILETIME, flavor='raw') - try: - if tp is None: - now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw') - try: - GetSystemTime(now) - if (not SystemTimeToFileTime(now, atime) or - not SystemTimeToFileTime(now, mtime)): - raise rwin32.lastWindowsError() - finally: - lltype.free(now, flavor='raw') - else: - actime, modtime = tp - time_t_to_FILE_TIME(actime, atime) - time_t_to_FILE_TIME(modtime, mtime) - if not SetFileTime(hFile, ctime, atime, mtime): - raise rwin32.lastWindowsError() - finally: - rwin32.CloseHandle(hFile) - lltype.free(atime, flavor='raw') - lltype.free(mtime, flavor='raw') - os_utime_llimpl._annspecialcase_ = 'specialize:argtype(1)' - - s_string = SomeString() + if traits.str is str: + s_string = SomeString() + else: + s_string = SomeUnicodeString() s_tuple_of_2_floats = SomeTuple([SomeFloat(), SomeFloat()]) def os_utime_normalize_args(s_path, s_times): @@ -445,12 +471,12 @@ else: raise Exception("os.utime() arg 2 must be None or a tuple of " "2 floats, got %s" % (s_times,)) + os_utime_normalize_args._default_signature_ = [traits.str, None] return extdef(os_utime_normalize_args, s_None, "ll_os.ll_os_utime", llimpl=os_utime_llimpl) - @registering(os.times) def register_os_times(self): if sys.platform.startswith('win'): @@ -687,22 +713,21 @@ def register_os_setsid(self): return self.extdef_for_os_function_returning_int('setsid') - @registering(os.open) - def register_os_open(self): - os_open = self.llexternal(underscore_on_windows+'open', - [rffi.CCHARP, rffi.INT, rffi.MODE_T], + @registering_str_unicode(os.open) + def register_os_open(self, traits): + os_open = self.llexternal(traits.posix_function_name('open'), + [traits.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT) - def os_open_llimpl(path, flags, mode): result = rffi.cast(rffi.LONG, os_open(path, flags, mode)) if result == -1: raise OSError(rposix.get_errno(), "os_open failed") return result - def os_open_oofakeimpl(o_path, flags, mode): - return os.open(o_path._str, flags, mode) + def os_open_oofakeimpl(path, flags, mode): + return os.open(OOSupport.from_rstr(path), flags, mode) - return extdef([str, int, int], int, "ll_os.ll_os_open", + return extdef([traits.str, int, int], int, traits.ll_os_name('open'), llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) # ------------------------------- os.read ------------------------------- @@ -862,10 +887,10 @@ llimpl=fdatasync_llimpl, export_name="ll_os.ll_os_fdatasync") - @registering(os.access) - def register_os_access(self): - os_access = self.llexternal(underscore_on_windows + 'access', - [rffi.CCHARP, rffi.INT], + @registering_str_unicode(os.access) + def register_os_access(self, traits): + os_access = self.llexternal(traits.posix_function_name('access'), + [traits.CCHARP, rffi.INT], rffi.INT) if sys.platform.startswith('win'): @@ -882,44 +907,22 @@ def os_access_oofakeimpl(path, mode): return os.access(OOSupport.from_rstr(path), mode) - return extdef([str, int], s_Bool, llimpl=access_llimpl, - export_name="ll_os.ll_os_access", + return extdef([traits.str, int], s_Bool, llimpl=access_llimpl, + export_name=traits.ll_os_name("access"), oofakeimpl=os_access_oofakeimpl) - @registering_if(posix, '_getfullpathname') - def register_posix__getfullpathname(self): - from pypy.rlib import rwin32 + @registering_str_unicode(getattr(posix, '_getfullpathname', None), + condition=sys.platform=='win32') + def register_posix__getfullpathname(self, traits): # this nt function is not exposed via os, but needed # to get a correct implementation of os.abspath - # XXX why do we ignore WINAPI conventions everywhere? - LPSTRP = rffi.CArrayPtr(rwin32.LPSTR) - # XXX unicode? - GetFullPathName = self.llexternal( - 'GetFullPathNameA', - [rwin32.LPCSTR, - rwin32.DWORD, - rwin32.LPSTR, - rffi.CArrayPtr(rwin32.LPSTR)], - rwin32.DWORD) - - def _getfullpathname_llimpl(lpFileName): - nBufferLength = rwin32.MAX_PATH + 1 - lpBuffer = lltype.malloc(rwin32.LPSTR.TO, nBufferLength, flavor='raw') - try: - res = GetFullPathName( - lpFileName, rffi.cast(rwin32.DWORD, nBufferLength), - lpBuffer, lltype.nullptr(LPSTRP.TO)) - if res == 0: - raise rwin32.lastWindowsError("_getfullpathname failed") - result = rffi.charp2str(lpBuffer) - return result - finally: - lltype.free(lpBuffer, flavor='raw') + from pypy.rpython.module.ll_win32file import make_getfullpathname_impl + getfullpathname_llimpl = make_getfullpathname_impl(traits) - return extdef([str], # a single argument which is a str - str, # returns a string - "ll_os.posix__getfullpathname", - llimpl=_getfullpathname_llimpl) + return extdef([traits.str], # a single argument which is a str + traits.str, # returns a string + traits.ll_os_name('_getfullpathname'), + llimpl=getfullpathname_llimpl) @registering(os.getcwd) def register_os_getcwd(self): @@ -953,71 +956,42 @@ "ll_os.ll_os_getcwd", llimpl=os_getcwd_llimpl, oofakeimpl=os_getcwd_oofakeimpl) - @registering(os.listdir) - def register_os_listdir(self): - # we need a different approach on Windows and on Posix - if sys.platform.startswith('win'): - from pypy.rlib import rwin32 - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'] - ) - WIN32_FIND_DATA = platform.Struct('struct _WIN32_FIND_DATAA', - [('cFileName', lltype.FixedSizeArray(rffi.CHAR, 1))]) - ERROR_FILE_NOT_FOUND = platform.ConstantInteger( - 'ERROR_FILE_NOT_FOUND') - ERROR_NO_MORE_FILES = platform.ConstantInteger( - 'ERROR_NO_MORE_FILES') + @registering(os.getcwdu, condition=sys.platform=='win32') + def register_os_getcwdu(self): + os_wgetcwd = self.llexternal(underscore_on_windows + 'wgetcwd', + [rffi.CWCHARP, rffi.SIZE_T], + rffi.CWCHARP) - config = platform.configure(CConfig) - WIN32_FIND_DATA = config['WIN32_FIND_DATA'] - ERROR_FILE_NOT_FOUND = config['ERROR_FILE_NOT_FOUND'] - ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] - LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) - - FindFirstFile = self.llexternal('FindFirstFile', - [rwin32.LPCSTR, LPWIN32_FIND_DATA], - rwin32.HANDLE) - FindNextFile = self.llexternal('FindNextFile', - [rwin32.HANDLE, LPWIN32_FIND_DATA], - rwin32.BOOL) - FindClose = self.llexternal('FindClose', - [rwin32.HANDLE], - rwin32.BOOL) + def os_getcwd_llimpl(): + bufsize = 256 + while True: + buf = lltype.malloc(rffi.CWCHARP.TO, bufsize, flavor='raw') + res = os_wgetcwd(buf, rffi.cast(rffi.SIZE_T, bufsize)) + if res: + break # ok + error = rposix.get_errno() + lltype.free(buf, flavor='raw') + if error != errno.ERANGE: + raise OSError(error, "getcwd failed") + # else try again with a larger buffer, up to some sane limit + bufsize *= 4 + if bufsize > 1024*1024: # xxx hard-coded upper limit + raise OSError(error, "getcwd result too large") + result = rffi.wcharp2unicode(res) + lltype.free(buf, flavor='raw') + return result - def os_listdir_llimpl(path): - if path and path[-1] not in ('/', '\\', ':'): - path += '/' - path += '*.*' - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - try: - result = [] - hFindFile = FindFirstFile(path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - error = rwin32.GetLastError() - if error == ERROR_FILE_NOT_FOUND: - return result - else: - raise WindowsError(error, "FindFirstFile failed") - while True: - name = rffi.charp2str(rffi.cast(rffi.CCHARP, - filedata.c_cFileName)) - if name != "." and name != "..": # skip these - result.append(name) - if not FindNextFile(hFindFile, filedata): - break - # FindNextFile sets error to ERROR_NO_MORE_FILES if - # it got to the end of the directory - error = rwin32.GetLastError() - FindClose(hFindFile) - if error == ERROR_NO_MORE_FILES: - return result - else: - raise WindowsError(error, "FindNextFile failed") - finally: - lltype.free(filedata, flavor='raw') + return extdef([], unicode, + "ll_os.ll_os_wgetcwd", llimpl=os_getcwd_llimpl) + @registering_str_unicode(os.listdir) + def register_os_listdir(self, traits): + # we need a different approach on Windows and on Posix + if sys.platform.startswith('win'): + from pypy.rpython.module.ll_win32file import make_listdir_impl + os_listdir_llimpl = make_listdir_impl(traits) else: + assert traits.str is str compilation_info = ExternalCompilationInfo( includes = ['sys/types.h', 'dirent.h'] ) @@ -1057,9 +1031,9 @@ raise OSError(error, "os_readdir failed") return result - return extdef([str], # a single argument which is a str - [str], # returns a list of strings - "ll_os.ll_os_listdir", + return extdef([traits.str], # a single argument which is a str + [traits.str], # returns a list of strings + traits.ll_os_name('listdir'), llimpl=os_listdir_llimpl) @registering(os.pipe) @@ -1234,38 +1208,40 @@ return extdef([str], int, llimpl=system_llimpl, export_name="ll_os.ll_os_system") - @registering(os.unlink) - def register_os_unlink(self): - os_unlink = self.llexternal(underscore_on_windows+'unlink', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.unlink) + def register_os_unlink(self, traits): + os_unlink = self.llexternal(traits.posix_function_name('unlink'), + [traits.CCHARP], rffi.INT) def unlink_llimpl(pathname): res = rffi.cast(lltype.Signed, os_unlink(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_unlink failed") - return extdef([str], s_None, llimpl=unlink_llimpl, - export_name="ll_os.ll_os_unlink") + return extdef([traits.str], s_None, llimpl=unlink_llimpl, + export_name=traits.ll_os_name('unlink')) - @registering(os.chdir) - def register_os_chdir(self): - os_chdir = self.llexternal(underscore_on_windows+'chdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.chdir) + def register_os_chdir(self, traits): + os_chdir = self.llexternal(traits.posix_function_name('chdir'), + [traits.CCHARP], rffi.INT) def chdir_llimpl(path): res = rffi.cast(lltype.Signed, os_chdir(path)) if res < 0: raise OSError(rposix.get_errno(), "os_chdir failed") - return extdef([str], s_None, llimpl=chdir_llimpl, - export_name="ll_os.ll_os_chdir") + return extdef([traits.str], s_None, llimpl=chdir_llimpl, + export_name=traits.ll_os_name('chdir')) - @registering(os.mkdir) - def register_os_mkdir(self): + @registering_str_unicode(os.mkdir) + def register_os_mkdir(self, traits): if os.name == 'nt': ARG2 = [] # no 'mode' argument on Windows - just ignored else: ARG2 = [rffi.MODE_T] - os_mkdir = self.llexternal(underscore_on_windows+'mkdir', - [rffi.CCHARP]+ARG2, rffi.INT) + os_mkdir = self.llexternal(traits.posix_function_name('mkdir'), + [traits.CCHARP] + ARG2, rffi.INT) IGNORE_MODE = len(ARG2) == 0 def mkdir_llimpl(pathname, mode): @@ -1277,46 +1253,47 @@ if res < 0: raise OSError(rposix.get_errno(), "os_mkdir failed") - return extdef([str, int], s_None, llimpl=mkdir_llimpl, - export_name="ll_os.ll_os_mkdir") + return extdef([traits.str, int], s_None, llimpl=mkdir_llimpl, + export_name=traits.ll_os_name('mkdir')) - @registering(os.rmdir) - def register_os_rmdir(self): - os_rmdir = self.llexternal(underscore_on_windows+'rmdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.rmdir) + def register_os_rmdir(self, traits): + os_rmdir = self.llexternal(traits.posix_function_name('rmdir'), + [traits.CCHARP], rffi.INT) def rmdir_llimpl(pathname): res = rffi.cast(lltype.Signed, os_rmdir(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_rmdir failed") - return extdef([str], s_None, llimpl=rmdir_llimpl, - export_name="ll_os.ll_os_rmdir") + return extdef([traits.str], s_None, llimpl=rmdir_llimpl, + export_name=traits.ll_os_name('rmdir')) - @registering(os.chmod) - def register_os_chmod(self): - os_chmod = self.llexternal(underscore_on_windows+'chmod', [rffi.CCHARP, rffi.MODE_T], - rffi.INT) + @registering_str_unicode(os.chmod) + def register_os_chmod(self, traits): + os_chmod = self.llexternal(traits.posix_function_name('chmod'), + [traits.CCHARP, rffi.MODE_T], rffi.INT) def chmod_llimpl(path, mode): res = rffi.cast(lltype.Signed, os_chmod(path, rffi.cast(rffi.MODE_T, mode))) if res < 0: raise OSError(rposix.get_errno(), "os_chmod failed") - return extdef([str, int], s_None, llimpl=chmod_llimpl, - export_name="ll_os.ll_os_chmod") + return extdef([traits.str, int], s_None, llimpl=chmod_llimpl, + export_name=traits.ll_os_name('chmod')) - @registering(os.rename) - def register_os_rename(self): - os_rename = self.llexternal('rename', [rffi.CCHARP, rffi.CCHARP], - rffi.INT) + @registering_str_unicode(os.rename) + def register_os_rename(self, traits): + os_rename = self.llexternal(traits.posix_function_name('rename'), + [traits.CCHARP, traits.CCHARP], rffi.INT) def rename_llimpl(oldpath, newpath): res = rffi.cast(lltype.Signed, os_rename(oldpath, newpath)) if res < 0: raise OSError(rposix.get_errno(), "os_rename failed") - return extdef([str, str], s_None, llimpl=rename_llimpl, - export_name="ll_os.ll_os_rename") + return extdef([traits.str, traits.str], s_None, llimpl=rename_llimpl, + export_name=traits.ll_os_name('rename')) @registering(os.umask) def register_os_umask(self): @@ -1425,17 +1402,17 @@ @registering(os.fstat) def register_os_fstat(self): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('fstat') + return ll_os_stat.register_stat_variant('fstat', StringTraits()) - @registering(os.stat) - def register_os_stat(self): + @registering_str_unicode(os.stat) + def register_os_stat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('stat') + return ll_os_stat.register_stat_variant('stat', traits) - @registering(os.lstat) - def register_os_lstat(self): + @registering_str_unicode(os.lstat) + def register_os_lstat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('lstat') + return ll_os_stat.register_stat_variant('lstat', traits) # ------------------------------- os.W* --------------------------------- Modified: pypy/trunk/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/trunk/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/trunk/pypy/rpython/module/ll_os_stat.py Fri Jul 16 23:46:23 2010 @@ -5,13 +5,14 @@ import os, sys from pypy.annotation import model as annmodel from pypy.tool.pairtype import pairtype -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rpython import extregistry -from pypy.rpython.extfunc import register_external +from pypy.rpython.extfunc import register_external, extdef from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.annlowlevel import hlstr @@ -211,13 +212,27 @@ return make_stat_result(result) -def register_stat_variant(name): - if sys.platform.startswith('win'): - _functions = {'stat': '_stati64', - 'fstat': '_fstati64', - 'lstat': '_stati64'} # no lstat on Windows - c_func_name = _functions[name] - elif sys.platform.startswith('linux'): +def register_stat_variant(name, traits): + if name != 'fstat': + arg_is_path = True + s_arg = traits.str + ARG1 = traits.CCHARP + else: + arg_is_path = False + s_arg = int + ARG1 = rffi.INT + + if sys.platform == 'win32': + # See Win32 implementation below + posix_stat_llimpl = make_win32_stat_impl(name, traits) + + return extdef( + [s_arg], s_StatResult, traits.ll_os_name(name), + llimpl=posix_stat_llimpl) + + assert traits.str is str + + if sys.platform.startswith('linux'): # because we always use _FILE_OFFSET_BITS 64 - this helps things work that are not a c compiler _functions = {'stat': 'stat64', 'fstat': 'fstat64', @@ -226,22 +241,26 @@ else: c_func_name = name - arg_is_path = (name != 'fstat') + posix_mystat = rffi.llexternal(c_func_name, + [ARG1, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info) + @func_renamer('os_%s_llimpl' % (name,)) def posix_stat_llimpl(arg): stresult = lltype.malloc(STAT_STRUCT.TO, flavor='raw') try: if arg_is_path: - arg = rffi.str2charp(arg) + arg = traits.str2charp(arg) error = rffi.cast(rffi.LONG, posix_mystat(arg, stresult)) if arg_is_path: - rffi.free_charp(arg) + traits.free_charp(arg) if error != 0: raise OSError(rposix.get_errno(), "os_?stat failed") return build_stat_result(stresult) finally: lltype.free(stresult, flavor='raw') + @func_renamer('os_%s_fake' % (name,)) def posix_fakeimpl(arg): if s_arg == str: arg = hlstr(arg) @@ -259,40 +278,17 @@ setattr(ll_tup, 'item%d' % i, val) return ll_tup - if arg_is_path: - s_arg = str - ARG1 = rffi.CCHARP - else: - s_arg = int - ARG1 = rffi.INT + return extdef( + [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), + llimpl=posix_stat_llimpl, llfakeimpl=posix_fakeimpl) - if sys.platform != 'win32': - posix_mystat = rffi.llexternal(c_func_name, - [ARG1, STAT_STRUCT], rffi.INT, - compilation_info=compilation_info) - - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(posix_stat_llimpl, - 'os_%s_llimpl' % (name,)), - llfakeimpl=func_with_new_name(posix_fakeimpl, - 'os_%s_fake' % (name,)), - ) - else: - # See Win32 implementation below - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(globals()['win32_%s_llimpl' % (name,)], - 'os_%s_llimpl' % (name,)), - ) +def make_win32_stat_impl(name, traits): + from pypy.rlib import rwin32 + from pypy.rpython.module.ll_win32file import make_win32_traits + win32traits = make_win32_traits(traits) -# ____________________________________________________________ -if sys.platform == 'win32': # The CRT of Windows has a number of flaws wrt. its stat() implementation: - # - for when we implement subsecond resolution in RPython, time stamps - # would be restricted to second resolution + # - time stamps are restricted to second resolution # - file modification times suffer from forth-and-back conversions between # UTC and local time # Therefore, we implement our own stat, based on the Win32 API directly. @@ -302,122 +298,18 @@ assert len(STAT_FIELDS) == 10 # no extra fields on Windows - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h', 'winbase.h', 'sys/stat.h'], - ) - - GetFileExInfoStandard = platform.ConstantInteger( - 'GetFileExInfoStandard') - FILE_ATTRIBUTE_DIRECTORY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_DIRECTORY') - FILE_ATTRIBUTE_READONLY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_READONLY') - ERROR_SHARING_VIOLATION = platform.ConstantInteger( - 'ERROR_SHARING_VIOLATION') - _S_IFDIR = platform.ConstantInteger('_S_IFDIR') - _S_IFREG = platform.ConstantInteger('_S_IFREG') - _S_IFCHR = platform.ConstantInteger('_S_IFCHR') - _S_IFIFO = platform.ConstantInteger('_S_IFIFO') - FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN') - FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') - FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') - - WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( - 'WIN32_FILE_ATTRIBUTE_DATA', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - BY_HANDLE_FILE_INFORMATION = platform.Struct( - 'BY_HANDLE_FILE_INFORMATION', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('nNumberOfLinks', rwin32.DWORD), - ('nFileIndexHigh', rwin32.DWORD), - ('nFileIndexLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - WIN32_FIND_DATA = platform.Struct( - 'WIN32_FIND_DATAA', - # Only interesting fields - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - globals().update(platform.configure(CConfig)) - GET_FILEEX_INFO_LEVELS = rffi.ULONG # an enumeration - - GetFileAttributesEx = rffi.llexternal( - 'GetFileAttributesExA', - [rffi.CCHARP, GET_FILEEX_INFO_LEVELS, - lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)], - rwin32.BOOL, - calling_conv='win') - - GetFileInformationByHandle = rffi.llexternal( - 'GetFileInformationByHandle', - [rwin32.HANDLE, lltype.Ptr(BY_HANDLE_FILE_INFORMATION)], - rwin32.BOOL, - calling_conv='win') - - GetFileType = rffi.llexternal( - 'GetFileType', - [rwin32.HANDLE], - rwin32.DWORD, - calling_conv='win') - - FindFirstFile = rffi.llexternal( - 'FindFirstFileA', - [rffi.CCHARP, lltype.Ptr(WIN32_FIND_DATA)], - rwin32.HANDLE, - calling_conv='win') - - FindClose = rffi.llexternal( - 'FindClose', - [rwin32.HANDLE], - rwin32.BOOL, - calling_conv='win') - def attributes_to_mode(attributes): m = 0 - if attributes & FILE_ATTRIBUTE_DIRECTORY: - m |= _S_IFDIR | 0111 # IFEXEC for user,group,other + if attributes & win32traits.FILE_ATTRIBUTE_DIRECTORY: + m |= win32traits._S_IFDIR | 0111 # IFEXEC for user,group,other else: - m |= _S_IFREG - if attributes & FILE_ATTRIBUTE_READONLY: + m |= win32traits._S_IFREG + if attributes & win32traits.FILE_ATTRIBUTE_READONLY: m |= 0444 else: m |= 0666 return m - def make_longlong(high, low): - return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) - - # Seconds between 1.1.1601 and 1.1.1970 - secs_between_epochs = lltype.r_longlong(11644473600) - - def FILE_TIME_to_time_t_nsec(filetime): - ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) - # FILETIME is in units of 100 nsec - nsec = (ft % 10000000) * 100 - time = (ft / 10000000) - secs_between_epochs - return time, nsec - - def time_t_to_FILE_TIME(time, filetime): - ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) - filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) - filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) - def attribute_data_to_stat(info): st_mode = attributes_to_mode(info.c_dwFileAttributes) st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) @@ -456,65 +348,94 @@ return make_stat_result(result) def attributes_from_dir(l_path, data): - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - hFindFile = FindFirstFile(l_path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - return 0 - FindClose(hFindFile) - data.c_dwFileAttributes = filedata.c_dwFileAttributes - rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) - rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) - rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) - data.c_nFileSizeHigh = filedata.c_nFileSizeHigh - data.c_nFileSizeLow = filedata.c_nFileSizeLow - return 1 + filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') + try: + hFindFile = win32traits.FindFirstFile(l_path, filedata) + if hFindFile == rwin32.INVALID_HANDLE_VALUE: + return 0 + win32traits.FindClose(hFindFile) + data.c_dwFileAttributes = filedata.c_dwFileAttributes + rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) + rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) + rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) + data.c_nFileSizeHigh = filedata.c_nFileSizeHigh + data.c_nFileSizeLow = filedata.c_nFileSizeLow + return 1 + finally: + lltype.free(filedata, flavor='raw') def win32_stat_llimpl(path): - data = lltype.malloc(WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') + data = lltype.malloc(win32traits.WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') try: - l_path = rffi.str2charp(path) - res = GetFileAttributesEx(l_path, GetFileExInfoStandard, data) + l_path = traits.str2charp(path) + res = win32traits.GetFileAttributesEx(l_path, win32traits.GetFileExInfoStandard, data) errcode = rwin32.GetLastError() if res == 0: - if errcode == ERROR_SHARING_VIOLATION: + if errcode == win32traits.ERROR_SHARING_VIOLATION: res = attributes_from_dir(l_path, data) errcode = rwin32.GetLastError() - rffi.free_charp(l_path) + traits.free_charp(l_path) if res == 0: raise WindowsError(errcode, "os_stat failed") return attribute_data_to_stat(data) finally: lltype.free(data, flavor='raw') - win32_lstat_llimpl = win32_stat_llimpl def win32_fstat_llimpl(fd): handle = rwin32._get_osfhandle(fd) - filetype = GetFileType(handle) - if filetype == FILE_TYPE_CHAR: + filetype = win32traits.GetFileType(handle) + if filetype == win32traits.FILE_TYPE_CHAR: # console or LPT device - return make_stat_result((_S_IFCHR, + return make_stat_result((win32traits._S_IFCHR, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_PIPE: + elif filetype == win32traits.FILE_TYPE_PIPE: # socket or named pipe - return make_stat_result((_S_IFIFO, + return make_stat_result((win32traits._S_IFIFO, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_UNKNOWN: + elif filetype == win32traits.FILE_TYPE_UNKNOWN: error = rwin32.GetLastError() if error != 0: raise WindowsError(error, "os_fstat failed") # else: unknown but valid file # normal disk file (FILE_TYPE_DISK) - info = lltype.malloc(BY_HANDLE_FILE_INFORMATION, flavor='raw', - zero=True) + info = lltype.malloc(win32traits.BY_HANDLE_FILE_INFORMATION, + flavor='raw', zero=True) try: - res = GetFileInformationByHandle(handle, info) + res = win32traits.GetFileInformationByHandle(handle, info) if res == 0: raise WindowsError(rwin32.GetLastError(), "os_fstat failed") return by_handle_info_to_stat(info) finally: lltype.free(info, flavor='raw') + if name == 'fstat': + return win32_fstat_llimpl + else: + return win32_stat_llimpl + + +#__________________________________________________ +# Helper functions for win32 + +def make_longlong(high, low): + return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) + +# Seconds between 1.1.1601 and 1.1.1970 +secs_between_epochs = lltype.r_longlong(11644473600) + +def FILE_TIME_to_time_t_nsec(filetime): + ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) + # FILETIME is in units of 100 nsec + nsec = (ft % 10000000) * 100 + time = (ft / 10000000) - secs_between_epochs + return time, nsec + +def time_t_to_FILE_TIME(time, filetime): + ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) + filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) + filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) + Copied: pypy/trunk/pypy/rpython/module/ll_win32file.py (from r76265, pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py) ============================================================================== --- pypy/branch/unicode_filename-2/pypy/rpython/module/ll_win32file.py (original) +++ pypy/trunk/pypy/rpython/module/ll_win32file.py Fri Jul 16 23:46:23 2010 @@ -76,8 +76,9 @@ config = platform.configure(CConfig) def external(*args, **kwargs): - kwargs['compilation_info'] = CConfig._compilation_info_ - return rffi.llexternal(calling_conv='win', *args, **kwargs) + kwargs['compilation_info'] = CConfig._compilation_info_ + llfunc = rffi.llexternal(calling_conv='win', *args, **kwargs) + return staticmethod(llfunc) class Win32Traits: apisuffix = suffix Modified: pypy/trunk/pypy/rpython/module/test/test_ll_os_stat.py ============================================================================== --- pypy/trunk/pypy/rpython/module/test/test_ll_os_stat.py (original) +++ pypy/trunk/pypy/rpython/module/test/test_ll_os_stat.py Fri Jul 16 23:46:23 2010 @@ -1,4 +1,4 @@ -from pypy.rpython.module import ll_os_stat +from pypy.rpython.module import ll_os_stat, ll_os import sys, os import py @@ -8,14 +8,18 @@ py.test.skip("win32 specific tests") def test_stat(self): - stat = ll_os_stat.win32_stat_llimpl + stat = ll_os_stat.make_win32_stat_impl('stat', ll_os.StringTraits()) + wstat = ll_os_stat.make_win32_stat_impl('stat', ll_os.UnicodeTraits()) def check(f): - assert stat(f).st_mtime == os.stat(f).st_mtime + expected = os.stat(f).st_mtime + assert stat(f).st_mtime == expected + assert wstat(unicode(f)).st_mtime == expected check('c:/') check('c:/temp') check('c:/pagefile.sys') def test_fstat(self): - stat = ll_os_stat.win32_fstat_llimpl(0) # stdout + fstat = ll_os_stat.make_win32_stat_impl('fstat', ll_os.StringTraits()) + stat = fstat(0) # stdout assert stat.st_mode != 0 Modified: pypy/trunk/pypy/rpython/test/test_extfunc.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_extfunc.py (original) +++ pypy/trunk/pypy/rpython/test/test_extfunc.py Fri Jul 16 23:46:23 2010 @@ -6,150 +6,164 @@ from pypy.annotation.policy import AnnotatorPolicy from pypy.rpython.test.test_llinterp import interpret -def b(x): - return eval("x+40") +class TestExtFuncEntry: -class BTestFuncEntry(ExtFuncEntry): - _about_ = b - name = 'b' - signature_args = [annmodel.SomeInteger()] - signature_result = annmodel.SomeInteger() - -def test_annotation_b(): - def f(): - return b(1) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def test_rtyping_b(): - def f(): - return b(2) - - res = interpret(f, []) - assert res == 42 - -def c(y, x): - yyy - -class CTestFuncEntry(ExtFuncEntry): - _about_ = c - name = 'ccc' - signature_args = [annmodel.SomeInteger()] * 2 - signature_result = annmodel.SomeInteger() - - def lltypeimpl(y, x): - return y + x - lltypeimpl = staticmethod(lltypeimpl) - -def test_interp_c(): - def f(): - return c(3, 4) - - res = interpret(f, []) - assert res == 7 - -def d(y): - return eval("y()") - -class DTestFuncEntry(ExtFuncEntry): - _about_ = d - name = 'd' - signature_args = [annmodel.SomeGenericCallable(args=[], result= - annmodel.SomeFloat())] - signature_result = annmodel.SomeFloat() - -def test_callback(): - def callback(): - return 2.5 - - def f(): - return d(callback) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeFloat) - assert a.translator._graphof(callback) - -def dd(): - pass - -register_external(dd, [int], int) - -def test_register_external_signature(): - def f(): - return dd(3) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - - -def function_with_tuple_arg(): - """ - Dummy function which is declared via register_external to take a tuple as - an argument so that register_external's behavior for tuple-taking functions - can be verified. - """ -register_external(function_with_tuple_arg, [(int,)], int) - -def test_register_external_tuple_args(): - """ - Verify the annotation of a registered external function which takes a tuple - argument. - """ - def f(): - return function_with_tuple_arg((1,)) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - - # Not a very good assertion, but at least it means _something_ happened. - assert isinstance(s, annmodel.SomeInteger) - -def function_with_list(): - pass -register_external(function_with_list, [[int]], int) - -def function_returning_list(): - pass -register_external(function_returning_list, [], [int]) - -def test_register_external_return_goes_back(): - """ - Check whether it works to pass the same list from one external - fun to another - [bookkeeper and list joining issues] - """ - def f(): - return function_with_list(function_returning_list()) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def function_withspecialcase(arg): - return repr(arg) -register_external(function_withspecialcase, args=None, result=str) - -def test_register_external_specialcase(): - def f(): - x = function_withspecialcase - return x(33) + x("aaa") + x([]) + "\n" - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeString) + def test_basic(self): + """ + A ExtFuncEntry provides an annotation for a function, no need to flow + its graph. + """ + def b(x): + "NOT_RPYTHON" + return eval("x+40") + + class BTestFuncEntry(ExtFuncEntry): + _about_ = b + name = 'b' + signature_args = [annmodel.SomeInteger()] + signature_result = annmodel.SomeInteger() + + def f(): + return b(2) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + res = interpret(f, []) + assert res == 42 + + def test_lltypeimpl(self): + """ + interpret() calls lltypeimpl instead of of the function/ + """ + def c(y, x): + yyy + + class CTestFuncEntry(ExtFuncEntry): + _about_ = c + name = 'ccc' + signature_args = [annmodel.SomeInteger()] * 2 + signature_result = annmodel.SomeInteger() + + def lltypeimpl(y, x): + return y + x + lltypeimpl = staticmethod(lltypeimpl) + + def f(): + return c(3, 4) + + res = interpret(f, []) + assert res == 7 + + def test_callback(self): + """ + Verify annotation when a callback function is in the arguments list. + """ + def d(y): + return eval("y()") + + class DTestFuncEntry(ExtFuncEntry): + _about_ = d + name = 'd' + signature_args = [annmodel.SomeGenericCallable(args=[], result= + annmodel.SomeFloat())] + signature_result = annmodel.SomeFloat() + + def callback(): + return 2.5 + + def f(): + return d(callback) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeFloat) + assert a.translator._graphof(callback) + + def test_register_external_signature(self): + """ + Test the standard interface for external functions. + """ + def dd(): + pass + register_external(dd, [int], int) + + def f(): + return dd(3) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_tuple_args(self): + """ + Verify the annotation of a registered external function which takes a + tuple argument. + """ + + def function_with_tuple_arg(): + """ + Dummy function which is declared via register_external to take a + tuple as an argument so that register_external's behavior for + tuple-taking functions can be verified. + """ + register_external(function_with_tuple_arg, [(int,)], int) + + def f(): + return function_with_tuple_arg((1,)) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + + # Not a very good assertion, but at least it means _something_ happened. + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_return_goes_back(self): + """ + Check whether it works to pass the same list from one external + fun to another + [bookkeeper and list joining issues] + """ + def function_with_list(): + pass + register_external(function_with_list, [[int]], int) + + def function_returning_list(): + pass + register_external(function_returning_list, [], [int]) + + def f(): + return function_with_list(function_returning_list()) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_specialcase(self): + """ + When args=None, the external function accepts any arguments unmodified. + """ + def function_withspecialcase(arg): + return repr(arg) + register_external(function_withspecialcase, args=None, result=str) + + def f(): + x = function_withspecialcase + return x(33) + x("aaa") + x([]) + "\n" + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeString) From dan at codespeak.net Sat Jul 17 00:41:04 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Sat, 17 Jul 2010 00:41:04 +0200 (CEST) Subject: [pypy-svn] r76268 - pypy/branch/micronumpy/pypy/tool Message-ID: <20100716224104.25C49282BD4@codespeak.net> Author: dan Date: Sat Jul 17 00:41:02 2010 New Revision: 76268 Added: pypy/branch/micronumpy/pypy/tool/convolve.py Modified: pypy/branch/micronumpy/pypy/tool/numpybench.py Log: Oops, I forgot the most important part of the benchmark! Added: pypy/branch/micronumpy/pypy/tool/convolve.py ============================================================================== --- (empty file) +++ pypy/branch/micronumpy/pypy/tool/convolve.py Sat Jul 17 00:41:02 2010 @@ -0,0 +1,43 @@ +from __future__ import division +from __main__ import numpy as np + +def naive_convolve(f, g): + # f is an image and is indexed by (v, w) + # g is a filter kernel and is indexed by (s, t), + # it needs odd dimensions + # h is the output image and is indexed by (x, y), + # it is not cropped + if g.shape[0] % 2 != 1 or g.shape[1] % 2 != 1: + raise ValueError("Only odd dimensions on filter supported") + # smid and tmid are number of pixels between the center pixel + # and the edge, ie for a 5x5 filter they will be 2. + # + # The output size is calculated by adding smid, tmid to each + # side of the dimensions of the input image. + vmax = f.shape[0] + wmax = f.shape[1] + smax = g.shape[0] + tmax = g.shape[1] + smid = smax // 2 + tmid = tmax // 2 + xmax = vmax + 2*smid + ymax = wmax + 2*tmid + # Allocate result image. + h = np.zeros([xmax, ymax], dtype=f.dtype) + # Do convolution + for x in range(xmax): + for y in range(ymax): + # Calculate pixel value for h at (x,y). Sum one component + # for each pixel (s, t) of the filter g. + s_from = max(smid - x, -smid) + s_to = min((xmax - x) - smid, smid + 1) + t_from = max(tmid - y, -tmid) + t_to = min((ymax - y) - tmid, tmid + 1) + value = 0 + for s in range(s_from, s_to): + for t in range(t_from, t_to): + v = x - smid + s + w = y - tmid + t + value += g[smid - s, tmid - t] * f[v, w] + h[x, y] = value + return h Modified: pypy/branch/micronumpy/pypy/tool/numpybench.py ============================================================================== --- pypy/branch/micronumpy/pypy/tool/numpybench.py (original) +++ pypy/branch/micronumpy/pypy/tool/numpybench.py Sat Jul 17 00:41:02 2010 @@ -21,13 +21,29 @@ return numpy.array(kernel) if __name__ == '__main__': - from sys import argv as args - width, height, kwidth, kheight = [int(x) for x in args[1:]] + from optparse import OptionParser + + option_parser = OptionParser() + option_parser.add_option('--kernel-size', dest='kernel', default='3x3', + help="The size of the convolution kernel, given as WxH. ie 3x3" + "Note that both dimensions must be odd.") + option_parser.add_option('--image-size', dest='image', default='256x256', + help="The size of the image, given as WxH. ie. 256x256") + option_parser.add_option('--runs', '--count', dest='count', default=1000, + help="The number of times to run the convolution filter") + + options, args = option_parser.parse_args() + + def parse_dimension(arg): + return [int(s.strip()) for s in arg.split('x')] + + width, height = parse_dimension(options.image) + kwidth, kheight = parse_dimension(options.kernel) + count = int(options.count) image = generate_image(width, height) kernel = generate_kernel(kwidth, kheight) from timeit import Timer convolve_timer = Timer('naive_convolve(image, kernel)', 'from convolve import naive_convolve; from __main__ import image, kernel; gc.enable()') - count = 100 print "%.5f sec/pass" % (convolve_timer.timeit(number=count)/count) From agaynor at codespeak.net Sat Jul 17 04:10:39 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sat, 17 Jul 2010 04:10:39 +0200 (CEST) Subject: [pypy-svn] r76269 - in pypy/trunk/pypy/interpreter/astcompiler: . test Message-ID: <20100717021039.16C1A282B9D@codespeak.net> Author: agaynor Date: Sat Jul 17 04:10:37 2010 New Revision: 76269 Modified: pypy/trunk/pypy/interpreter/astcompiler/ast.py pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py Log: Ensure that all elements of a list comprehension are optimized by the AST optimizer. Previously wasn't turning None into a LOAD_CONST. Modified: pypy/trunk/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/ast.py Sat Jul 17 04:10:37 2010 @@ -1398,6 +1398,7 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + visitor._mutate_sequence(self.generators) return visitor.visit_ListComp(self) def sync_app_attrs(self, space): @@ -2295,6 +2296,13 @@ def walkabout(self, visitor): visitor.visit_comprehension(self) + + def mutate_over(self, visitor): + self.target = self.target.mutate_over(visitor) + self.iter = self.iter.mutate_over(visitor) + if self.ifs: + visitor._mutate_sequence(self.ifs) + return visitor.visit_comprehension(self) def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: Modified: pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py Sat Jul 17 04:10:37 2010 @@ -216,6 +216,9 @@ "l", [(2, 0), (4, 0), (5, 3), (6, 0), (7, 3), (8, 0), (8, 6), (9, 3)]) + + def test_list_comprehensions(self): + yield (self.st, "l = [x for x in range(10) if None]", "l", []) def test_genexprs(self): yield (self.st, From benjamin at codespeak.net Sat Jul 17 06:37:38 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 17 Jul 2010 06:37:38 +0200 (CEST) Subject: [pypy-svn] r76270 - in pypy/trunk/pypy/interpreter/astcompiler: . test Message-ID: <20100717043738.C508B282B9D@codespeak.net> Author: benjamin Date: Sat Jul 17 06:37:36 2010 New Revision: 76270 Modified: pypy/trunk/pypy/interpreter/astcompiler/ast.py pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py Log: revert 76269; slightly incorrect and rather untested Modified: pypy/trunk/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/ast.py Sat Jul 17 06:37:36 2010 @@ -1398,7 +1398,6 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) - visitor._mutate_sequence(self.generators) return visitor.visit_ListComp(self) def sync_app_attrs(self, space): @@ -2296,13 +2295,6 @@ def walkabout(self, visitor): visitor.visit_comprehension(self) - - def mutate_over(self, visitor): - self.target = self.target.mutate_over(visitor) - self.iter = self.iter.mutate_over(visitor) - if self.ifs: - visitor._mutate_sequence(self.ifs) - return visitor.visit_comprehension(self) def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: Modified: pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py Sat Jul 17 06:37:36 2010 @@ -216,9 +216,6 @@ "l", [(2, 0), (4, 0), (5, 3), (6, 0), (7, 3), (8, 0), (8, 6), (9, 3)]) - - def test_list_comprehensions(self): - yield (self.st, "l = [x for x in range(10) if None]", "l", []) def test_genexprs(self): yield (self.st, From hakanardo at codespeak.net Sat Jul 17 10:42:03 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 17 Jul 2010 10:42:03 +0200 (CEST) Subject: [pypy-svn] r76271 - in pypy/branch/interplevel-array/pypy: jit/metainterp jit/metainterp/test module/pypyjit/test Message-ID: <20100717084203.35F61282B9D@codespeak.net> Author: hakanardo Date: Sat Jul 17 10:42:02 2010 New Revision: 76271 Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py pypy/branch/interplevel-array/pypy/jit/metainterp/resoperation.py pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_executor.py pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py Log: bool rewriting tests and UINT support Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py Sat Jul 17 10:42:02 2010 @@ -620,36 +620,49 @@ self.emit_operation(op) + def try_boolinvers(self, op, targs): + oldop = self.pure_operations.get(targs, None) + if oldop is not None and oldop.descr is op.descr: + value = self.getvalue(oldop.result) + if value.is_constant(): + if value.box is CONST_1: + self.make_constant(op.result, CONST_0) + return True + elif value.box is CONST_0: + self.make_constant(op.result, CONST_1) + return True + return False + + def find_rewriteable_bool(self, op, args): try: oldopnum = opboolinvers[op.opnum] - targ = [args[0], args[1], ConstInt(oldopnum)] - oldop = self.pure_operations.get(targ, None) - if oldop is not None and oldop.descr is op.descr: - value = self.getvalue(oldop.result) - if value.is_constant(): - if value.box is CONST_1: - self.make_constant(op.result, CONST_0) - return True - elif value.box is CONST_0: - self.make_constant(op.result, CONST_1) - return True + targs = [args[0], args[1], ConstInt(oldopnum)] + if self.try_boolinvers(op, targs): + return True except KeyError: pass try: oldopnum = opboolreflex[op.opnum] - targ = [args[1], args[0], ConstInt(oldopnum)] - oldop = self.pure_operations.get(targ, None) + targs = [args[1], args[0], ConstInt(oldopnum)] + oldop = self.pure_operations.get(targs, None) if oldop is not None and oldop.descr is op.descr: self.make_equal_to(op.result, self.getvalue(oldop.result)) return True except KeyError: pass - # FIXME: oldopnum = opboolinvers[opboolreflex[op.opnum]]? - + try: + oldopnum = opboolinvers[opboolreflex[op.opnum]] + targs = [args[1], args[0], ConstInt(oldopnum)] + if self.try_boolinvers(op, targs): + return True + except KeyError: + pass + return False + def optimize_JUMP(self, op): orgop = self.loop.operations[-1] Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/resoperation.py Sat Jul 17 10:42:02 2010 @@ -283,6 +283,11 @@ rop.INT_GT: rop.INT_LE, rop.INT_LE: rop.INT_GT, + rop.UINT_LT: rop.UINT_GE, + rop.UINT_GE: rop.UINT_LT, + rop.UINT_GT: rop.UINT_LE, + rop.UINT_LE: rop.UINT_GT, + rop.FLOAT_EQ: rop.FLOAT_NE, rop.FLOAT_NE: rop.FLOAT_EQ, rop.FLOAT_LT: rop.FLOAT_GE, @@ -302,6 +307,11 @@ rop.INT_GT: rop.INT_LT, rop.INT_LE: rop.INT_GE, + rop.UINT_LT: rop.UINT_GT, + rop.UINT_GE: rop.UINT_LE, + rop.UINT_GT: rop.UINT_LT, + rop.UINT_LE: rop.UINT_GE, + rop.FLOAT_EQ: rop.FLOAT_EQ, rop.FLOAT_NE: rop.FLOAT_NE, rop.FLOAT_LT: rop.FLOAT_GT, Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_executor.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_executor.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_executor.py Sat Jul 17 10:42:02 2010 @@ -4,14 +4,14 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.jit.metainterp.executor import execute from pypy.jit.metainterp.executor import execute_varargs, execute_nonspec -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp.resoperation import rop, opboolinvers, opboolreflex, opname from pypy.jit.metainterp.history import BoxInt, ConstInt from pypy.jit.metainterp.history import BoxPtr, ConstPtr from pypy.jit.metainterp.history import BoxFloat, ConstFloat from pypy.jit.metainterp.history import AbstractDescr, Box from pypy.jit.metainterp import history from pypy.jit.backend.model import AbstractCPU - +from pypy.rpython.lltypesystem import llmemory, rffi class FakeDescr(AbstractDescr): pass @@ -312,3 +312,40 @@ assert box.getint() == retvalue else: assert 0, "rettype is %r" % (rettype,) + +def make_args_for_op(op, a, b): + n=opname[op] + if n[0:3] == 'INT' or n[0:4] == 'UINT': + arg1 = ConstInt(a) + arg2 = ConstInt(b) + elif n[0:5] == 'FLOAT': + arg1 = ConstFloat(float(a)) + arg2 = ConstFloat(float(b)) + elif n[0:3] == 'PTR': + arg1 = ConstPtr(rffi.cast(llmemory.GCREF, a)) + arg2 = ConstPtr(rffi.cast(llmemory.GCREF, b)) + else: + raise NotImplementedError( + "Don't know how to make args for " + n) + return arg1, arg2 + + +def test_opboolinvers(): + cpu = FakeCPU() + for op1, op2 in opboolinvers.items(): + for a in (1,2,3): + for b in (1,2,3): + arg1, arg2 = make_args_for_op(op1, a, b) + box1 = execute(cpu, None, op1, None, arg1, arg2) + box2 = execute(cpu, None, op2, None, arg1, arg2) + assert box1.value == (not box2.value) + +def test_opboolreflex(): + cpu = FakeCPU() + for op1, op2 in opboolreflex.items(): + for a in (1,2,3): + for b in (1,2,3): + arg1, arg2 = make_args_for_op(op1, a, b) + box1 = execute(cpu, None, op1, None, arg1, arg2) + box2 = execute(cpu, None, op2, None, arg2, arg1) + assert box1.value == box2.value Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py Sat Jul 17 10:42:02 2010 @@ -364,7 +364,7 @@ """ self.optimize_loop(ops, 'Not', expected) - def test_constant_boolrewrite1(self): + def test_constant_boolrewrite_lt(self): ops = """ [i0] i1 = int_lt(i0, 0) @@ -381,7 +381,7 @@ """ self.optimize_loop(ops, 'Not', expected) - def test_constant_boolrewrite2(self): + def test_constant_boolrewrite_gt(self): ops = """ [i0] i1 = int_gt(i0, 0) @@ -398,7 +398,7 @@ """ self.optimize_loop(ops, 'Not', expected) - def test_constant_boolrewrite3(self): + def test_constant_boolrewrite_reflex(self): ops = """ [i0] i1 = int_gt(i0, 0) @@ -415,6 +415,23 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_constant_boolrewrite_reflex_invers(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_ge(0, i0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + def test_remove_consecutive_guard_value_constfold(self): ops = """ [] Modified: pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py Sat Jul 17 10:42:02 2010 @@ -741,7 +741,7 @@ '''%(op1, float(a)/4.0, float(b)/4.0, op2), 109, ([], res)) def test_boolrewrite_ptr(self): - compares = ('a == b', 'b == a', 'a != b', 'a == c') + compares = ('a == b', 'b == a', 'a != b', 'b != a', 'a == c', 'c != b') for e1 in compares: for e2 in compares: a, b, c = 1, 2, 3 From hakanardo at codespeak.net Sat Jul 17 11:01:53 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 17 Jul 2010 11:01:53 +0200 (CEST) Subject: [pypy-svn] r76272 - pypy/branch/interplevel-array/pypy/jit/metainterp/test Message-ID: <20100717090153.54AFB282B9D@codespeak.net> Author: hakanardo Date: Sat Jul 17 11:01:51 2010 New Revision: 76272 Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py Log: old test that failed because of imporved optimization fixed Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py Sat Jul 17 11:01:51 2010 @@ -874,16 +874,10 @@ guard_nonnull(p0) [] i7 = ptr_ne(p0, p1) guard_true(i7) [] - i8 = ptr_eq(p0, p1) - guard_false(i8) [] i9 = ptr_ne(p0, p2) guard_true(i9) [] - i10 = ptr_eq(p0, p2) - guard_false(i10) [] i11 = ptr_ne(p2, p1) guard_true(i11) [] - i12 = ptr_eq(p2, p1) - guard_false(i12) [] jump(p0, p1, p2) """ self.optimize_loop(ops, 'Not, Not, Not', expected2) From afa at codespeak.net Sat Jul 17 12:17:56 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 17 Jul 2010 12:17:56 +0200 (CEST) Subject: [pypy-svn] r76273 - pypy/trunk/pypy/rlib Message-ID: <20100717101756.230D9282B9C@codespeak.net> Author: afa Date: Sat Jul 17 12:17:54 2010 New Revision: 76273 Modified: pypy/trunk/pypy/rlib/rwin32.py Log: Fix translation when --cc=mingw32 is used, but no Microsoft compiler is installed. Modified: pypy/trunk/pypy/rlib/rwin32.py ============================================================================== --- pypy/trunk/pypy/rlib/rwin32.py (original) +++ pypy/trunk/pypy/rlib/rwin32.py Sat Jul 17 12:17:54 2010 @@ -136,13 +136,29 @@ } return 0; }''') - exename = static_platform.compile( - [cfile], ExternalCompilationInfo(), - outputfilename = "dosmaperr", - standalone=True) - output = os.popen(str(exename)) - errors = dict(map(int, line.split()) - for line in output) + try: + exename = static_platform.compile( + [cfile], ExternalCompilationInfo(), + outputfilename = "dosmaperr", + standalone=True) + except WindowsError: + # Fallback for the mingw32 compiler + errors = { + 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, + 11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, + 22: 13, 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13, + 29: 13, 30: 13, 31: 13, 32: 13, 33: 13, 34: 13, 35: 13, + 36: 13, 53: 2, 65: 13, 67: 2, 80: 17, 82: 13, 83: 13, 89: 11, + 108: 13, 109: 32, 112: 28, 114: 9, 128: 10, 129: 10, 130: 9, + 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17, + 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8, + 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8, + 202: 8, 206: 2, 215: 11, 1816: 12, + } + else: + output = os.popen(str(exename)) + errors = dict(map(int, line.split()) + for line in output) return errors, errno.EINVAL # A bit like strerror... From afa at codespeak.net Sat Jul 17 14:24:21 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 17 Jul 2010 14:24:21 +0200 (CEST) Subject: [pypy-svn] r76274 - pypy/trunk/pypy/module/cpyext Message-ID: <20100717122421.601D7282B9C@codespeak.net> Author: afa Date: Sat Jul 17 14:24:19 2010 New Revision: 76274 Modified: pypy/trunk/pypy/module/cpyext/methodobject.py Log: Don't crash when ml_doc is left to NULL. It happens when property() is called with a getter written in C. Modified: pypy/trunk/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/methodobject.py (original) +++ pypy/trunk/pypy/module/cpyext/methodobject.py Sat Jul 17 14:24:19 2010 @@ -100,7 +100,11 @@ return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) def get_doc(space, self): - return space.wrap(rffi.charp2str(self.ml.c_ml_doc)) + doc = self.ml.c_ml_doc + if doc: + return space.wrap(rffi.charp2str(doc)) + else: + return space.w_None class W_PyCMethodObject(W_PyCFunctionObject): From hakanardo at codespeak.net Sat Jul 17 15:56:46 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 17 Jul 2010 15:56:46 +0200 (CEST) Subject: [pypy-svn] r76275 - pypy/branch/interplevel-array/pypy/module/pypyjit/test Message-ID: <20100717135646.E12B2282B9C@codespeak.net> Author: hakanardo Date: Sat Jul 17 15:56:45 2010 New Revision: 76275 Modified: pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py Log: adjusted Modified: pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py Sat Jul 17 15:56:45 2010 @@ -757,7 +757,7 @@ if 'c' in e1 or 'c' in e2: - n = 245 + n = 337 else: n = 215 From afa at codespeak.net Mon Jul 19 01:10:33 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Jul 2010 01:10:33 +0200 (CEST) Subject: [pypy-svn] r76277 - pypy/extradoc/talk/ep2010/talk Message-ID: <20100718231033.0F37E282BAD@codespeak.net> Author: afa Date: Mon Jul 19 01:10:31 2010 New Revision: 76277 Modified: pypy/extradoc/talk/ep2010/talk/talk.rst Log: Save intermediate version, needs a lot of formatting Modified: pypy/extradoc/talk/ep2010/talk/talk.rst ============================================================================== --- pypy/extradoc/talk/ep2010/talk/talk.rst (original) +++ pypy/extradoc/talk/ep2010/talk/talk.rst Mon Jul 19 01:10:31 2010 @@ -470,15 +470,58 @@ - ``pypy-c setup.py build`` +- included in PyPy 1.3 + - still beta -- not 100% of CPython API is supported +- 50% of the CPython API is supported + + * enough for 90% of extension modules + +features +-------- + +- C API written in Python! + +- Testable on top of an interpreted py.py + +- Source compatibility + + * PyString_AS_STRING is actually a function call + +implementation +-------------- + +- Was not supposed to work + + * different garbage collector + + * no "borrowed reference" + + * all the PyTypeObject slots + +- Written on top of the object space:: + + @cpython_api([PyObject], Py_ssize_t, error=-1) + def PyDict_Size(space, w_obj): + return space.int_w(space.len(w_obj)) + +- *not* faster than python code! + +The Reference Counting Issue +---------------------------- + +- pypy uses a moving garbage collector + +- -- not included in PyPy 1.2 -- Known to work: +suppported modules +------------------ - * wxPython (after a patch) +- Known to work (after small patches): + + * wxPython * _sre @@ -486,6 +529,8 @@ * PIL + * cx_Oracle + wxPython on PyPy (1) --------------------- @@ -496,13 +541,56 @@ wxPython on PyPy (2) --------------------- - .. image:: wxpython2.png :scale: 30 +performance +----------- + +Test script:: + + from cx_Oracle import connect + c = connect('scott/tiger at db') + cur = c.cursor() + var = cur.var(STRING) + + def f(): + for i in range(10000): + var.setvalue(0, str(i)) + var.getvalue(0) + +Time it:: + + python -m timeit -s "from test_oracle import f" "f()" + cpython2.6: 8.25 msec per loop + pypy-c: 161 msec per loop + pypy-c-jit: 121 msec per loop + +Compare with:: + def f(): + for i in range(10000): + x = str(i) + y = int(x) + cpython2.6: 8.18 msec per loop + pypy-c-jit: 1.22 msec per loop +Future developments +------------------- + +- Some care about speed + +- Fill missing API functions + +- Better suppport of the PyTypeObject slots + + * don't try to call the base class' slot! + +- Think about threads and the GIL + +- Think about reference cycles + Contact / Q&A -------------- @@ -511,6 +599,10 @@ * Armin Rigo: arigo (at) tunes.org +* Amaury Forgeot d'Arc: amauryfa (at) gmail + +* And the #pypy channel! + * Links: - PyPy: http://pypy.org/ From afa at codespeak.net Mon Jul 19 01:20:13 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Jul 2010 01:20:13 +0200 (CEST) Subject: [pypy-svn] r76278 - pypy/extradoc/talk/ep2010/talk Message-ID: <20100718232013.B168F282BAD@codespeak.net> Author: afa Date: Mon Jul 19 01:20:12 2010 New Revision: 76278 Modified: pypy/extradoc/talk/ep2010/talk/talk.rst Log: Describe more cpyext details Modified: pypy/extradoc/talk/ep2010/talk/talk.rst ============================================================================== --- pypy/extradoc/talk/ep2010/talk/talk.rst (original) +++ pypy/extradoc/talk/ep2010/talk/talk.rst Mon Jul 19 01:20:12 2010 @@ -508,13 +508,36 @@ - *not* faster than python code! +- PyObject contains ob_type and ob_refcnt + + * The "abstract object interface" is used. + +- Some objects contain more: + + * PyString_AsString() must keep the buffer alive at a fixed location + + * PyTypeObject exposes all its fields + + The Reference Counting Issue ---------------------------- -- pypy uses a moving garbage collector +- pypy uses a moving garbage collector, starts with static roots to + find objects. + +- CPython objects don't move, and PyObject* can point to deallocated + memory. + +- cpyext builds PyObject as proxies to the "real" interpreter objects + +- one dictionary lookup each time the boundary is crossed + +- More tricks needed for borrowing references -- + * The object lifetime is tied to its container. + * "out of nothing" borrowed references are kept until the end of the + current pypy->C call. suppported modules ------------------ From arigo at codespeak.net Mon Jul 19 11:14:25 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Jul 2010 11:14:25 +0200 (CEST) Subject: [pypy-svn] r76279 - pypy/extradoc/talk/ep2010/talk Message-ID: <20100719091425.1915D282B9E@codespeak.net> Author: arigo Date: Mon Jul 19 11:14:23 2010 New Revision: 76279 Added: pypy/extradoc/talk/ep2010/talk/overview3.png (contents, props changed) Modified: pypy/extradoc/talk/ep2010/talk/talk.rst Log: Add another image. Added: pypy/extradoc/talk/ep2010/talk/overview3.png ============================================================================== Binary file. No diff available. Modified: pypy/extradoc/talk/ep2010/talk/talk.rst ============================================================================== --- pypy/extradoc/talk/ep2010/talk/talk.rst (original) +++ pypy/extradoc/talk/ep2010/talk/talk.rst Mon Jul 19 11:14:23 2010 @@ -362,6 +362,13 @@ :scale: 60 +The architecture of PyPy (2) +---------------------------- + +.. image:: overview3.png + :scale: 60 + + Speed of the PyPy JIT --------------------- From arigo at codespeak.net Mon Jul 19 11:22:26 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Jul 2010 11:22:26 +0200 (CEST) Subject: [pypy-svn] r76280 - pypy/extradoc/talk/ep2010/talk Message-ID: <20100719092226.537B5282B9E@codespeak.net> Author: arigo Date: Mon Jul 19 11:22:24 2010 New Revision: 76280 Modified: pypy/extradoc/talk/ep2010/talk/overview3.png Log: Rename "static bytecode" into "static jitcode". Modified: pypy/extradoc/talk/ep2010/talk/overview3.png ============================================================================== Binary files. No diff available. From antocuni at codespeak.net Mon Jul 19 11:44:24 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 19 Jul 2010 11:44:24 +0200 (CEST) Subject: [pypy-svn] r76281 - pypy/extradoc/talk/ep2010/talk Message-ID: <20100719094424.8D9B5282B9E@codespeak.net> Author: antocuni Date: Mon Jul 19 11:44:22 2010 New Revision: 76281 Modified: pypy/extradoc/talk/ep2010/talk/talk.rst Log: (antocuni, amaury, arigo) last minute changes Modified: pypy/extradoc/talk/ep2010/talk/talk.rst ============================================================================== --- pypy/extradoc/talk/ep2010/talk/talk.rst (original) +++ pypy/extradoc/talk/ep2010/talk/talk.rst Mon Jul 19 11:44:22 2010 @@ -492,14 +492,26 @@ - Testable on top of an interpreted py.py +- Written on top of the object space + - Source compatibility - * PyString_AS_STRING is actually a function call + * PyString_AS_STRING is actually a function call (instead of a macro) + +|small| + +.. sourcecode:: python + + @cpython_api([PyObject], Py_ssize_t, error=-1) + def PyDict_Size(space, w_obj): + return space.int_w(space.len(w_obj)) + +|end_small| implementation -------------- -- Was not supposed to work +- It was not supposed to work! * different garbage collector @@ -507,12 +519,6 @@ * all the PyTypeObject slots -- Written on top of the object space:: - - @cpython_api([PyObject], Py_ssize_t, error=-1) - def PyDict_Size(space, w_obj): - return space.int_w(space.len(w_obj)) - - *not* faster than python code! - PyObject contains ob_type and ob_refcnt @@ -546,7 +552,10 @@ * "out of nothing" borrowed references are kept until the end of the current pypy->C call. -suppported modules + + + +supported modules ------------------ - Known to work (after small patches): @@ -561,6 +570,32 @@ * cx_Oracle + * MySQLdb + + * sqlite + + +Why your module will crash +--------------------------- + +Likely: + +|small| + +.. sourcecode:: c + + static PyObject *myException; + + void init_foo() { + myException = PyException_New(...); + Py_AddModule(m, myException); // steals a reference + } + + { + PyErr_SetString(myException, "message"); // crash + } + +|end_small| wxPython on PyPy (1) --------------------- @@ -574,12 +609,14 @@ .. image:: wxpython2.png :scale: 30 -performance ------------ +performance (1) +--------------- -Test script:: +|small| - from cx_Oracle import connect +.. sourcecode:: python + + from cx_Oracle import connect, STRING c = connect('scott/tiger at db') cur = c.cursor() var = cur.var(STRING) @@ -589,33 +626,48 @@ var.setvalue(0, str(i)) var.getvalue(0) -Time it:: + +.. sourcecode:: bash python -m timeit -s "from test_oracle import f" "f()" - cpython2.6: 8.25 msec per loop + python2.6: 8.25 msec per loop pypy-c: 161 msec per loop pypy-c-jit: 121 msec per loop -Compare with:: +|end_small| + +performance (2) +--------------- + + +Compare with: + +|small| + +.. sourcecode:: python def f(): for i in range(10000): x = str(i) y = int(x) - cpython2.6: 8.18 msec per loop + +.. sourcecode:: bash + + python2.6: 8.18 msec per loop pypy-c-jit: 1.22 msec per loop +|end_small| Future developments ------------------- - Some care about speed -- Fill missing API functions + * The JIT can help to remove (some of) the overhead -- Better suppport of the PyTypeObject slots +- Fill missing API functions (when needed) - * don't try to call the base class' slot! +- Better suppport of the PyTypeObject slots - Think about threads and the GIL @@ -631,7 +683,7 @@ * Amaury Forgeot d'Arc: amauryfa (at) gmail -* And the #pypy channel! +* And the #pypy IRC channel on freenode.net! * Links: From fijal at codespeak.net Mon Jul 19 19:41:21 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Jul 2010 19:41:21 +0200 (CEST) Subject: [pypy-svn] r76282 - pypy/branch/interplevel-array/pypy/module/_demo Message-ID: <20100719174121.A6112282B9E@codespeak.net> Author: fijal Date: Mon Jul 19 19:41:18 2010 New Revision: 76282 Added: pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py Modified: pypy/branch/interplevel-array/pypy/module/_demo/__init__.py pypy/branch/interplevel-array/pypy/module/_demo/demo.py Log: A shot at multimethods for user-supplied types. Not working so far Modified: pypy/branch/interplevel-array/pypy/module/_demo/__init__.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_demo/__init__.py (original) +++ pypy/branch/interplevel-array/pypy/module/_demo/__init__.py Mon Jul 19 19:41:18 2010 @@ -4,13 +4,10 @@ """A demo built-in module based on ctypes.""" interpleveldefs = { - 'measuretime' : 'demo.measuretime', - 'sieve' : 'demo.sieve', - 'MyType' : 'demo.W_MyType', + 'tp' : 'interp_demo.w_type', } appleveldefs = { - 'DemoError' : 'app_demo.DemoError', } # Used in tests Modified: pypy/branch/interplevel-array/pypy/module/_demo/demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_demo/demo.py (original) +++ pypy/branch/interplevel-array/pypy/module/_demo/demo.py Mon Jul 19 19:41:18 2010 @@ -6,69 +6,3 @@ from pypy.rpython.tool import rffi_platform import sys, math -time_t = rffi_platform.getsimpletype('time_t', '#include ', rffi.LONG) - -time = rffi.llexternal('time', [rffi.VOIDP], time_t, includes=['time.h']) - -def get(space, name): - w_module = space.getbuiltinmodule('_demo') - return space.getattr(w_module, space.wrap(name)) - - -def measuretime(space, repetitions, w_callable): - if repetitions <= 0: - w_DemoError = get(space, 'DemoError') - msg = "repetition count must be > 0" - raise OperationError(w_DemoError, space.wrap(msg)) - starttime = time(None) - for i in range(repetitions): - space.call_function(w_callable) - endtime = time(None) - return space.wrap(endtime - starttime) -measuretime.unwrap_spec = [ObjSpace, int, W_Root] - -def sieve(space, n): - lst = range(2, n + 1) - head = 0 - while 1: - first = lst[head] - if first > math.sqrt(n) + 1: - lst_w = [space.newint(i) for i in lst] - return space.newlist(lst_w) - newlst = [] - for element in lst: - if element <= first: - newlst.append(element) - elif element % first != 0: - newlst.append(element) - lst = newlst - head += 1 -sieve.unwrap_spec = [ObjSpace, int] - -class W_MyType(Wrappable): - def __init__(self, space, x=1): - self.space = space - self.x = x - - def multiply(self, w_y): - space = self.space - y = space.int_w(w_y) - return space.wrap(self.x * y) - - def fget_x(space, self): - return space.wrap(self.x) - - def fset_x(space, self, w_value): - self.x = space.int_w(w_value) - -def mytype_new(space, w_subtype, x): - return space.wrap(W_MyType(space, x)) -mytype_new.unwrap_spec = [ObjSpace, W_Root, int] - -getset_x = GetSetProperty(W_MyType.fget_x, W_MyType.fset_x, cls=W_MyType) - -W_MyType.typedef = TypeDef('MyType', - __new__ = interp2app(mytype_new), - x = getset_x, - multiply = interp2app(W_MyType.multiply), -) Added: pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py Mon Jul 19 19:41:18 2010 @@ -0,0 +1,31 @@ + +from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable +from pypy.objspace.std.stdtypedef import SMM, StdTypeDef +from pypy.objspace.std.register_all import register_all + +def w_type(space, arg): + if arg == 0: + return space.wrap(W_Zero()) + else: + return space.wrap(W_One()) +w_type.unwrap_spec = [ObjSpace, int] + +type_repr = SMM('__repr__', 1, 'a docstring') + +type_typedef = StdTypeDef("tp", + __new__ = w_type) +type_typedef.registermethods(globals()) + +class W_Zero(Wrappable): + typedef = type_typedef + +class W_One(Wrappable): + typedef = type_typedef + +def repr__Zero(space, w_zero): + return space.wrap("zero") + +def repr__One(space, w_one): + return space.wrap("one") + +register_all(locals(), globals()) From wlav at codespeak.net Mon Jul 19 19:56:57 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Mon, 19 Jul 2010 19:56:57 +0200 (CEST) Subject: [pypy-svn] r76283 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src test Message-ID: <20100719175657.6BE8B282B9E@codespeak.net> Author: wlav Date: Mon Jul 19 19:56:55 2010 New Revision: 76283 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Log: Initial converters/executors for char and unsigned char. Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Mon Jul 19 19:56:55 2010 @@ -49,6 +49,10 @@ "cppyy_call_b", [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.INT, compilation_info=eci) +c_call_c = rffi.llexternal( + "cppyy_call_c", + [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.CHAR, + compilation_info=eci) c_call_l = rffi.llexternal( "cppyy_call_l", [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.LONG, Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Mon Jul 19 19:56:55 2010 @@ -31,6 +31,15 @@ x[0] = arg return rffi.cast(rffi.VOIDP, x) +class CharConverter(TypeConverter): + def convert_argument(self, space, w_obj): + arg = space.str_w(w_obj) + if len(arg) != 1: + raise OperationError(space.w_TypeError, + space.wrap("char expecter, got string of size %d" % len(arg))) + x = rffi.str2charp(arg) + return rffi.cast(rffi.VOIDP, x) + class IntConverter(TypeConverter): def convert_argument(self, space, w_obj): arg = space.c_int_w(w_obj) @@ -100,6 +109,8 @@ _converters["bool"] = BoolConverter() +_converters["char"] = CharConverter() +_converters["unsigned char"] = CharConverter() _converters["int"] = IntConverter() _converters["double"] = DoubleConverter() _converters["const char*"] = CStringConverter() Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Mon Jul 19 19:56:55 2010 @@ -20,6 +20,11 @@ result = capi.c_call_b(func.cpptype.handle, func.method_index, cppthis, num_args, args) return space.wrap(result) +class CharExecutor(FunctionExecutor): + def execute(self, space, func, cppthis, num_args, args): + result = capi.c_call_c(func.cpptype.handle, func.method_index, cppthis, num_args, args) + return space.wrap(result) + class LongExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): result = capi.c_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) @@ -69,6 +74,8 @@ _executors["void"] = VoidExecutor() _executors["bool"] = BoolExecutor() +_executors["char"] = CharExecutor() +_executors["unsigned char"] = CharExecutor() _executors["int"] = LongExecutor() _executors["long int"] = LongExecutor() _executors["double"] = DoubleExecutor() Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Mon Jul 19 19:56:55 2010 @@ -16,6 +16,7 @@ void cppyy_call_v(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); int cppyy_call_b(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); + char cppyy_call_c(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); long cppyy_call_l(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); double cppyy_call_d(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); void cppyy_destruct(cppyy_typehandle_t handle, cppyy_object_t self); Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Mon Jul 19 19:56:55 2010 @@ -52,6 +52,11 @@ return (int)cppyy_call_T(handle, method_index, self, numargs, args); } +char cppyy_call_c(cppyy_typehandle_t handle, int method_index, + cppyy_object_t self, int numargs, void* args[]) { + return cppyy_call_T(handle, method_index, self, numargs, args); +} + long cppyy_call_l(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]) { return cppyy_call_T(handle, method_index, self, numargs, args); @@ -88,16 +93,16 @@ int cppyy_num_methods(cppyy_typehandle_t handle) { Reflex::Type t((Reflex::TypeName*)handle); - for (int i = 0; i < (int)t.FunctionMemberSize(); i++) { - Reflex::Member m = t.FunctionMemberAt(i); - std::cout << i << " " << m.Name() << std::endl; - std::cout << " " << "Stubfunction: " << (void*)m.Stubfunction() << std::endl; - std::cout << " " << "MethPtrGetter: " << (void*)get_methptr_getter(m) << std::endl; - for (int j = 0; j < (int)m.FunctionParameterSize(); j++) { - Reflex::Type at = m.TypeOf().FunctionParameterAt(j); - std::cout << " " << j << " " << at.Name() << std::endl; - } - } + // for (int i = 0; i < (int)t.FunctionMemberSize(); i++) { + // Reflex::Member m = t.FunctionMemberAt(i); + // std::cout << i << " " << m.Name() << std::endl; + // std::cout << " " << "Stubfunction: " << (void*)m.Stubfunction() << std::endl; + // std::cout << " " << "MethPtrGetter: " << (void*)get_methptr_getter(m) << std::endl; + // for (int j = 0; j < (int)m.FunctionParameterSize(); j++) { + // Reflex::Type at = m.TypeOf().FunctionParameterAt(j); + // std::cout << " " << j << " " << at.Name() << std::endl; + // } + // } return t.FunctionMemberSize(); } Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Mon Jul 19 19:56:55 2010 @@ -53,6 +53,8 @@ """ raises(TypeError, 'c.set_bool(10)') + c.set_char('c'); assert c.get_char() == 'c' + c.set_uchar('e'); assert c.get_uchar() == 'e' """ # char types c.m_char = 'b'; assert c.get_char() == 'b' @@ -63,11 +65,13 @@ c.m_uchar = 42; assert c.get_uchar() == chr(42) c.set_uchar('e'); assert c.m_uchar == 'e' c.set_uchar(43); assert c.m_uchar == chr(43) + """ raises(TypeError, 'c.set_char("string")') - raises(TypeError, 'c.set_uchar(-1)') +# raises(TypeError, 'c.set_uchar(-1)') raises(TypeError, 'c.set_uchar("string")') + """ # integer types names = ['short', 'ushort', 'int', 'uint', 'long', 'ulong'] for i in range(len(names)): From fijal at codespeak.net Mon Jul 19 20:41:54 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Jul 2010 20:41:54 +0200 (CEST) Subject: [pypy-svn] r76284 - in pypy/branch/interplevel-array/pypy: module/_demo module/_demo/test objspace/std Message-ID: <20100719184154.8CFFB282B9E@codespeak.net> Author: fijal Date: Mon Jul 19 20:41:52 2010 New Revision: 76284 Added: pypy/branch/interplevel-array/pypy/module/_demo/test/test_demo.py Modified: pypy/branch/interplevel-array/pypy/module/_demo/__init__.py pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py pypy/branch/interplevel-array/pypy/objspace/std/model.py Log: Necessary changes for multimethods to work here Modified: pypy/branch/interplevel-array/pypy/module/_demo/__init__.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_demo/__init__.py (original) +++ pypy/branch/interplevel-array/pypy/module/_demo/__init__.py Mon Jul 19 20:41:52 2010 @@ -1,4 +1,9 @@ -from pypy.interpreter.mixedmodule import MixedModule +from pypy.interpreter.mixedmodule import MixedModule +from pypy.module._demo.interp_demo import W_Zero, W_One +from pypy.objspace.std.model import registerimplementation + +registerimplementation(W_Zero) +registerimplementation(W_One) class Module(MixedModule): """A demo built-in module based on ctypes.""" Modified: pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py (original) +++ pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py Mon Jul 19 20:41:52 2010 @@ -2,12 +2,13 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable from pypy.objspace.std.stdtypedef import SMM, StdTypeDef from pypy.objspace.std.register_all import register_all +from pypy.objspace.std.model import W_Object def w_type(space, arg): if arg == 0: - return space.wrap(W_Zero()) + return W_Zero() else: - return space.wrap(W_One()) + return W_One() w_type.unwrap_spec = [ObjSpace, int] type_repr = SMM('__repr__', 1, 'a docstring') @@ -16,10 +17,18 @@ __new__ = w_type) type_typedef.registermethods(globals()) -class W_Zero(Wrappable): +class W_Zero(W_Object): + @staticmethod + def register(typeorder): + typeorder[W_Zero] = [] + typedef = type_typedef -class W_One(Wrappable): +class W_One(W_Object): + @staticmethod + def register(typeorder): + typeorder[W_One] = [] + typedef = type_typedef def repr__Zero(space, w_zero): Added: pypy/branch/interplevel-array/pypy/module/_demo/test/test_demo.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/_demo/test/test_demo.py Mon Jul 19 20:41:52 2010 @@ -0,0 +1,11 @@ + +from pypy.conftest import gettestobjspace + +class AppTestDemo(object): + def setup_class(cls): + cls.space = gettestobjspace(usemodules = ['_demo']) + + def test_one(self): + import _demo + assert repr(_demo.tp(1)) == 'one' + assert repr(_demo.tp(0)) == 'zero' Modified: pypy/branch/interplevel-array/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/interplevel-array/pypy/objspace/std/model.py (original) +++ pypy/branch/interplevel-array/pypy/objspace/std/model.py Mon Jul 19 20:41:52 2010 @@ -140,6 +140,8 @@ # check if we missed implementations for implcls in _registered_implementations: + if hasattr(implcls, 'register'): + implcls.register(self.typeorder) assert (implcls in self.typeorder or implcls in self.imported_but_not_registered), ( "please add %r in StdTypeModel.typeorder" % (implcls,)) From agaynor at codespeak.net Mon Jul 19 21:33:50 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Mon, 19 Jul 2010 21:33:50 +0200 (CEST) Subject: [pypy-svn] r76285 - in pypy/trunk/pypy/interpreter: astcompiler astcompiler/tools test Message-ID: <20100719193350.ADE97282BE8@codespeak.net> Author: agaynor Date: Mon Jul 19 21:33:47 2010 New Revision: 76285 Modified: pypy/trunk/pypy/interpreter/astcompiler/ast.py pypy/trunk/pypy/interpreter/astcompiler/codegen.py pypy/trunk/pypy/interpreter/astcompiler/symtable.py pypy/trunk/pypy/interpreter/astcompiler/tools/asdl_py.py pypy/trunk/pypy/interpreter/test/test_compiler.py Log: Ensure that things in the if clause of a list comprehension are optimized by the AST optimizer. Modified: pypy/trunk/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/ast.py Mon Jul 19 21:33:47 2010 @@ -230,6 +230,7 @@ visitor.visit_FunctionDef(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) if self.body: visitor._mutate_sequence(self.body) if self.decorators: @@ -784,6 +785,8 @@ def mutate_over(self, visitor): if self.body: visitor._mutate_sequence(self.body) + if self.handlers: + visitor._mutate_sequence(self.handlers) if self.orelse: visitor._mutate_sequence(self.orelse) return visitor.visit_TryExcept(self) @@ -927,6 +930,8 @@ visitor.visit_Import(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_Import(self) def sync_app_attrs(self, space): @@ -965,6 +970,8 @@ visitor.visit_ImportFrom(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_ImportFrom(self) def sync_app_attrs(self, space): @@ -1282,6 +1289,7 @@ visitor.visit_Lambda(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) self.body = self.body.mutate_over(visitor) return visitor.visit_Lambda(self) @@ -1398,6 +1406,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_ListComp(self) def sync_app_attrs(self, space): @@ -1437,6 +1447,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_GeneratorExp(self) def sync_app_attrs(self, space): @@ -1562,6 +1574,8 @@ self.func = self.func.mutate_over(visitor) if self.args: visitor._mutate_sequence(self.args) + if self.keywords: + visitor._mutate_sequence(self.keywords) if self.starargs: self.starargs = self.starargs.mutate_over(visitor) if self.kwargs: @@ -2293,6 +2307,13 @@ self.w_ifs = None self.initialization_state = 7 + def mutate_over(self, visitor): + self.target = self.target.mutate_over(visitor) + self.iter = self.iter.mutate_over(visitor) + if self.ifs: + visitor._mutate_sequence(self.ifs) + return visitor.visit_comprehension(self) + def walkabout(self, visitor): visitor.visit_comprehension(self) @@ -2327,6 +2348,15 @@ self.col_offset = col_offset self.initialization_state = 31 + def mutate_over(self, visitor): + if self.type: + self.type = self.type.mutate_over(visitor) + if self.name: + self.name = self.name.mutate_over(visitor) + if self.body: + visitor._mutate_sequence(self.body) + return visitor.visit_excepthandler(self) + def walkabout(self, visitor): visitor.visit_excepthandler(self) @@ -2366,6 +2396,13 @@ self.w_defaults = None self.initialization_state = 15 + def mutate_over(self, visitor): + if self.args: + visitor._mutate_sequence(self.args) + if self.defaults: + visitor._mutate_sequence(self.defaults) + return visitor.visit_arguments(self) + def walkabout(self, visitor): visitor.visit_arguments(self) @@ -2407,6 +2444,10 @@ self.value = value self.initialization_state = 3 + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + return visitor.visit_keyword(self) + def walkabout(self, visitor): visitor.visit_keyword(self) @@ -2426,6 +2467,9 @@ self.asname = asname self.initialization_state = 3 + def mutate_over(self, visitor): + return visitor.visit_alias(self) + def walkabout(self, visitor): visitor.visit_alias(self) Modified: pypy/trunk/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/codegen.py Mon Jul 19 21:33:47 2010 @@ -258,9 +258,11 @@ # Load decorators first, but apply them after the function is created. if func.decorators: self.visit_sequence(func.decorators) - if func.args.defaults: - self.visit_sequence(func.args.defaults) - num_defaults = len(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + num_defaults = len(args.defaults) else: num_defaults = 0 code = self.sub_scope(FunctionCodeGenerator, func.name, func, @@ -274,9 +276,11 @@ def visit_Lambda(self, lam): self.update_position(lam.lineno) - if lam.args.defaults: - self.visit_sequence(lam.args.defaults) - default_count = len(lam.args.defaults) + args = lam.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + default_count = len(args.defaults) else: default_count = 0 code = self.sub_scope(LambdaCodeGenerator, "", lam, lam.lineno) @@ -1275,9 +1279,11 @@ else: self.add_const(self.space.w_None) start = 0 - if func.args.args: - self._handle_nested_args(func.args.args) - self.argcount = len(func.args.args) + args = func.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) for i in range(start, len(func.body)): func.body[i].walkabout(self) @@ -1286,9 +1292,11 @@ def _compile(self, lam): assert isinstance(lam, ast.Lambda) - if lam.args.args: - self._handle_nested_args(lam.args.args) - self.argcount = len(lam.args.args) + args = lam.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) # Prevent a string from being the first constant and thus a docstring. self.add_const(self.space.w_None) lam.body.walkabout(self) Modified: pypy/trunk/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/symtable.py Mon Jul 19 21:33:47 2010 @@ -353,8 +353,10 @@ def visit_FunctionDef(self, func): self.note_symbol(func.name, SYM_ASSIGNED) # Function defaults and decorators happen in the outer scope. - if func.args.defaults: - self.visit_sequence(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) if func.decorators: self.visit_sequence(func.decorators) new_scope = FunctionScope(func.name, func.lineno, func.col_offset) @@ -420,8 +422,10 @@ self.note_symbol(name, SYM_GLOBAL) def visit_Lambda(self, lamb): - if lamb.args.defaults: - self.visit_sequence(lamb.args.defaults) + args = lamb.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) new_scope = FunctionScope("lambda", lamb.lineno, lamb.col_offset) self.push_scope(new_scope, lamb) lamb.args.walkabout(self) Modified: pypy/trunk/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/tools/asdl_py.py Mon Jul 19 21:33:47 2010 @@ -100,6 +100,7 @@ self.emit("") self.make_constructor(product.fields, product) self.emit("") + self.make_mutate_over(product, name) self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (name,), 2) self.emit("") @@ -183,6 +184,26 @@ have_everything = self.data.required_masks[node] | \ self.data.optional_masks[node] self.emit("self.initialization_state = %i" % (have_everything,), 2) + + def make_mutate_over(self, cons, name): + self.emit("def mutate_over(self, visitor):", 1) + for field in cons.fields: + if (field.type.value not in asdl.builtin_types and + field.type.value not in self.data.simple_types): + if field.opt or field.seq: + level = 3 + self.emit("if self.%s:" % (field.name,), 2) + else: + level = 2 + if field.seq: + sub = (field.name,) + self.emit("visitor._mutate_sequence(self.%s)" % sub, level) + else: + sub = (field.name, field.name) + self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, + level) + self.emit("return visitor.visit_%s(self)" % (name,), 2) + self.emit("") def visitConstructor(self, cons, base, extra_attributes): self.emit("class %s(%s):" % (cons.name, base)) @@ -199,24 +220,7 @@ self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (cons.name,), 2) self.emit("") - self.emit("def mutate_over(self, visitor):", 1) - for field in cons.fields: - if field.type.value not in asdl.builtin_types and \ - field.type.value not in self.data.prod_simple: - if field.opt or field.seq: - level = 3 - self.emit("if self.%s:" % (field.name,), 2) - else: - level = 2 - if field.seq: - sub = (field.name,) - self.emit("visitor._mutate_sequence(self.%s)" % sub, level) - else: - sub = (field.name, field.name) - self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, - level) - self.emit("return visitor.visit_%s(self)" % (cons.name,), 2) - self.emit("") + self.make_mutate_over(cons, cons.name) self.make_var_syncer(cons.fields + self.data.cons_attributes[cons], cons, cons.name) Modified: pypy/trunk/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_compiler.py (original) +++ pypy/trunk/pypy/interpreter/test/test_compiler.py Mon Jul 19 21:33:47 2010 @@ -838,6 +838,23 @@ sys.stdout = save_stdout output = s.getvalue() assert "STOP_CODE" not in output + + def test_optimize_list_comp(self): + source = """def _f(a): + return [x for x in a if None] + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = sys.__stdout__ + output = s.getvalue() + assert "LOAD_GLOBAL" not in output class AppTestExceptions: def test_indentation_error(self): From jcreigh at codespeak.net Mon Jul 19 21:39:57 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 19 Jul 2010 21:39:57 +0200 (CEST) Subject: [pypy-svn] r76286 - in pypy/branch/asmgcc-64/pypy: rpython/memory/gctransform translator/c/gcc translator/c/gcc/test translator/c/gcc/test/elf64 Message-ID: <20100719193957.5C37C282BE8@codespeak.net> Author: jcreigh Date: Mon Jul 19 21:39:55 2010 New Revision: 76286 Added: pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/ pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_32bit_reg_zeroextend.s pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_basic_argument_registers.s pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_jumptable.c pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_jumptable.s pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_negative_rsp_offset.s pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_varargs_function.s Modified: pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py pypy/branch/asmgcc-64/pypy/translator/c/gcc/instruction.py pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/conftest.py pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py Log: first attempt at a 64-bit asmgcc. very rough, breaks 32-bit support in some places for now Modified: pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py Mon Jul 19 21:39:55 2010 @@ -326,7 +326,7 @@ ll_assert(reg < CALLEE_SAVED_REGS, "bad register location") return callee.regs_stored_at[reg] elif kind == LOC_ESP_PLUS: # in the caller stack frame at N(%esp) - esp_in_caller = callee.frame_address + 4 + esp_in_caller = callee.frame_address + sizeofaddr return esp_in_caller + offset elif kind == LOC_EBP_PLUS: # in the caller stack frame at N(%ebp) ebp_in_caller = callee.regs_stored_at[INDEX_OF_EBP].address[0] @@ -465,9 +465,16 @@ # - frame address (actually the addr of the retaddr of the current function; # that's the last word of the frame in memory) # +""" CALLEE_SAVED_REGS = 4 # there are 4 callee-saved registers INDEX_OF_EBP = 3 FRAME_PTR = CALLEE_SAVED_REGS # the frame is at index 4 in the array +""" + +# FIXME: hard-coded for 64-bit +CALLEE_SAVED_REGS = 6 +INDEX_OF_EBP = 5 +FRAME_PTR = CALLEE_SAVED_REGS ASM_CALLBACK_PTR = lltype.Ptr(lltype.FuncType([], lltype.Void)) Modified: pypy/branch/asmgcc-64/pypy/translator/c/gcc/instruction.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/gcc/instruction.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/instruction.py Mon Jul 19 21:39:55 2010 @@ -5,6 +5,16 @@ LOC_MASK = 0x03 LOC_NOWHERE = LOC_REG | 0 +# XXX: Although it simplifies things, it's ugly to mix x86-64 and x86-32 +# registers in the same tuple +ARGUMENT_REGISTERS = ( + # x86-64 registers used to pass arguments + ('%rdi', '%rsi', '%rdx', '%rcx', '%r8', '%r9') + + # x86-32 registers sometimes used to pass arguments when gcc optimizes + # a function's calling convention + ('%eax', '%edx', '%ecx') +) + def frameloc_esp(offset): assert offset >= 0 assert offset % 4 == 0 @@ -19,7 +29,8 @@ class SomeNewValue(object): - pass + def __repr__(self): + return 'somenewvalue' somenewvalue = SomeNewValue() class LocalVar(object): @@ -42,7 +53,7 @@ else: return 1 - def getlocation(self, framesize, uses_frame_pointer): + def getlocation(self, framesize, uses_frame_pointer, wordsize): if (self.hint == 'esp' or not uses_frame_pointer or self.ofs_from_frame_end % 2 != 0): # try to use esp-relative addressing @@ -52,7 +63,7 @@ # we can get an odd value if the framesize is marked as bogus # by visit_andl() assert uses_frame_pointer - ofs_from_ebp = self.ofs_from_frame_end + 4 + ofs_from_ebp = self.ofs_from_frame_end + wordsize return frameloc_ebp(ofs_from_ebp) @@ -81,6 +92,7 @@ self.previous_insns = [] # all insns that jump (or fallthrough) here class InsnFunctionStart(Insn): + _args_ = ['arguments'] framesize = 0 previous_insns = () def __init__(self, registers): @@ -90,7 +102,7 @@ def source_of(self, localvar, tag): if localvar not in self.arguments: - if localvar in ('%eax', '%edx', '%ecx'): + if localvar in ARGUMENT_REGISTERS: # xxx this might show a bug in trackgcroot.py failing to # figure out which instruction stored a value in these # registers. However, this case also occurs when the @@ -219,9 +231,12 @@ class InsnPrologue(Insn): def __setattr__(self, attr, value): + # FIXME: Correct this assert to work for both 32-bit and 64-bit + """ if attr == 'framesize': assert value == 4, ("unrecognized function prologue - " "only supports push %ebp; movl %esp, %ebp") + """ Insn.__setattr__(self, attr, value) class InsnEpilogue(Insn): Modified: pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/conftest.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/conftest.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/conftest.py Mon Jul 19 21:39:55 2010 @@ -1,8 +1,6 @@ import py from pypy.jit.backend import detect_cpu - cpu = detect_cpu.autodetect() def pytest_runtest_setup(item): - if cpu != 'x86': + if cpu not in ('x86', 'x86_64'): py.test.skip("x86 directory skipped: cpu is %r" % (cpu,)) - Added: pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_32bit_reg_zeroextend.s ============================================================================== --- (empty file) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_32bit_reg_zeroextend.s Mon Jul 19 21:39:55 2010 @@ -0,0 +1,15 @@ + .type foobar, @function +foobar: + pushq %rbp + movq %rsp, %rbp + call some_function + ;; expected {8(%rbp) | %rbx, %r12, %r13, %r14, %r15, (%rbp) | } + movl $const1, %edx + movl $const2, %r10d + xorl %r10d, %r11d + /* GCROOT %rdx */ + /* GCROOT %r10 */ + /* GCROOT %r11 */ + leave + ret + .size foobar, .-foobar Added: pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_basic_argument_registers.s ============================================================================== --- (empty file) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_basic_argument_registers.s Mon Jul 19 21:39:55 2010 @@ -0,0 +1,31 @@ + .type foobar, @function +foobar: +.LFB0: + .cfi_startproc + pushq %rbp + .cfi_def_cfa_offset 16 + movq %rsp, %rbp + .cfi_offset 6, -16 + .cfi_def_cfa_register 6 + subq $48, %rsp + movq %rdi, -8(%rbp) + movq %rsi, -16(%rbp) + movq %rdx, -24(%rbp) + movq %rcx, -32(%rbp) + movq %r8, -40(%rbp) + movq %r9, -48(%rbp) + movl $0, %eax + call some_function + ;; expected {8(%rbp) | %rbx, %r12, %r13, %r14, %r15, (%rbp) | -8(%rbp), -16(%rbp), -24(%rbp), -32(%rbp), -40(%rbp), -48(%rbp)} + /* GCROOT -8(%rbp) */ + /* GCROOT -16(%rbp) */ + /* GCROOT -24(%rbp) */ + /* GCROOT -32(%rbp) */ + /* GCROOT -40(%rbp) */ + /* GCROOT -48(%rbp) */ + movq -24(%rbp), %rax + leave + ret + .cfi_endproc +.LFE0: + .size foobar, .-foobar Added: pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_jumptable.c ============================================================================== --- (empty file) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_jumptable.c Mon Jul 19 21:39:55 2010 @@ -0,0 +1,18 @@ +#include + +int foobar(int n) { + switch(n) { + case 0: + return 1; + case 1: + return 12; + case 2: + return 123; + case 3: + return 1234; + case 4: + return 12345; + default: + return 42; + } +} Added: pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_jumptable.s ============================================================================== --- (empty file) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_jumptable.s Mon Jul 19 21:39:55 2010 @@ -0,0 +1,48 @@ + .type foobar, @function +foobar: +.LFB0: + .cfi_startproc + pushq %rbp + .cfi_def_cfa_offset 16 + movq %rsp, %rbp + .cfi_offset 6, -16 + .cfi_def_cfa_register 6 + movl %edi, -4(%rbp) + cmpl $4, -4(%rbp) + ja .L2 + mov -4(%rbp), %eax + movq .L8(,%rax,8), %rax + jmp *%rax + .section .rodata + .align 8 + .align 4 +.L8: + .quad .L3 + .quad .L4 + .quad .L5 + .quad .L6 + .quad .L7 + .text +.L3: + movl $1, %eax + jmp .L9 +.L4: + movl $12, %eax + jmp .L9 +.L5: + movl $123, %eax + jmp .L9 +.L6: + movl $1234, %eax + jmp .L9 +.L7: + movl $12345, %eax + jmp .L9 +.L2: + movl $42, %eax +.L9: + leave + ret + .cfi_endproc +.LFE0: + .size foobar, .-foobar Added: pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_negative_rsp_offset.s ============================================================================== --- (empty file) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_negative_rsp_offset.s Mon Jul 19 21:39:55 2010 @@ -0,0 +1,17 @@ + .type some_function, @function +some_function: + ;; Test using a negative offset from %rsp (gcc sometimes does this) + movq %rbx, -8(%rsp) + subq $8, %rsp + + movq %rdi, %rbx + + call some_other_function + ;; expected {8(%rsp) | (%rsp), %r12, %r13, %r14, %r15, %rbp | %rbx} + /* GCROOT %rbx */ + + movq %rbx, %rax + ;; Same as where %rbx was saved above + movq (%rsp), %rbx + ret + .size some_function, .-some_function Added: pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_varargs_function.s ============================================================================== --- (empty file) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/elf64/track_varargs_function.s Mon Jul 19 21:39:55 2010 @@ -0,0 +1,58 @@ + .type PyErr_Format, @function +PyErr_Format: +.LFB67: + .cfi_startproc + pushq %rbp + .cfi_def_cfa_offset 16 + movzbl %al, %eax + pushq %rbx + .cfi_def_cfa_offset 24 + movq %rdi, %rbx + .cfi_offset 3, -24 + .cfi_offset 6, -16 + movq %rsi, %rdi + subq $216, %rsp + .cfi_def_cfa_offset 240 + movq %rdx, 48(%rsp) + leaq 0(,%rax,4), %rdx + movl $.L21, %eax + movq %rcx, 56(%rsp) + movq %r8, 64(%rsp) + movq %rsp, %rsi + subq %rdx, %rax + leaq 207(%rsp), %rdx + movq %r9, 72(%rsp) + jmp *%rax + movaps %xmm7, -15(%rdx) + movaps %xmm6, -31(%rdx) + movaps %xmm5, -47(%rdx) + movaps %xmm4, -63(%rdx) + movaps %xmm3, -79(%rdx) + movaps %xmm2, -95(%rdx) + movaps %xmm1, -111(%rdx) + movaps %xmm0, -127(%rdx) +.L21: + leaq 240(%rsp), %rax + movl $16, (%rsp) + movl $48, 4(%rsp) + movq %rax, 8(%rsp) + leaq 32(%rsp), %rax + movq %rax, 16(%rsp) + call PyString_FromFormatV + ;; expected {232(%rsp) | 216(%rsp), %r12, %r13, %r14, %r15, 224(%rsp) | } + movq %rbx, %rdi + movq %rax, %rbp + movq %rax, %rsi + call PyErr_SetObject + ;; expected {232(%rsp) | 216(%rsp), %r12, %r13, %r14, %r15, 224(%rsp) | } + movq %rbp, %rdi + call Py_DecRef + ;; expected {232(%rsp) | 216(%rsp), %r12, %r13, %r14, %r15, 224(%rsp) | } + addq $216, %rsp + xorl %eax, %eax + popq %rbx + popq %rbp + ret + .cfi_endproc +.LFE67: + .size PyErr_Format, .-PyErr_Format Modified: pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/test_trackgcroot.py Mon Jul 19 21:39:55 2010 @@ -11,11 +11,14 @@ from pypy.translator.c.gcc.trackgcroot import decompress_callshape from pypy.translator.c.gcc.trackgcroot import PARSERS from StringIO import StringIO +import py.test this_dir = py.path.local(__file__).dirpath() def test_format_location(): + # FIXME + py.test.skip() assert format_location(LOC_NOWHERE) == '?' assert format_location(LOC_REG | (1<<2)) == '%ebx' assert format_location(LOC_REG | (2<<2)) == '%esi' @@ -28,6 +31,8 @@ assert format_location(LOC_ESP_PLUS + 4) == '4(%esp)' def test_format_callshape(): + # FIXME + py.test.skip() expected = ('{4(%ebp) ' # position of the return address '| 8(%ebp), 12(%ebp), 16(%ebp), 20(%ebp) ' # 4 saved regs '| 24(%ebp), 28(%ebp)}') # GC roots @@ -108,7 +113,7 @@ def test_computegcmaptable(): tests = [] - for format in ('elf', 'darwin', 'msvc'): + for format in ('elf', 'darwin', 'msvc', 'elf64'): for path in this_dir.join(format).listdir("track*.s"): n = path.purebasename[5:] try: @@ -138,7 +143,7 @@ tabledict = {} seen = {} for entry in table: - print '%s: %s' % (entry[0], format_callshape(entry[1])) + print '%s: %s' % (entry[0], format_callshape(PARSERS[format], entry[1])) tabledict[entry[0]] = entry[1] # find the ";; expected" lines prevline = "" @@ -151,7 +156,7 @@ label = prevmatch.group(1) assert label in tabledict got = tabledict[label] - assert format_callshape(got) == expected + assert format_callshape(PARSERS[format], got) == expected seen[label] = True if format == 'msvc': expectedlines.insert(i-2, 'PUBLIC\t%s\n' % (label,)) Modified: pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py Mon Jul 19 21:39:55 2010 @@ -72,7 +72,7 @@ if self.is_stack_bottom: retaddr = LOC_NOWHERE # end marker for asmgcroot.py elif self.uses_frame_pointer: - retaddr = frameloc_ebp(4) + retaddr = frameloc_ebp(self.WORD) else: retaddr = frameloc_esp(insn.framesize) shape = [retaddr] @@ -84,7 +84,8 @@ for localvar, tag in insn.gcroots.items(): if isinstance(localvar, LocalVar): loc = localvar.getlocation(insn.framesize, - self.uses_frame_pointer) + self.uses_frame_pointer, + self.WORD) elif localvar in self.REG2LOC: loc = self.REG2LOC[localvar] else: @@ -263,7 +264,7 @@ ofs_from_ebp = int(match.group(1) or '0') if self.format == 'msvc': ofs_from_ebp += int(match.group(2) or '0') - localvar = ofs_from_ebp - 4 + localvar = ofs_from_ebp - self.WORD assert localvar != 0 # that's the return address return LocalVar(localvar, hint='ebp') return localvar @@ -385,10 +386,11 @@ 'inc', 'dec', 'not', 'neg', 'or', 'and', 'sbb', 'adc', 'shl', 'shr', 'sal', 'sar', 'rol', 'ror', 'mul', 'imul', 'div', 'idiv', 'bswap', 'bt', 'rdtsc', + 'punpck', 'pshufd', # zero-extending moves should not produce GC pointers 'movz', # quadword operations - 'movq', + # 'movq', ]) visit_movb = visit_nop @@ -400,7 +402,7 @@ visit_xorb = visit_nop visit_xorw = visit_nop - def visit_addl(self, line, sign=+1): + def _visit_add(self, line, sign=+1): match = self.r_binaryinsn.match(line) source = match.group("source") target = match.group("target") @@ -415,8 +417,8 @@ else: return [] - def visit_subl(self, line): - return self.visit_addl(line, sign=-1) + def _visit_sub(self, line): + return self._visit_add(line, sign=-1) def unary_insn(self, line): match = self.r_unaryinsn.match(line) @@ -439,8 +441,6 @@ else: return [] - visit_xorl = binary_insn # used in "xor reg, reg" to create a NULL GC ptr - visit_orl = binary_insn # The various cmov* operations for name in ''' e ne g ge l le a ae b be p np s ns o no @@ -448,7 +448,7 @@ locals()['visit_cmov' + name] = binary_insn locals()['visit_cmov' + name + 'l'] = binary_insn - def visit_andl(self, line): + def _visit_and(self, line): match = self.r_binaryinsn.match(line) target = match.group("target") if target == self.ESP: @@ -460,9 +460,7 @@ else: return self.binary_insn(line) - visit_and = visit_andl - - def visit_leal(self, line): + def _visit_lea(self, line): match = self.r_binaryinsn.match(line) target = match.group("target") if target == self.ESP: @@ -474,7 +472,7 @@ raise UnrecognizedOperation('epilogue without prologue') ofs_from_ebp = int(match.group(1) or '0') assert ofs_from_ebp <= 0 - framesize = 4 - ofs_from_ebp + framesize = self.WORD - ofs_from_ebp else: match = self.r_localvar_esp.match(source) # leal 12(%esp), %esp @@ -489,8 +487,13 @@ def insns_for_copy(self, source, target): source = self.replace_symbols(source) target = self.replace_symbols(target) - if source == self.ESP or target == self.ESP: + if target == self.ESP: raise UnrecognizedOperation('%s -> %s' % (source, target)) + elif source == self.ESP: + # eg, mov %rsp, %rdi + # Since we are just creating a pointer to the stack, it cannot + # possibly be a GC root + return [] elif self.r_localvar.match(target): if self.r_localvar.match(source): return [InsnCopyLocal(source, target)] @@ -499,7 +502,7 @@ else: return [] - def visit_movl(self, line): + def _visit_mov(self, line): match = self.r_binaryinsn.match(line) source = match.group("source") target = match.group("target") @@ -513,23 +516,13 @@ # gcc -fno-unit-at-a-time. return self.insns_for_copy(source, target) - visit_mov = visit_movl - - def visit_pushl(self, line): + def _visit_push(self, line): match = self.r_unaryinsn.match(line) source = match.group(1) - return [InsnStackAdjust(-4)] + self.insns_for_copy(source, self.TOP_OF_STACK) - - def visit_pushw(self, line): - return [InsnStackAdjust(-2)] # rare but not impossible + return [InsnStackAdjust(-self.WORD)] + self.insns_for_copy(source, self.TOP_OF_STACK) def _visit_pop(self, target): - return self.insns_for_copy(self.TOP_OF_STACK, target) + [InsnStackAdjust(+4)] - - def visit_popl(self, line): - match = self.r_unaryinsn.match(line) - target = match.group(1) - return self._visit_pop(target) + return self.insns_for_copy(self.TOP_OF_STACK, target) + [InsnStackAdjust(+self.WORD)] def _visit_prologue(self): # for the prologue of functions that use %ebp as frame pointer @@ -540,7 +533,7 @@ def _visit_epilogue(self): if not self.uses_frame_pointer: raise UnrecognizedOperation('epilogue without prologue') - return [InsnEpilogue(4)] + return [InsnEpilogue(self.WORD)] def visit_leave(self, line): return self._visit_epilogue() + self._visit_pop(self.EBP) @@ -662,7 +655,7 @@ visit_jc = conditional_jump visit_jnc = conditional_jump - def visit_xchgl(self, line): + def _visit_xchg(self, line): # only support the format used in VALGRIND_DISCARD_TRANSLATIONS # which is to use a marker no-op "xchgl %ebx, %ebx" match = self.r_binaryinsn.match(line) @@ -741,8 +734,97 @@ insns.append(InsnStackAdjust(16)) return insns +class FunctionGcRootTracker32(FunctionGcRootTracker): + WORD = 4 + + visit_mov = FunctionGcRootTracker._visit_mov + visit_movl = FunctionGcRootTracker._visit_mov + visit_pushl = FunctionGcRootTracker._visit_push + visit_leal = FunctionGcRootTracker._visit_lea + + visit_addl = FunctionGcRootTracker._visit_add + visit_subl = FunctionGcRootTracker._visit_sub + visit_andl = FunctionGcRootTracker._visit_and + visit_and = FunctionGcRootTracker._visit_and + + visit_xchgl = FunctionGcRootTracker._visit_xchg + + # used in "xor reg, reg" to create a NULL GC ptr + visit_xorl = FunctionGcRootTracker.binary_insn + visit_orl = FunctionGcRootTracker.binary_insn + + def visit_pushw(self, line): + return [InsnStackAdjust(-2)] # rare but not impossible + + def visit_popl(self, line): + match = self.r_unaryinsn.match(line) + target = match.group(1) + return self._visit_pop(target) + +class FunctionGcRootTracker64(FunctionGcRootTracker): + WORD = 8 + + # Regex ignores destination + r_save_xmm_register = re.compile(r"\tmovaps\s+%xmm(\d+)") + + def _maybe_32bit_dest(func): + def wrapper(self, line): + # Using a 32-bit reg as a destination in 64-bit mode zero-extends + # to 64-bits, so sometimes gcc uses a 32-bit operation to copy a + # statically known pointer to a register + + # %eax -> %rax + new_line = re.sub(r"%e(ax|bx|cx|dx|di|si)$", r"%r\1", line) + # %r10d -> %r10 + new_line = re.sub(r"%r(\d+)d$", r"%r\1", new_line) + return func(self, new_line) + return wrapper + + visit_addl = FunctionGcRootTracker.visit_nop + visit_subl = FunctionGcRootTracker.visit_nop + visit_leal = FunctionGcRootTracker.visit_nop + + visit_cltq = FunctionGcRootTracker.visit_nop + + visit_movq = FunctionGcRootTracker._visit_mov + # just a special assembler mnemonic for mov + visit_movabsq = FunctionGcRootTracker._visit_mov + visit_mov = _maybe_32bit_dest(FunctionGcRootTracker._visit_mov) + visit_movl = visit_mov + + visit_xorl = _maybe_32bit_dest(FunctionGcRootTracker.binary_insn) + + visit_pushq = FunctionGcRootTracker._visit_push + + visit_addq = FunctionGcRootTracker._visit_add + visit_subq = FunctionGcRootTracker._visit_sub + + visit_leaq = FunctionGcRootTracker._visit_lea + + visit_xorq = FunctionGcRootTracker.binary_insn + + # FIXME: similar to visit_popl for 32-bit + def visit_popq(self, line): + match = self.r_unaryinsn.match(line) + target = match.group(1) + return self._visit_pop(target) -class ElfFunctionGcRootTracker(FunctionGcRootTracker): + def visit_jmp(self, line): + # Check for a very special case that occurs in varags functions. + # See test/elf64/track_varargs_function.s for an example of the + # pattern we are looking for + if (self.currentlineno + 8) < len(self.lines) and self.r_unaryinsn_star.match(line): + matches = [self.r_save_xmm_register.match(self.lines[self.currentlineno + 1 + i]) for i in range(8)] + if all(m and int(m.group(1)) == (7 - i) for i, m in enumerate(matches)): + # We assume that the JMP is to somewhere in the block of + # movaps instructions, and ignore it. + return [] + + return FunctionGcRootTracker.visit_jmp(self, line) + + + +class ElfFunctionGcRootTracker32(FunctionGcRootTracker32): format = 'elf' ESP = '%esp' @@ -791,7 +873,66 @@ match = self.r_functionend.match(lines[-1]) assert funcname == match.group(1) assert funcname == match.group(2) - super(ElfFunctionGcRootTracker, self).__init__( + super(ElfFunctionGcRootTracker32, self).__init__( + funcname, lines, filetag) + + def extract_immediate(self, value): + if not value.startswith('$'): + return None + return int(value[1:]) + +ElfFunctionGcRootTracker32.init_regexp() + +class ElfFunctionGcRootTracker64(FunctionGcRootTracker64): + format = 'elf64' + ESP = '%rsp' + EBP = '%rbp' + EAX = '%rax' + CALLEE_SAVE_REGISTERS = ['%rbx', '%r12', '%r13', '%r14', '%r15', '%rbp'] + # XXX: Is this okay? + REG2LOC = dict((_reg, LOC_REG | ((_i+1)<<2)) + for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS)) + OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' + LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' + OFFSET_LABELS = 2**30 + TOP_OF_STACK = '0(%rsp)' + + r_functionstart = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") + r_functionend = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$") + LOCALVAR = r"%rax|%rbx|%rcx|%rdx|%rdi|%rsi|%rbp|%r8|%r9|%r10|%r11|%r12|%r13|%r14|%r15|-?\d*[(]%rsp[)]" + LOCALVARFP = LOCALVAR + r"|-?\d*[(]%rbp[)]" + r_localvarnofp = re.compile(LOCALVAR) + r_localvarfp = re.compile(LOCALVARFP) + r_localvar_esp = re.compile(r"(-?\d*)[(]%rsp[)]") + r_localvar_ebp = re.compile(r"(-?\d*)[(]%rbp[)]") + + r_rel_label = re.compile(r"(\d+):\s*$") + r_jump_rel_label = re.compile(r"\tj\w+\s+"+"(\d+)f"+"\s*$") + + r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+[*]("+OPERAND+")\s*$") + r_jmptable_item = re.compile(r"\t.quad\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") + r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) + + r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") + r_gcnocollect_marker = re.compile(r"\t/[*] GC_NOCOLLECT ("+OPERAND+") [*]/") + r_bottom_marker = re.compile(r"\t/[*] GC_STACK_BOTTOM [*]/") + + FUNCTIONS_NOT_RETURNING = { + 'abort': None, + '_exit': None, + '__assert_fail': None, + '___assert_rtn': None, + 'L___assert_rtn$stub': None, + 'L___eprintf$stub': None, + } + + def __init__(self, lines, filetag=0): + match = self.r_functionstart.match(lines[0]) + funcname = match.group(1) + match = self.r_functionend.match(lines[-1]) + assert funcname == match.group(1) + assert funcname == match.group(2) + super(ElfFunctionGcRootTracker64, self).__init__( funcname, lines, filetag) def extract_immediate(self, value): @@ -799,9 +940,9 @@ return None return int(value[1:]) -ElfFunctionGcRootTracker.init_regexp() +ElfFunctionGcRootTracker64.init_regexp() -class DarwinFunctionGcRootTracker(ElfFunctionGcRootTracker): +class DarwinFunctionGcRootTracker(ElfFunctionGcRootTracker32): format = 'darwin' r_functionstart = re.compile(r"_(\w+):\s*$") @@ -810,7 +951,7 @@ def __init__(self, lines, filetag=0): match = self.r_functionstart.match(lines[0]) funcname = '_' + match.group(1) - FunctionGcRootTracker.__init__(self, funcname, lines, filetag) + FunctionGcRootTracker32.__init__(self, funcname, lines, filetag) class Mingw32FunctionGcRootTracker(DarwinFunctionGcRootTracker): format = 'mingw32' @@ -821,7 +962,7 @@ '__assert': None, } -class MsvcFunctionGcRootTracker(FunctionGcRootTracker): +class MsvcFunctionGcRootTracker(FunctionGcRootTracker32): format = 'msvc' ESP = 'esp' EBP = 'ebp' @@ -906,12 +1047,12 @@ push pop mov lea xor sub add '''.split(): - locals()['visit_' + name] = getattr(FunctionGcRootTracker, + locals()['visit_' + name] = getattr(FunctionGcRootTracker32, 'visit_' + name + 'l') - visit_int = FunctionGcRootTracker.visit_nop + visit_int = FunctionGcRootTracker32.visit_nop # probably not GC pointers - visit_cdq = FunctionGcRootTracker.visit_nop + visit_cdq = FunctionGcRootTracker32.visit_nop def visit_npad(self, line): # MASM has a nasty bug: it implements "npad 5" with "add eax, 0" @@ -1038,7 +1179,7 @@ table = tracker.computegcmaptable(self.verbose) if self.verbose > 1: for label, state in table: - print >> sys.stderr, label, '\t', format_callshape(state) + print >> sys.stderr, label, '\t', format_callshape(self, state) table = compress_gcmaptable(table) if self.shuffle and random.random() < 0.5: self.gcmaptable[:0] = table @@ -1049,7 +1190,7 @@ class ElfAssemblerParser(AssemblerParser): format = "elf" - FunctionGcRootTracker = ElfFunctionGcRootTracker + FunctionGcRootTracker = ElfFunctionGcRootTracker32 def find_functions(self, iterlines): functionlines = [] @@ -1072,6 +1213,10 @@ "missed the end of the previous function") yield False, functionlines +class ElfAssemblerParser64(ElfAssemblerParser): + format = "elf64" + FunctionGcRootTracker = ElfFunctionGcRootTracker64 + class DarwinAssemblerParser(AssemblerParser): format = "darwin" FunctionGcRootTracker = DarwinFunctionGcRootTracker @@ -1241,6 +1386,7 @@ PARSERS = { 'elf': ElfAssemblerParser, + 'elf64': ElfAssemblerParser64, 'darwin': DarwinAssemblerParser, 'mingw32': Mingw32AssemblerParser, 'msvc': MsvcAssemblerParser, @@ -1281,6 +1427,11 @@ txt = kwargs[self.format] print >> output, "\t%s" % txt + if self.format == 'elf64': + word_decl = '.quad' + else: + word_decl = '.long' + # The pypy_asm_stackwalk() function if self.format == 'msvc': @@ -1327,7 +1478,56 @@ } } """ + elif self.format == 'elf64': + print >> output, "\t.text" + print >> output, "\t.globl %s" % _globalname('pypy_asm_stackwalk') + print >> output, "\t.type pypy_asm_stackwalk, @function" + print >> output, "%s:" % _globalname('pypy_asm_stackwalk') + + print >> output, """\ + /* See description in asmgcroot.py */ + movq\t%rdi, %rdx\t/* 1st argument, which is the callback */ + movq\t%rsi, %rcx\t/* 2nd argument, which is gcrootanchor */ + movq\t%rsp, %rax\t/* my frame top address */ + pushq\t%rax\t\t/* ASM_FRAMEDATA[8] */ + pushq\t%rbp\t\t/* ASM_FRAMEDATA[7] */ + pushq\t%r15\t\t/* ASM_FRAMEDATA[6] */ + pushq\t%r14\t\t/* ASM_FRAMEDATA[5] */ + pushq\t%r13\t\t/* ASM_FRAMEDATA[4] */ + pushq\t%r12\t\t/* ASM_FRAMEDATA[3] */ + pushq\t%rbx\t\t/* ASM_FRAMEDATA[2] */ + /* Add this ASM_FRAMEDATA to the front of the circular linked */ + /* list. Let's call it 'self'. */ + + movq\t8(%rcx), %rax\t/* next = gcrootanchor->next */ + pushq\t%rax\t\t\t\t/* self->next = next */ + pushq\t%rcx\t\t\t/* self->prev = gcrootanchor */ + movq\t%rsp, 8(%rcx)\t/* gcrootanchor->next = self */ + movq\t%rsp, 0(%rax)\t\t\t/* next->prev = self */ + + /* note: the Mac OS X 16 bytes aligment must be respected. */ + call\t*%rdx\t\t/* invoke the callback */ + + /* Detach this ASM_FRAMEDATA from the circular linked list */ + popq\t%rsi\t\t/* prev = self->prev */ + popq\t%rdi\t\t/* next = self->next */ + movq\t%rdi, 8(%rsi)\t/* prev->next = next */ + movq\t%rsi, 0(%rdi)\t/* next->prev = prev */ + + popq\t%rbx\t\t/* restore from ASM_FRAMEDATA[2] */ + popq\t%r12\t\t/* restore from ASM_FRAMEDATA[3] */ + popq\t%r13\t\t/* restore from ASM_FRAMEDATA[4] */ + popq\t%r14\t\t/* restore from ASM_FRAMEDATA[5] */ + popq\t%r15\t\t/* restore from ASM_FRAMEDATA[6] */ + popq\t%rbp\t\t/* restore from ASM_FRAMEDATA[7] */ + popq\t%rcx\t\t/* ignored ASM_FRAMEDATA[8] */ + + /* the return value is the one of the 'call' above, */ + /* because %rax (and possibly %rdx) are unmodified */ + ret + .size pypy_asm_stackwalk, .-pypy_asm_stackwalk + """ else: print >> output, "\t.text" print >> output, "\t.globl %s" % _globalname('pypy_asm_stackwalk') @@ -1440,10 +1640,11 @@ shapeofs += len(bytes) if is_range: n = ~ n - print >> output, '\t.long\t%s-%d' % ( + print >> output, '\t%s\t%s-%d' % ( + word_decl, label, PARSERS[self.format].FunctionGcRootTracker.OFFSET_LABELS) - print >> output, '\t.long\t%d' % (n,) + print >> output, '\t%s\t%d' % (word_decl, n) print >> output, """\ .globl __gcmapend @@ -1451,6 +1652,7 @@ """.replace("__gcmapend", _globalname("__gcmapend")) _variant(elf='.section\t.rodata', + elf64='.section\t.rodata', darwin='.const', mingw32='') @@ -1485,35 +1687,36 @@ # __________ debugging output __________ -def format_location(loc): +def format_location(parser, loc): # A 'location' is a single number describing where a value is stored # across a call. It can be in one of the CALLEE_SAVE_REGISTERS, or # in the stack frame at an address relative to either %esp or %ebp. # The last two bits of the location number are used to tell the cases # apart; see format_location(). + tracker_cls = parser.FunctionGcRootTracker assert loc >= 0 kind = loc & LOC_MASK if kind == LOC_REG: if loc == LOC_NOWHERE: return '?' reg = (loc >> 2) - 1 - return ElfFunctionGcRootTracker.CALLEE_SAVE_REGISTERS[reg] + return '%' + tracker_cls.CALLEE_SAVE_REGISTERS[reg].replace("%", "") else: offset = loc & ~ LOC_MASK if kind == LOC_EBP_PLUS: - result = '(%ebp)' + result = '(%' + tracker_cls.EBP.replace("%", "") + ')' elif kind == LOC_EBP_MINUS: - result = '(%ebp)' + result = '(%' + tracker_cls.EBP.replace("%", "") + ')' offset = -offset elif kind == LOC_ESP_PLUS: - result = '(%esp)' + result = '(%' + tracker_cls.ESP.replace("%", "") + ')' else: assert 0, kind if offset != 0: result = str(offset) + result return result -def format_callshape(shape): +def format_callshape(parser, shape): # A 'call shape' is a tuple of locations in the sense of format_location(). # They describe where in a function frame interesting values are stored, # when this function executes a 'call' instruction. @@ -1526,12 +1729,14 @@ # shape[4] is where the fn saved its own caller's %ebp value # shape[>=5] are GC roots: where the fn has put its local GCPTR vars # + num_callee_save_regs = len(parser.FunctionGcRootTracker.CALLEE_SAVE_REGISTERS) assert isinstance(shape, tuple) - assert len(shape) >= 5 - result = [format_location(loc) for loc in shape] + # + 1 for the return address + assert len(shape) >= (num_callee_save_regs + 1) + result = [format_location(parser, loc) for loc in shape] return '{%s | %s | %s}' % (result[0], - ', '.join(result[1:5]), - ', '.join(result[5:])) + ', '.join(result[1:(num_callee_save_regs+1)]), + ', '.join(result[(num_callee_save_regs+1):])) # __________ table compression __________ @@ -1568,10 +1773,15 @@ # whose value is up to a few thousands, which the algorithm below # compresses down to 2 bytes. Very small values compress down to a # single byte. - assert len(shape) >= 5 + + # FIXME + # MIN_SIZE = 5 + MIN_SIZE = 7 + + assert len(shape) >= MIN_SIZE shape = list(shape) - assert 0 not in shape[5:] - shape.insert(5, 0) + assert 0 not in shape[MIN_SIZE:] + shape.insert(MIN_SIZE, 0) result = [] for loc in shape: assert loc >= 0 @@ -1626,7 +1836,9 @@ elif sys.platform == 'win32': format = 'mingw32' else: - format = 'elf' + # FIXME + # format = 'elf' + format = 'elf64' entrypoint = 'main' while len(sys.argv) > 1: if sys.argv[1] == '-v': From fijal at codespeak.net Mon Jul 19 21:50:37 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Jul 2010 21:50:37 +0200 (CEST) Subject: [pypy-svn] r76287 - pypy/branch/interplevel-array/pypy/module/_demo Message-ID: <20100719195037.77797282BE8@codespeak.net> Author: fijal Date: Mon Jul 19 21:50:36 2010 New Revision: 76287 Modified: pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py Log: Strike redundant line Modified: pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py (original) +++ pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py Mon Jul 19 21:50:36 2010 @@ -11,8 +11,6 @@ return W_One() w_type.unwrap_spec = [ObjSpace, int] -type_repr = SMM('__repr__', 1, 'a docstring') - type_typedef = StdTypeDef("tp", __new__ = w_type) type_typedef.registermethods(globals()) From hakanardo at codespeak.net Mon Jul 19 22:15:16 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 19 Jul 2010 22:15:16 +0200 (CEST) Subject: [pypy-svn] r76288 - in pypy/branch/interplevel-array/pypy/module/_demo: . test Message-ID: <20100719201516.27A5A282BE8@codespeak.net> Author: hakanardo Date: Mon Jul 19 22:15:13 2010 New Revision: 76288 Modified: pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py pypy/branch/interplevel-array/pypy/module/_demo/test/test_demo.py Log: a few more methods Modified: pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py (original) +++ pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py Mon Jul 19 22:15:13 2010 @@ -11,8 +11,10 @@ return W_One() w_type.unwrap_spec = [ObjSpace, int] -type_typedef = StdTypeDef("tp", - __new__ = w_type) +#type_repr = SMM('__repr__', 1, 'a docstring') +tp_pop = SMM('pop', 2, defaults=(-1,)) + +type_typedef = StdTypeDef("tp", __new__ = w_type) type_typedef.registermethods(globals()) class W_Zero(W_Object): @@ -35,4 +37,19 @@ def repr__One(space, w_one): return space.wrap("one") + +def len__Zero(space, w_zero): + return space.wrap(7) + +def len__One(space, w_zero): + return space.wrap(42) + + +def tp_pop__One_ANY(space, w_one, w_count): + return w_count + +def tp_pop__Zero_ANY(space, w_zero, w_count): + return space.wrap(9) + + register_all(locals(), globals()) Modified: pypy/branch/interplevel-array/pypy/module/_demo/test/test_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_demo/test/test_demo.py (original) +++ pypy/branch/interplevel-array/pypy/module/_demo/test/test_demo.py Mon Jul 19 22:15:13 2010 @@ -9,3 +9,11 @@ import _demo assert repr(_demo.tp(1)) == 'one' assert repr(_demo.tp(0)) == 'zero' + assert len(_demo.tp(1)) == 42 + + o0 = _demo.tp(0) + o1 = _demo.tp(1) + assert o0.pop(7) == 9 + assert o1.pop(7) == 7 + + assert type(_demo.tp(1)) is _demo.tp From fijal at codespeak.net Mon Jul 19 22:27:00 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Jul 2010 22:27:00 +0200 (CEST) Subject: [pypy-svn] r76289 - pypy/branch/interplevel-array/pypy/module/_demo Message-ID: <20100719202700.D7220282BE8@codespeak.net> Author: fijal Date: Mon Jul 19 22:26:58 2010 New Revision: 76289 Modified: pypy/branch/interplevel-array/pypy/module/_demo/__init__.py pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py Log: Fix a couple of things - expose type Modified: pypy/branch/interplevel-array/pypy/module/_demo/__init__.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_demo/__init__.py (original) +++ pypy/branch/interplevel-array/pypy/module/_demo/__init__.py Mon Jul 19 22:26:58 2010 @@ -9,7 +9,8 @@ """A demo built-in module based on ctypes.""" interpleveldefs = { - 'tp' : 'interp_demo.w_type', + 'tp' : 'interp_demo.W_Zero', # W_One would do as well, gateway + # is getting type of an object anyway (which they share) } appleveldefs = { Modified: pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py (original) +++ pypy/branch/interplevel-array/pypy/module/_demo/interp_demo.py Mon Jul 19 22:26:58 2010 @@ -3,18 +3,19 @@ from pypy.objspace.std.stdtypedef import SMM, StdTypeDef from pypy.objspace.std.register_all import register_all from pypy.objspace.std.model import W_Object +from pypy.interpreter.gateway import interp2app -def w_type(space, arg): +def w_type(space, w_subtype, arg): + # XXX handle subclasses if arg == 0: return W_Zero() else: return W_One() -w_type.unwrap_spec = [ObjSpace, int] +w_type.unwrap_spec = [ObjSpace, W_Root, int] -#type_repr = SMM('__repr__', 1, 'a docstring') tp_pop = SMM('pop', 2, defaults=(-1,)) -type_typedef = StdTypeDef("tp", __new__ = w_type) +type_typedef = StdTypeDef("tp", __new__ = interp2app(w_type)) type_typedef.registermethods(globals()) class W_Zero(W_Object): From jcreigh at codespeak.net Tue Jul 20 15:24:30 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 20 Jul 2010 15:24:30 +0200 (CEST) Subject: [pypy-svn] r76290 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100720132430.4EDC2282BEF@codespeak.net> Author: jcreigh Date: Tue Jul 20 15:24:22 2010 New Revision: 76290 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Log: add some comments, remove useless @specialize.arg Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Tue Jul 20 15:24:22 2010 @@ -208,7 +208,9 @@ float_constants = (float_constants + 15) & ~15 # align to 16 bytes addr = rffi.cast(rffi.CArrayPtr(lltype.Char), float_constants) qword_padding = '\x00\x00\x00\x00\x00\x00\x00\x00' + # 0x8000000000000000 neg_const = '\x00\x00\x00\x00\x00\x00\x00\x80' + # 0x7FFFFFFFFFFFFFFF abs_const = '\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F' data = neg_const + qword_padding + abs_const + qword_padding for i in range(len(data)): @@ -1460,8 +1462,9 @@ # returns in eax the fail_index # now we return from the complete frame, which starts from - # _assemble_bootstrap_code(). The LEA below throws away most - # of the frame, including all the PUSHes that we did just above. + # _assemble_bootstrap_code(). The LEA in _call_footer below throws + # away most of the frame, including all the PUSHes that we did just + # above. self._call_footer() self.mc.done() @@ -1510,9 +1513,6 @@ # exit function self._call_footer() - # FIXME: I changed the third argument to this method, but I don't know - # what to do with @specialize - @specialize.arg(2) def implement_guard(self, guard_token, condition=None): self.mc.reserve_bytes(guard_token.recovery_stub_size()) self.pending_guard_tokens.append(guard_token) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Tue Jul 20 15:24:22 2010 @@ -149,6 +149,19 @@ eax, ecx, edx, ebx, esp, ebp, esi, edi, r8, r9, r10, r11, r12, r13, r14, r15 = REGLOCS xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15 = XMMREGLOCS +# We use a scratch register to simulate having 64-bit immediates. When we +# want to do something like: +# mov rax, [0xDEADBEEFDEADBEEF] +# we actually do: +# mov r11, 0xDEADBEEFDEADBEEF +# mov rax, [r11] +# +# NB: You can use the scratch register as a temporary register in +# assembly.py, but great care must be taken when doing so. A call to a +# method in LocationCodeBuilder could clobber the scratch register when +# certain location types are passed in. In additional, if a new MC is +# allocated, and it happens to be more than 32-bits away, the JMP to it +# will also clobber the scratch register. X86_64_SCRATCH_REG = r11 # XXX: a GPR scratch register is definitely needed, but we could probably do # without an xmm scratch reg. From jcreigh at codespeak.net Tue Jul 20 16:42:50 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 20 Jul 2010 16:42:50 +0200 (CEST) Subject: [pypy-svn] r76291 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . test Message-ID: <20100720144250.0ADEE282BF2@codespeak.net> Author: jcreigh Date: Tue Jul 20 16:42:49 2010 New Revision: 76291 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_assembler.py Log: try to be a bit safer about scratch register use Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Tue Jul 20 16:42:49 2010 @@ -92,8 +92,19 @@ def make_new_mc(self): new_mc = self._instantiate_mc() debug_print('[new machine code block at', new_mc.tell(), ']') + + if IS_X86_64: + # The scratch register is sometimes used as a temporary + # register, but the JMP below might clobber it. Rather than risk + # subtle bugs, we preserve the scratch register across the jump. + self._mc.PUSH_r(X86_64_SCRATCH_REG.value) + self._mc.JMP(imm(new_mc.tell())) + if IS_X86_64: + # Restore scratch reg + new_mc.POP_r(X86_64_SCRATCH_REG.value) + if self.function_name is not None: self.end_function(done=False) self.start_pos = new_mc.get_relative_pos() @@ -492,7 +503,6 @@ for i in range(len(get_from_stack)): loc, is_xmm = get_from_stack[i] - self.mc.ensure_bytes_available(32) if is_xmm: self.mc.MOVSD_xb(X86_64_XMM_SCRATCH_REG.value, (2 + i) * WORD) self.mc.MOVSD(loc, X86_64_XMM_SCRATCH_REG) @@ -650,9 +660,6 @@ tmp2 = result_loc.higher8bits() elif IS_X86_64: tmp2 = X86_64_SCRATCH_REG.lowest8bits() - # We can't do a jump in the middle below, because that could - # clobber the scratch register - self.mc.ensure_bytes_available(32) self.mc.SET_ir(rx86.Conditions[cond], tmp1.value) if is_ne: @@ -771,7 +778,6 @@ self.mc.MOVSD(X86_64_XMM_SCRATCH_REG, loc) self.mc.MOVSD_sx(i*WORD, X86_64_XMM_SCRATCH_REG.value) else: - self.mc.ensure_bytes_available(32) self.mc.MOV(X86_64_SCRATCH_REG, loc) self.mc.MOV_sr(i*WORD, X86_64_SCRATCH_REG.value) else: Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Tue Jul 20 16:42:49 2010 @@ -157,12 +157,11 @@ # mov rax, [r11] # # NB: You can use the scratch register as a temporary register in -# assembly.py, but great care must be taken when doing so. A call to a -# method in LocationCodeBuilder could clobber the scratch register when -# certain location types are passed in. In additional, if a new MC is -# allocated, and it happens to be more than 32-bits away, the JMP to it -# will also clobber the scratch register. +# assembler.py, but care must be taken when doing so. A call to a method in +# LocationCodeBuilder could clobber the scratch register when certain +# location types are passed in. X86_64_SCRATCH_REG = r11 + # XXX: a GPR scratch register is definitely needed, but we could probably do # without an xmm scratch reg. X86_64_XMM_SCRATCH_REG = xmm15 @@ -187,6 +186,15 @@ def INSN(self, loc1, loc2): code1 = loc1.location_code() code2 = loc2.location_code() + + # You can pass in the scratch register as a location, but you + # must be careful not to combine it with location types that + # might need to use the scratch register themselves. + if loc2 is X86_64_SCRATCH_REG: + assert code1 not in ('j', 'i') + if loc1 is X86_64_SCRATCH_REG: + assert code2 not in ('j', 'i') + for possible_code1 in unrolling_location_codes: if code1 == possible_code1: for possible_code2 in unrolling_location_codes: Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_assembler.py Tue Jul 20 16:42:49 2010 @@ -30,6 +30,10 @@ self.content.append(("JMP", args)) def done(self): pass + def PUSH_r(self, reg): + pass + def POP_r(self, reg): + pass class FakeAssembler: def write_pending_failure_recoveries(self): From jcreigh at codespeak.net Tue Jul 20 20:27:10 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 20 Jul 2010 20:27:10 +0200 (CEST) Subject: [pypy-svn] r76292 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100720182710.7D77F282BF2@codespeak.net> Author: jcreigh Date: Tue Jul 20 20:27:08 2010 New Revision: 76292 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/jump.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Log: fix some asserts and add an "is_memory_reference" helper method for clarity Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/jump.py Tue Jul 20 20:27:08 2010 @@ -63,7 +63,7 @@ assert pending_dests == 0 def _move(assembler, src, dst, tmpreg): - if isinstance(dst, StackLoc) and isinstance(src, StackLoc): + if dst.is_memory_reference() and src.is_memory_reference(): assembler.regalloc_mov(src, tmpreg) src = tmpreg assembler.regalloc_mov(src, dst) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Tue Jul 20 20:27:08 2010 @@ -19,6 +19,9 @@ def _getregkey(self): return self.value + def is_memory_reference(self): + return self.location_code() in ('b', 's', 'j', 'a', 'm') + def value_r(self): return self.value def value_b(self): return self.value def value_s(self): return self.value @@ -191,8 +194,8 @@ # must be careful not to combine it with location types that # might need to use the scratch register themselves. if loc2 is X86_64_SCRATCH_REG: - assert code1 not in ('j', 'i') - if loc1 is X86_64_SCRATCH_REG: + assert code1 != 'j' + if loc1 is X86_64_SCRATCH_REG and not name.startswith("MOV"): assert code2 not in ('j', 'i') for possible_code1 in unrolling_location_codes: From jcreigh at codespeak.net Tue Jul 20 20:55:35 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 20 Jul 2010 20:55:35 +0200 (CEST) Subject: [pypy-svn] r76293 - in pypy/branch/x86-64-jit-backend: . lib-python/modified-2.5.2 lib_pypy lib_pypy/pypy_test pypy/annotation pypy/annotation/test pypy/interpreter pypy/interpreter/astcompiler pypy/interpreter/astcompiler/tools pypy/interpreter/test pypy/jit/backend/llgraph pypy/jit/backend/llsupport pypy/jit/backend/llsupport/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/test pypy/jit/tool pypy/module/_file pypy/module/_file/test pypy/module/_locale pypy/module/cpyext pypy/module/marshal pypy/module/posix pypy/module/posix/test pypy/module/termios pypy/module/test_lib_pypy/ctypes_tests pypy/objspace/std pypy/rlib pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/lltypesystem/test pypy/rpython/memory pypy/rpython/module pypy/rpython/module/test pypy/rpython/test pypy/translator/c/test pypy/translator/platform Message-ID: <20100720185535.6C800282BF2@codespeak.net> Author: jcreigh Date: Tue Jul 20 20:55:30 2010 New Revision: 76293 Added: pypy/branch/x86-64-jit-backend/pypy/jit/tool/gen-trace-mode-keywords.py - copied unchanged from r76292, pypy/trunk/pypy/jit/tool/gen-trace-mode-keywords.py pypy/branch/x86-64-jit-backend/pypy/jit/tool/pypytrace-mode.el - copied unchanged from r76292, pypy/trunk/pypy/jit/tool/pypytrace-mode.el pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_rposix.py - copied unchanged from r76292, pypy/trunk/pypy/rlib/test/test_rposix.py pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_win32file.py - copied unchanged from r76292, pypy/trunk/pypy/rpython/module/ll_win32file.py Modified: pypy/branch/x86-64-jit-backend/ (props changed) pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/site.py pypy/branch/x86-64-jit-backend/lib_pypy/ (props changed) pypy/branch/x86-64-jit-backend/lib_pypy/dbm.py (props changed) pypy/branch/x86-64-jit-backend/lib_pypy/pypy_test/test_functools.py (props changed) pypy/branch/x86-64-jit-backend/pypy/annotation/bookkeeper.py pypy/branch/x86-64-jit-backend/pypy/annotation/model.py pypy/branch/x86-64-jit-backend/pypy/annotation/test/test_model.py pypy/branch/x86-64-jit-backend/pypy/annotation/unaryop.py pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/ast.py pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/codegen.py pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/symtable.py pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/tools/asdl_py.py pypy/branch/x86-64-jit-backend/pypy/interpreter/baseobjspace.py pypy/branch/x86-64-jit-backend/pypy/interpreter/error.py pypy/branch/x86-64-jit-backend/pypy/interpreter/gateway.py pypy/branch/x86-64-jit-backend/pypy/interpreter/pyopcode.py pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_compiler.py pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_gateway.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/llgraph/llimpl.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/llgraph/runner.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/descr.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/gc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/llmodel.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/support.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/symbolic.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/test/test_symbolic.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/codewriter.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/jtransform.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/support.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_flatten.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/blackhole.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/executor.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/pyjitpl.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/resoperation.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_basic.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_recursive.py pypy/branch/x86-64-jit-backend/pypy/module/_file/interp_file.py pypy/branch/x86-64-jit-backend/pypy/module/_file/test/test_file.py pypy/branch/x86-64-jit-backend/pypy/module/_locale/interp_locale.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/api.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/methodobject.py pypy/branch/x86-64-jit-backend/pypy/module/marshal/interp_marshal.py pypy/branch/x86-64-jit-backend/pypy/module/posix/interp_posix.py pypy/branch/x86-64-jit-backend/pypy/module/posix/test/test_posix2.py pypy/branch/x86-64-jit-backend/pypy/module/termios/interp_termios.py pypy/branch/x86-64-jit-backend/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) pypy/branch/x86-64-jit-backend/pypy/objspace/std/stringobject.py pypy/branch/x86-64-jit-backend/pypy/rlib/libffi.py pypy/branch/x86-64-jit-backend/pypy/rlib/rarithmetic.py pypy/branch/x86-64-jit-backend/pypy/rlib/rdynload.py pypy/branch/x86-64-jit-backend/pypy/rlib/rposix.py pypy/branch/x86-64-jit-backend/pypy/rlib/rwin32.py pypy/branch/x86-64-jit-backend/pypy/rlib/streamio.py pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_rarithmetic.py pypy/branch/x86-64-jit-backend/pypy/rpython/extfunc.py pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rffi.py pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/test/test_rffi.py pypy/branch/x86-64-jit-backend/pypy/rpython/memory/lltypelayout.py pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os.py pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os_stat.py pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_termios.py pypy/branch/x86-64-jit-backend/pypy/rpython/module/test/test_ll_os_stat.py pypy/branch/x86-64-jit-backend/pypy/rpython/rstr.py pypy/branch/x86-64-jit-backend/pypy/rpython/test/test_extfunc.py pypy/branch/x86-64-jit-backend/pypy/rpython/test/test_rstr.py pypy/branch/x86-64-jit-backend/pypy/translator/c/test/test_typed.py pypy/branch/x86-64-jit-backend/pypy/translator/platform/__init__.py pypy/branch/x86-64-jit-backend/pypy/translator/platform/darwin.py pypy/branch/x86-64-jit-backend/pypy/translator/platform/posix.py pypy/branch/x86-64-jit-backend/pypy/translator/platform/windows.py Log: merged changes from trunk through r76292 Modified: pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/site.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/site.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/site.py Tue Jul 20 20:55:30 2010 @@ -175,7 +175,7 @@ def addsitepackages(known_paths): """Add site-packages to sys.path, in a PyPy-specific way.""" - if hasattr(sys, 'pypy_version_info'): + if hasattr(sys, 'pypy_version_info') and hasattr(sys, 'prefix'): from distutils.sysconfig import get_python_lib sitedir = get_python_lib(standard_lib=False) if os.path.isdir(sitedir): Modified: pypy/branch/x86-64-jit-backend/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/annotation/bookkeeper.py Tue Jul 20 20:55:30 2010 @@ -739,7 +739,7 @@ """ w_tuple = SomeTuple def newtuple(self, items_s): - if items_s == [Ellipsis]: + if len(items_s) == 1 and items_s[0] is Ellipsis: res = SomeObject() # hack to get a SomeObject as the *arg res.from_ellipsis = True return res Modified: pypy/branch/x86-64-jit-backend/pypy/annotation/model.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/annotation/model.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/annotation/model.py Tue Jul 20 20:55:30 2010 @@ -34,7 +34,7 @@ from pypy.tool.pairtype import pair, extendabletype from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint, r_ulonglong, base_int -from pypy.rlib.rarithmetic import r_singlefloat +from pypy.rlib.rarithmetic import r_singlefloat, isnan import inspect, weakref DEBUG = False # set to False to disable recording of debugging information @@ -162,6 +162,13 @@ # pretend it's a float. immutable = True + def __eq__(self, other): + # NaN unpleasantness. + if (self.is_constant() and other.is_constant() and + isnan(self.const) and isnan(other.const)): + return True + return super(SomeFloat, self).__eq__(other) + def can_be_none(self): return False @@ -229,11 +236,15 @@ class SomeChar(SomeString): "Stands for an object known to be a string of length 1." + can_be_None = False + def __init__(self): # no 'can_be_None' argument here + pass class SomeUnicodeCodePoint(SomeUnicodeString): "Stands for an object known to be a unicode codepoint." - def can_be_none(self): - return False + can_be_None = False + def __init__(self): # no 'can_be_None' argument here + pass SomeString.basestringclass = SomeString SomeString.basecharclass = SomeChar Modified: pypy/branch/x86-64-jit-backend/pypy/annotation/test/test_model.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/annotation/test/test_model.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/annotation/test/test_model.py Tue Jul 20 20:55:30 2010 @@ -200,6 +200,15 @@ s2.const=cls assert s1.contains(s2) +def test_nan(): + f1 = SomeFloat() + f1.const = float("nan") + f2 = SomeFloat() + f2.const = float("nan") + assert f1.contains(f1) + assert f2.contains(f1) + assert f1.contains(f2) + if __name__ == '__main__': for name, value in globals().items(): if name.startswith('test_'): Modified: pypy/branch/x86-64-jit-backend/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/annotation/unaryop.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/annotation/unaryop.py Tue Jul 20 20:55:30 2010 @@ -5,7 +5,7 @@ from types import MethodType from pypy.annotation.model import \ SomeObject, SomeInteger, SomeBool, SomeString, SomeChar, SomeList, \ - SomeDict, SomeTuple, SomeImpossibleValue, \ + SomeDict, SomeTuple, SomeImpossibleValue, SomeUnicodeCodePoint, \ SomeInstance, SomeBuiltin, SomeFloat, SomeIterator, SomePBC, \ SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \ s_ImpossibleValue, s_Bool, s_None, \ @@ -494,9 +494,6 @@ def getanyitem(str): return str.basecharclass() - def ord(str): - return SomeInteger(nonneg=True) - def method_split(str, patt): # XXX getbookkeeper().count("str_split", str, patt) return getbookkeeper().newlist(str.basestringclass()) @@ -541,11 +538,16 @@ return SomeUnicodeString() method_decode.can_only_throw = [UnicodeDecodeError] -class __extend__(SomeChar): +class __extend__(SomeChar, SomeUnicodeCodePoint): def len(chr): return immutablevalue(1) + def ord(str): + return SomeInteger(nonneg=True) + +class __extend__(SomeChar): + def method_isspace(chr): return s_Bool Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/ast.py Tue Jul 20 20:55:30 2010 @@ -230,6 +230,7 @@ visitor.visit_FunctionDef(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) if self.body: visitor._mutate_sequence(self.body) if self.decorators: @@ -784,6 +785,8 @@ def mutate_over(self, visitor): if self.body: visitor._mutate_sequence(self.body) + if self.handlers: + visitor._mutate_sequence(self.handlers) if self.orelse: visitor._mutate_sequence(self.orelse) return visitor.visit_TryExcept(self) @@ -927,6 +930,8 @@ visitor.visit_Import(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_Import(self) def sync_app_attrs(self, space): @@ -965,6 +970,8 @@ visitor.visit_ImportFrom(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_ImportFrom(self) def sync_app_attrs(self, space): @@ -1282,6 +1289,7 @@ visitor.visit_Lambda(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) self.body = self.body.mutate_over(visitor) return visitor.visit_Lambda(self) @@ -1398,6 +1406,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_ListComp(self) def sync_app_attrs(self, space): @@ -1437,6 +1447,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_GeneratorExp(self) def sync_app_attrs(self, space): @@ -1562,6 +1574,8 @@ self.func = self.func.mutate_over(visitor) if self.args: visitor._mutate_sequence(self.args) + if self.keywords: + visitor._mutate_sequence(self.keywords) if self.starargs: self.starargs = self.starargs.mutate_over(visitor) if self.kwargs: @@ -2293,6 +2307,13 @@ self.w_ifs = None self.initialization_state = 7 + def mutate_over(self, visitor): + self.target = self.target.mutate_over(visitor) + self.iter = self.iter.mutate_over(visitor) + if self.ifs: + visitor._mutate_sequence(self.ifs) + return visitor.visit_comprehension(self) + def walkabout(self, visitor): visitor.visit_comprehension(self) @@ -2327,6 +2348,15 @@ self.col_offset = col_offset self.initialization_state = 31 + def mutate_over(self, visitor): + if self.type: + self.type = self.type.mutate_over(visitor) + if self.name: + self.name = self.name.mutate_over(visitor) + if self.body: + visitor._mutate_sequence(self.body) + return visitor.visit_excepthandler(self) + def walkabout(self, visitor): visitor.visit_excepthandler(self) @@ -2366,6 +2396,13 @@ self.w_defaults = None self.initialization_state = 15 + def mutate_over(self, visitor): + if self.args: + visitor._mutate_sequence(self.args) + if self.defaults: + visitor._mutate_sequence(self.defaults) + return visitor.visit_arguments(self) + def walkabout(self, visitor): visitor.visit_arguments(self) @@ -2407,6 +2444,10 @@ self.value = value self.initialization_state = 3 + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + return visitor.visit_keyword(self) + def walkabout(self, visitor): visitor.visit_keyword(self) @@ -2426,6 +2467,9 @@ self.asname = asname self.initialization_state = 3 + def mutate_over(self, visitor): + return visitor.visit_alias(self) + def walkabout(self, visitor): visitor.visit_alias(self) Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/codegen.py Tue Jul 20 20:55:30 2010 @@ -258,9 +258,11 @@ # Load decorators first, but apply them after the function is created. if func.decorators: self.visit_sequence(func.decorators) - if func.args.defaults: - self.visit_sequence(func.args.defaults) - num_defaults = len(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + num_defaults = len(args.defaults) else: num_defaults = 0 code = self.sub_scope(FunctionCodeGenerator, func.name, func, @@ -274,9 +276,11 @@ def visit_Lambda(self, lam): self.update_position(lam.lineno) - if lam.args.defaults: - self.visit_sequence(lam.args.defaults) - default_count = len(lam.args.defaults) + args = lam.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + default_count = len(args.defaults) else: default_count = 0 code = self.sub_scope(LambdaCodeGenerator, "", lam, lam.lineno) @@ -1275,9 +1279,11 @@ else: self.add_const(self.space.w_None) start = 0 - if func.args.args: - self._handle_nested_args(func.args.args) - self.argcount = len(func.args.args) + args = func.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) for i in range(start, len(func.body)): func.body[i].walkabout(self) @@ -1286,9 +1292,11 @@ def _compile(self, lam): assert isinstance(lam, ast.Lambda) - if lam.args.args: - self._handle_nested_args(lam.args.args) - self.argcount = len(lam.args.args) + args = lam.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) # Prevent a string from being the first constant and thus a docstring. self.add_const(self.space.w_None) lam.body.walkabout(self) Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/symtable.py Tue Jul 20 20:55:30 2010 @@ -353,8 +353,10 @@ def visit_FunctionDef(self, func): self.note_symbol(func.name, SYM_ASSIGNED) # Function defaults and decorators happen in the outer scope. - if func.args.defaults: - self.visit_sequence(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) if func.decorators: self.visit_sequence(func.decorators) new_scope = FunctionScope(func.name, func.lineno, func.col_offset) @@ -420,8 +422,10 @@ self.note_symbol(name, SYM_GLOBAL) def visit_Lambda(self, lamb): - if lamb.args.defaults: - self.visit_sequence(lamb.args.defaults) + args = lamb.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) new_scope = FunctionScope("lambda", lamb.lineno, lamb.col_offset) self.push_scope(new_scope, lamb) lamb.args.walkabout(self) Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/tools/asdl_py.py Tue Jul 20 20:55:30 2010 @@ -100,6 +100,7 @@ self.emit("") self.make_constructor(product.fields, product) self.emit("") + self.make_mutate_over(product, name) self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (name,), 2) self.emit("") @@ -183,6 +184,26 @@ have_everything = self.data.required_masks[node] | \ self.data.optional_masks[node] self.emit("self.initialization_state = %i" % (have_everything,), 2) + + def make_mutate_over(self, cons, name): + self.emit("def mutate_over(self, visitor):", 1) + for field in cons.fields: + if (field.type.value not in asdl.builtin_types and + field.type.value not in self.data.simple_types): + if field.opt or field.seq: + level = 3 + self.emit("if self.%s:" % (field.name,), 2) + else: + level = 2 + if field.seq: + sub = (field.name,) + self.emit("visitor._mutate_sequence(self.%s)" % sub, level) + else: + sub = (field.name, field.name) + self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, + level) + self.emit("return visitor.visit_%s(self)" % (name,), 2) + self.emit("") def visitConstructor(self, cons, base, extra_attributes): self.emit("class %s(%s):" % (cons.name, base)) @@ -199,24 +220,7 @@ self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (cons.name,), 2) self.emit("") - self.emit("def mutate_over(self, visitor):", 1) - for field in cons.fields: - if field.type.value not in asdl.builtin_types and \ - field.type.value not in self.data.prod_simple: - if field.opt or field.seq: - level = 3 - self.emit("if self.%s:" % (field.name,), 2) - else: - level = 2 - if field.seq: - sub = (field.name,) - self.emit("visitor._mutate_sequence(self.%s)" % sub, level) - else: - sub = (field.name, field.name) - self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, - level) - self.emit("return visitor.visit_%s(self)" % (cons.name,), 2) - self.emit("") + self.make_mutate_over(cons, cons.name) self.make_var_syncer(cons.fields + self.data.cons_attributes[cons], cons, cons.name) Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/baseobjspace.py Tue Jul 20 20:55:30 2010 @@ -1102,17 +1102,6 @@ self.wrap('argument must be a unicode')) return self.unicode_w(w_obj) - def path_w(self, w_obj): - """ Like str_w, but if the object is unicode, encode it using - filesystemencoding - """ - filesystemencoding = self.sys.filesystemencoding - if (filesystemencoding and - self.is_true(self.isinstance(w_obj, self.w_unicode))): - w_obj = self.call_method(w_obj, "encode", - self.wrap(filesystemencoding)) - return self.str_w(w_obj) - def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. # This is here mostly just for gateway.int_unwrapping_space_method(). Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/error.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/error.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/error.py Tue Jul 20 20:55:30 2010 @@ -344,7 +344,7 @@ else: _WINDOWS = True - def wrap_windowserror(space, e, filename=None): + def wrap_windowserror(space, e, w_filename=None): from pypy.rlib import rwin32 winerror = e.winerror @@ -353,19 +353,19 @@ except ValueError: msg = 'Windows Error %d' % winerror exc = space.w_WindowsError - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(winerror), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: w_error = space.call_function(exc, space.wrap(winerror), space.wrap(msg)) return OperationError(exc, w_error) -def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): +def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError'): assert isinstance(e, OSError) if _WINDOWS and isinstance(e, WindowsError): - return wrap_windowserror(space, e, filename) + return wrap_windowserror(space, e, w_filename) errno = e.errno try: @@ -373,10 +373,21 @@ except ValueError: msg = 'error %d' % errno exc = getattr(space, exception_name) - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(errno), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: - w_error = space.call_function(exc, space.wrap(errno), space.wrap(msg)) + w_error = space.call_function(exc, space.wrap(errno), + space.wrap(msg)) return OperationError(exc, w_error) +wrap_oserror2._annspecialcase_ = 'specialize:arg(3)' + +def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): + if filename is not None: + return wrap_oserror2(space, e, space.wrap(filename), + exception_name=exception_name) + else: + return wrap_oserror2(space, e, None, + exception_name=exception_name) wrap_oserror._annspecialcase_ = 'specialize:arg(3)' + Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/gateway.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/gateway.py Tue Jul 20 20:55:30 2010 @@ -137,9 +137,6 @@ def visit_c_nonnegint(self, el, app_sig): self.checked_space_method(el, app_sig) - def visit_path(self, el, app_sig): - self.checked_space_method(el, app_sig) - def visit__Wrappable(self, el, app_sig): name = el.__name__ argname = self.orig_arg() @@ -241,9 +238,6 @@ def visit_bufferstr(self, typ): self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),)) - def visit_path(self, typ): - self.run_args.append("space.path_w(%s)" % (self.scopenext(),)) - def visit_nonnegint(self, typ): self.run_args.append("space.nonnegint_w(%s)" % (self.scopenext(),)) @@ -371,9 +365,6 @@ def visit_bufferstr(self, typ): self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),)) - def visit_path(self, typ): - self.unwrap.append("space.path_w(%s)" % (self.nextarg(),)) - def visit_nonnegint(self, typ): self.unwrap.append("space.nonnegint_w(%s)" % (self.nextarg(),)) Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/pyopcode.py Tue Jul 20 20:55:30 2010 @@ -14,6 +14,7 @@ from pypy.rlib import jit, rstackovf, rstack from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.debug import check_nonneg from pypy.tool.stdlib_opcode import (bytecode_spec, host_bytecode_spec, unrolling_all_opcode_descs, opmap, host_opmap) @@ -186,7 +187,7 @@ lo = ord(co_code[next_instr]) hi = ord(co_code[next_instr+1]) next_instr += 2 - oparg = (hi << 8) | lo + oparg = (hi * 256) | lo else: oparg = 0 @@ -197,7 +198,7 @@ lo = ord(co_code[next_instr+1]) hi = ord(co_code[next_instr+2]) next_instr += 3 - oparg = (oparg << 16) | (hi << 8) | lo + oparg = (oparg * 65536) | (hi * 256) | lo if opcode == self.opcodedesc.RETURN_VALUE.index: w_returnvalue = self.popvalue() @@ -210,10 +211,6 @@ next_instr = block.handle(self, unroller) return next_instr # now inside a 'finally' block - if opcode == self.opcodedesc.YIELD_VALUE.index: - #self.last_instr = intmask(next_instr - 1) XXX clean up! - raise Yield - if opcode == self.opcodedesc.END_FINALLY.index: unroller = self.end_finally() if isinstance(unroller, SuspendedUnroller): @@ -238,7 +235,7 @@ if not opdesc.is_enabled(space): continue if opdesc.methodname in ( - 'EXTENDED_ARG', 'RETURN_VALUE', 'YIELD_VALUE', + 'EXTENDED_ARG', 'RETURN_VALUE', 'END_FINALLY', 'JUMP_ABSOLUTE'): continue # opcodes implemented above @@ -323,7 +320,9 @@ ## def NOP(self, oparg, next_instr): - pass + # annotation-time check: if it fails, it means that the decoding + # of oparg failed to produce an integer which is annotated as non-neg + check_nonneg(oparg) def LOAD_FAST(self, varindex, next_instr): # access a local variable directly @@ -811,6 +810,9 @@ self.space.str_w(w_name)) self.pushvalue(w_obj) + def YIELD_VALUE(self, oparg, next_instr): + raise Yield + def jump_absolute(self, jumpto, next_instr, ec): return jumpto Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_compiler.py Tue Jul 20 20:55:30 2010 @@ -838,6 +838,23 @@ sys.stdout = save_stdout output = s.getvalue() assert "STOP_CODE" not in output + + def test_optimize_list_comp(self): + source = """def _f(a): + return [x for x in a if None] + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = sys.__stdout__ + output = s.getvalue() + assert "LOAD_GLOBAL" not in output class AppTestExceptions: def test_indentation_error(self): Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_gateway.py Tue Jul 20 20:55:30 2010 @@ -454,16 +454,6 @@ assert len(l) == 1 assert space.eq_w(l[0], w("foo")) - def test_interp2app_unwrap_spec_path(self, monkeypatch): - space = self.space - def g(space, p): - return p - - app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'path']) - w_app_g = space.wrap(app_g) - monkeypatch.setattr(space.sys, "filesystemencoding", "utf-8") - w_res = space.call_function(w_app_g, space.wrap(u"?")) - def test_interp2app_classmethod(self): space = self.space w = space.wrap Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/llgraph/llimpl.py Tue Jul 20 20:55:30 2010 @@ -421,11 +421,9 @@ global _last_exception assert _last_exception is None, "exception left behind" verbose = True - operations = self.loop.operations - opindex = 0 + self.opindex = 0 while True: - self.opindex = opindex - op = operations[opindex] + op = self.loop.operations[self.opindex] args = [self.getenv(v) for v in op.args] if not op.is_final(): try: @@ -439,8 +437,8 @@ args = [self.getenv(v) for v in op.fail_args if v] assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) - operations = op.jump_target.operations - opindex = 0 + self.loop = op.jump_target + self.opindex = 0 continue else: self._populate_fail_args(op) @@ -465,14 +463,13 @@ raise Exception("op.result.concretetype is %r" % (RESTYPE,)) self.env[op.result] = x - opindex += 1 + self.opindex += 1 continue if op.opnum == rop.JUMP: assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) self.loop = op.jump_target - operations = self.loop.operations - opindex = 0 + self.opindex = 0 _stats.exec_jumps += 1 elif op.opnum == rop.FINISH: if self.verbose: @@ -1199,10 +1196,18 @@ array = array._obj.container return cast_to_int(array.getitem(index)) +def do_getarrayitem_raw_int(array, index): + array = array.adr.ptr._obj + return cast_to_int(array.getitem(index)) + def do_getarrayitem_gc_float(array, index): array = array._obj.container return cast_to_float(array.getitem(index)) +def do_getarrayitem_raw_float(array, index): + array = array.adr.ptr._obj + return cast_to_float(array.getitem(index)) + def do_getarrayitem_gc_ptr(array, index): array = array._obj.container return cast_to_ptr(array.getitem(index)) @@ -1251,12 +1256,24 @@ newvalue = cast_from_int(ITEMTYPE, newvalue) array.setitem(index, newvalue) +def do_setarrayitem_raw_int(array, index, newvalue): + array = array.adr.ptr + ITEMTYPE = lltype.typeOf(array).TO.OF + newvalue = cast_from_int(ITEMTYPE, newvalue) + array._obj.setitem(index, newvalue) + def do_setarrayitem_gc_float(array, index, newvalue): array = array._obj.container ITEMTYPE = lltype.typeOf(array).OF newvalue = cast_from_float(ITEMTYPE, newvalue) array.setitem(index, newvalue) +def do_setarrayitem_raw_float(array, index, newvalue): + array = array.adr.ptr + ITEMTYPE = lltype.typeOf(array).TO.OF + newvalue = cast_from_int(ITEMTYPE, newvalue) + array._obj.setitem(index, newvalue) + def do_setarrayitem_gc_ptr(array, index, newvalue): array = array._obj.container ITEMTYPE = lltype.typeOf(array).OF @@ -1529,6 +1546,8 @@ setannotation(do_getarrayitem_gc_int, annmodel.SomeInteger()) setannotation(do_getarrayitem_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) setannotation(do_getarrayitem_gc_float, annmodel.SomeFloat()) +setannotation(do_getarrayitem_raw_int, annmodel.SomeInteger()) +setannotation(do_getarrayitem_raw_float, annmodel.SomeFloat()) setannotation(do_getfield_gc_int, annmodel.SomeInteger()) setannotation(do_getfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) setannotation(do_getfield_gc_float, annmodel.SomeFloat()) @@ -1540,6 +1559,8 @@ setannotation(do_setarrayitem_gc_int, annmodel.s_None) setannotation(do_setarrayitem_gc_ptr, annmodel.s_None) setannotation(do_setarrayitem_gc_float, annmodel.s_None) +setannotation(do_setarrayitem_raw_int, annmodel.s_None) +setannotation(do_setarrayitem_raw_float, annmodel.s_None) setannotation(do_setfield_gc_int, annmodel.s_None) setannotation(do_setfield_gc_ptr, annmodel.s_None) setannotation(do_setfield_gc_float, annmodel.s_None) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/llgraph/runner.py Tue Jul 20 20:55:30 2010 @@ -294,7 +294,6 @@ return llimpl.grab_exc_value() def arraydescrof(self, A): - assert isinstance(A, lltype.GcArray) assert A.OF != lltype.Void size = symbolic.get_size(A) token = history.getkind(A.OF) @@ -317,12 +316,18 @@ def bh_getarrayitem_gc_i(self, arraydescr, array, index): assert isinstance(arraydescr, Descr) return llimpl.do_getarrayitem_gc_int(array, index) + def bh_getarrayitem_raw_i(self, arraydescr, array, index): + assert isinstance(arraydescr, Descr) + return llimpl.do_getarrayitem_raw_int(array, index) def bh_getarrayitem_gc_r(self, arraydescr, array, index): assert isinstance(arraydescr, Descr) return llimpl.do_getarrayitem_gc_ptr(array, index) def bh_getarrayitem_gc_f(self, arraydescr, array, index): assert isinstance(arraydescr, Descr) return llimpl.do_getarrayitem_gc_float(array, index) + def bh_getarrayitem_raw_f(self, arraydescr, array, index): + assert isinstance(arraydescr, Descr) + return llimpl.do_getarrayitem_raw_float(array, index) def bh_getfield_gc_i(self, struct, fielddescr): assert isinstance(fielddescr, Descr) @@ -372,6 +377,10 @@ assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_int(array, index, newvalue) + def bh_setarrayitem_raw_i(self, arraydescr, array, index, newvalue): + assert isinstance(arraydescr, Descr) + llimpl.do_setarrayitem_raw_int(array, index, newvalue) + def bh_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_ptr(array, index, newvalue) @@ -380,6 +389,10 @@ assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_float(array, index, newvalue) + def bh_setarrayitem_raw_f(self, arraydescr, array, index, newvalue): + assert isinstance(arraydescr, Descr) + llimpl.do_setarrayitem_raw_float(array, index, newvalue) + def bh_setfield_gc_i(self, struct, fielddescr, newvalue): assert isinstance(fielddescr, Descr) llimpl.do_setfield_gc_int(struct, fielddescr.ofs, newvalue) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/descr.py Tue Jul 20 20:55:30 2010 @@ -23,10 +23,10 @@ self._cache_call = {} def init_size_descr(self, STRUCT, sizedescr): - pass + assert isinstance(STRUCT, lltype.GcStruct) def init_array_descr(self, ARRAY, arraydescr): - pass + assert isinstance(ARRAY, lltype.GcArray) # ____________________________________________________________ @@ -67,9 +67,11 @@ class BaseFieldDescr(AbstractDescr): offset = 0 # help translation + name = '' _clsname = '' - def __init__(self, offset): + def __init__(self, name, offset): + self.name = name self.offset = offset def sort_key(self): @@ -88,7 +90,7 @@ return self._is_float_field def repr_of_descr(self): - return '<%s %s>' % (self._clsname, self.offset) + return '<%s %s %s>' % (self._clsname, self.name, self.offset) class NonGcPtrFieldDescr(BaseFieldDescr): @@ -113,7 +115,8 @@ offset, _ = symbolic.get_field_token(STRUCT, fieldname, gccache.translate_support_code) FIELDTYPE = getattr(STRUCT, fieldname) - fielddescr = getFieldDescrClass(FIELDTYPE)(offset) + name = '%s.%s' % (STRUCT._name, fieldname) + fielddescr = getFieldDescrClass(FIELDTYPE)(name, offset) cachedict = cache.setdefault(STRUCT, {}) cachedict[fieldname] = fielddescr return fielddescr @@ -151,7 +154,6 @@ def repr_of_descr(self): return '<%s>' % self._clsname - class NonGcPtrArrayDescr(BaseArrayDescr): _clsname = 'NonGcPtrArrayDescr' def get_item_size(self, translate_support_code): @@ -161,24 +163,53 @@ _clsname = 'GcPtrArrayDescr' _is_array_of_pointers = True +_CA = rffi.CArray(lltype.Signed) + +class BaseArrayNoLengthDescr(BaseArrayDescr): + def get_base_size(self, translate_support_code): + basesize, _, _ = symbolic.get_array_token(_CA, translate_support_code) + return basesize + + def get_ofs_length(self, translate_support_code): + _, _, ofslength = symbolic.get_array_token(_CA, translate_support_code) + return ofslength + +class NonGcPtrArrayNoLengthDescr(BaseArrayNoLengthDescr): + _clsname = 'NonGcPtrArrayNoLengthDescr' + def get_item_size(self, translate_support_code): + return symbolic.get_size_of_ptr(translate_support_code) + +class GcPtrArrayNoLengthDescr(NonGcPtrArrayNoLengthDescr): + _clsname = 'GcPtrArrayNoLengthDescr' + _is_array_of_pointers = True + def getArrayDescrClass(ARRAY): return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr, NonGcPtrArrayDescr, 'Array', 'get_item_size', '_is_array_of_floats') +def getArrayNoLengthDescrClass(ARRAY): + return getDescrClass(ARRAY.OF, BaseArrayNoLengthDescr, GcPtrArrayNoLengthDescr, + NonGcPtrArrayNoLengthDescr, 'ArrayNoLength', 'get_item_size', + '_is_array_of_floats') + def get_array_descr(gccache, ARRAY): cache = gccache._cache_array try: return cache[ARRAY] except KeyError: - arraydescr = getArrayDescrClass(ARRAY)() + if ARRAY._hints.get('nolength', False): + arraydescr = getArrayNoLengthDescrClass(ARRAY)() + else: + arraydescr = getArrayDescrClass(ARRAY)() # verify basic assumption that all arrays' basesize and ofslength # are equal basesize, itemsize, ofslength = symbolic.get_array_token(ARRAY, False) assert basesize == arraydescr.get_base_size(False) assert itemsize == arraydescr.get_item_size(False) assert ofslength == arraydescr.get_ofs_length(False) - gccache.init_array_descr(ARRAY, arraydescr) + if isinstance(ARRAY, lltype.GcArray): + gccache.init_array_descr(ARRAY, arraydescr) cache[ARRAY] = arraydescr return arraydescr Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/gc.py Tue Jul 20 20:55:30 2010 @@ -351,7 +351,7 @@ gcrootmap = cls() self.gcrootmap = gcrootmap self.gcrefs = GcRefList() - self.single_gcref_descr = GcPtrFieldDescr(0) + self.single_gcref_descr = GcPtrFieldDescr('', 0) # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/llmodel.py Tue Jul 20 20:55:30 2010 @@ -250,6 +250,7 @@ ofs = arraydescr.get_ofs_length(self.translate_support_code) return rffi.cast(rffi.CArrayPtr(lltype.Signed), array)[ofs/WORD] + @specialize.argtype(2) def bh_getarrayitem_gc_i(self, arraydescr, gcref, itemindex): ofs, size = self.unpack_arraydescr_size(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -272,6 +273,7 @@ # --- end of GC unsafe code --- return pval + @specialize.argtype(2) def bh_getarrayitem_gc_f(self, arraydescr, gcref, itemindex): ofs = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -281,6 +283,7 @@ # --- end of GC unsafe code --- return fval + @specialize.argtype(2) def bh_setarrayitem_gc_i(self, arraydescr, gcref, itemindex, newvalue): ofs, size = self.unpack_arraydescr_size(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -303,6 +306,7 @@ items[itemindex] = self.cast_gcref_to_int(newvalue) # --- end of GC unsafe code --- + @specialize.argtype(2) def bh_setarrayitem_gc_f(self, arraydescr, gcref, itemindex, newvalue): ofs = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- @@ -311,6 +315,12 @@ items[itemindex] = newvalue # --- end of GC unsafe code --- + bh_setarrayitem_raw_i = bh_setarrayitem_gc_i + bh_setarrayitem_raw_f = bh_setarrayitem_gc_f + + bh_getarrayitem_raw_i = bh_getarrayitem_gc_i + bh_getarrayitem_raw_f = bh_getarrayitem_gc_f + def bh_strlen(self, string): s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) return len(s.chars) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/support.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/support.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/support.py Tue Jul 20 20:55:30 2010 @@ -30,7 +30,13 @@ funcobj = get_funcobj(fnptr) if hasattr(funcobj, 'graph'): - llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) + # cache the llinterp; otherwise the remember_malloc/remember_free + # done on the LLInterpreter don't match + try: + llinterp = rtyper._on_top_of_llinterp_llinterp + except AttributeError: + llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) + rtyper._on_top_of_llinterp_llinterp = llinterp def on_top_of_llinterp(*args): real_args = process_args(args) return llinterp.eval_graph(funcobj.graph, real_args) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/symbolic.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/symbolic.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/symbolic.py Tue Jul 20 20:55:30 2010 @@ -36,8 +36,11 @@ ofs_length = (llmemory.offsetof(T, T._arrayfld) + llmemory.ArrayLengthOffset(SUBARRAY)) else: + if T._hints.get('nolength', None): + ofs_length = -1 + else: + ofs_length = llmemory.ArrayLengthOffset(T) itemsize = llmemory.sizeof(T.OF) - ofs_length = llmemory.ArrayLengthOffset(T) else: if isinstance(T, lltype.Struct): assert T._arrayfld is not None, "%r is not variable-sized" % (T,) @@ -48,8 +51,11 @@ else: before_array_part = 0 carray = ll2ctypes.get_ctypes_type(T) - assert carray.length.size == WORD - ofs_length = before_array_part + carray.length.offset + if T._hints.get('nolength', None): + ofs_length = -1 + else: + assert carray.length.size == WORD + ofs_length = before_array_part + carray.length.offset basesize = before_array_part + carray.items.offset carrayitem = ll2ctypes.get_ctypes_type(T.OF) itemsize = ctypes.sizeof(carrayitem) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/test/test_descr.py Tue Jul 20 20:55:30 2010 @@ -53,6 +53,10 @@ assert descr_y.__class__ is GcPtrFieldDescr assert descr_z.__class__ is NonGcPtrFieldDescr assert descr_f.__class__ is clsf + assert descr_x.name == 'S.x' + assert descr_y.name == 'S.y' + assert descr_z.name == 'S.z' + assert descr_f.name == 'S.f' if not tsc: assert descr_x.offset < descr_y.offset < descr_z.offset assert descr_x.sort_key() < descr_y.sort_key() < descr_z.sort_key() @@ -140,7 +144,25 @@ assert isinstance(descr2.get_item_size(True), Symbolic) assert isinstance(descr3.get_item_size(True), Symbolic) assert isinstance(descr4.get_item_size(True), Symbolic) - + CA = rffi.CArray(lltype.Signed) + descr = get_array_descr(c0, CA) + assert not descr.is_array_of_floats() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Ptr(lltype.GcStruct('S'))) + descr = get_array_descr(c0, CA) + assert descr.is_array_of_pointers() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Ptr(lltype.Struct('S'))) + descr = get_array_descr(c0, CA) + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 + CA = rffi.CArray(lltype.Float) + descr = get_array_descr(c0, CA) + assert descr.is_array_of_floats() + assert descr.get_base_size(False) == 0 + assert descr.get_ofs_length(False) == -1 def test_get_call_descr_not_translated(): c0 = GcCache(False) @@ -210,11 +232,11 @@ # descr2 = get_field_descr(c0, S, 'y') o, _ = symbolic.get_field_token(S, 'y', False) - assert descr2.repr_of_descr() == '' % o + assert descr2.repr_of_descr() == '' % o # descr2i = get_field_descr(c0, S, 'x') o, _ = symbolic.get_field_token(S, 'x', False) - assert descr2i.repr_of_descr() == '' % o + assert descr2i.repr_of_descr() == '' % o # descr3 = get_array_descr(c0, lltype.GcArray(lltype.Ptr(S))) assert descr3.repr_of_descr() == '' Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/test/test_symbolic.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/test/test_symbolic.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/llsupport/test/test_symbolic.py Tue Jul 20 20:55:30 2010 @@ -61,6 +61,12 @@ assert basesize >= WORD # at least the 'length', maybe some gc headers assert itemsize == WORD assert ofs_length == basesize - WORD + A = rffi.CArray(lltype.Signed) + arraytok = get_array_token(A, translate_support_code) + basesize, itemsize, ofs_length = convert(arraytok) + assert basesize == 0 + assert itemsize == WORD + assert ofs_length == -1 def test_varsized_struct_size(): S1 = lltype.GcStruct('S1', ('parent', S), Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Tue Jul 20 20:55:30 2010 @@ -1004,6 +1004,7 @@ raise NotImplementedError() genop_getarrayitem_gc_pure = genop_getarrayitem_gc + genop_getarrayitem_raw = genop_getarrayitem_gc def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py Tue Jul 20 20:55:30 2010 @@ -881,6 +881,7 @@ result_loc = self.force_allocate_reg(op.result) self.Perform(op, [base_loc, ofs_loc, imm(scale), imm(ofs)], result_loc) + consider_getarrayitem_raw = consider_getarrayitem_gc consider_getarrayitem_gc_pure = consider_getarrayitem_gc def consider_int_is_true(self, op, guard_op): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_gc_integration.py Tue Jul 20 20:55:30 2010 @@ -53,7 +53,7 @@ def initialize(self): self.gcrefs = GcRefList() self.gcrefs.initialize() - self.single_gcref_descr = GcPtrFieldDescr(0) + self.single_gcref_descr = GcPtrFieldDescr('', 0) rewrite_assembler = GcLLDescr_framework.rewrite_assembler.im_func Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/codewriter.py Tue Jul 20 20:55:30 2010 @@ -14,11 +14,12 @@ class CodeWriter(object): callcontrol = None # for tests - def __init__(self, cpu=None, jitdrivers_sd=[]): + def __init__(self, cpu=None, jitdrivers_sd=[], debug=False): self.cpu = cpu self.assembler = Assembler() self.callcontrol = CallControl(cpu, jitdrivers_sd) self._seen_files = set() + self.debug = debug def transform_func_to_jitcode(self, func, values, type_system='lltype'): """For testing.""" @@ -60,7 +61,8 @@ self.assembler.assemble(ssarepr, jitcode) # # print the resulting assembler - self.print_ssa_repr(ssarepr, portal_jd, verbose) + if self.debug: + self.print_ssa_repr(ssarepr, portal_jd, verbose) def make_jitcodes(self, verbose=False): log.info("making JitCodes...") Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/jtransform.py Tue Jul 20 20:55:30 2010 @@ -420,9 +420,14 @@ return SpaceOperation('new_array', [arraydescr, op.args[2]], op.result) + def rewrite_op_free(self, op): + assert op.args[1].value == 'raw' + ARRAY = op.args[0].concretetype.TO + return self._do_builtin_call(op, 'raw_free', [op.args[0]], + extra = (ARRAY,), extrakey = ARRAY) + def rewrite_op_getarrayitem(self, op): ARRAY = op.args[0].concretetype.TO - assert ARRAY._gckind == 'gc' if self._array_of_voids(ARRAY): return [] if op.args[0] in self.vable_array_vars: # for virtualizables @@ -436,13 +441,12 @@ # normal case follows arraydescr = self.cpu.arraydescrof(ARRAY) kind = getkind(op.result.concretetype) - return SpaceOperation('getarrayitem_gc_%s' % kind[0], + return SpaceOperation('getarrayitem_%s_%s' % (ARRAY._gckind, kind[0]), [op.args[0], arraydescr, op.args[1]], op.result) def rewrite_op_setarrayitem(self, op): ARRAY = op.args[0].concretetype.TO - assert ARRAY._gckind == 'gc' if self._array_of_voids(ARRAY): return [] if op.args[0] in self.vable_array_vars: # for virtualizables @@ -455,7 +459,7 @@ op.args[1], op.args[2]], None)] arraydescr = self.cpu.arraydescrof(ARRAY) kind = getkind(op.args[2].concretetype) - return SpaceOperation('setarrayitem_gc_%s' % kind[0], + return SpaceOperation('setarrayitem_%s_%s' % (ARRAY._gckind, kind[0]), [op.args[0], arraydescr, op.args[1], op.args[2]], None) @@ -663,7 +667,7 @@ return self._rewrite_symmetric(op) def _is_gc(self, v): - return v.concretetype.TO._gckind == 'gc' + return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc' def _rewrite_cmp_ptrs(self, op): if self._is_gc(op.args[0]): @@ -697,6 +701,42 @@ #return op raise NotImplementedError("cast_ptr_to_int") + def rewrite_op_force_cast(self, op): + from pypy.rpython.lltypesystem.rffi import size_and_sign, sizeof + from pypy.rlib.rarithmetic import intmask + assert not self._is_gc(op.args[0]) + size1, unsigned1 = size_and_sign(op.args[0].concretetype) + size2, unsigned2 = size_and_sign(op.result.concretetype) + if size2 >= sizeof(lltype.Signed): + return # the target type is LONG or ULONG + # + def bounds(size, unsigned): + if unsigned: + return 0, 1<<(8*size) + else: + return -(1<<(8*size-1)), 1<<(8*size-1) + min1, max1 = bounds(size1, unsigned1) + min2, max2 = bounds(size2, unsigned2) + if min2 <= min1 <= max1 <= max2: + return # the target type includes the source range + # + result = [] + v1 = op.args[0] + if min2: + c_min2 = Constant(min2, lltype.Signed) + v2 = Variable(); v2.concretetype = lltype.Signed + result.append(SpaceOperation('int_sub', [v1, c_min2], v2)) + else: + v2 = v1 + c_mask = Constant(int((1<<(8*size2))-1), lltype.Signed) + v3 = Variable(); v3.concretetype = lltype.Signed + result.append(SpaceOperation('int_and', [v2, c_mask], v3)) + if min2: + result.append(SpaceOperation('int_add', [v3, c_min2], op.result)) + else: + result[-1].result = op.result + return result + # ---------- # Renames, from the _old opname to the _new one. # The new operation is optionally further processed by rewrite_operation(). Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/support.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/support.py Tue Jul 20 20:55:30 2010 @@ -297,6 +297,11 @@ return lltype.malloc(ARRAY, n, flavor='raw') return _ll_1_raw_malloc + def build_ll_1_raw_free(ARRAY): + def _ll_1_raw_free(p): + lltype.free(p, flavor='raw') + return _ll_1_raw_free + class OOtypeHelpers: # ---------- dict ---------- Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_codewriter.py Tue Jul 20 20:55:30 2010 @@ -24,12 +24,17 @@ def as_vtable_size_descr(self): return self +class FakeArrayDescr(AbstractDescr): + def __init__(self, ARRAY): + self.ARRAY = ARRAY + class FakeCPU: def __init__(self, rtyper): self.rtyper = rtyper calldescrof = FakeCallDescr fielddescrof = FakeFieldDescr sizeof = FakeSizeDescr + arraydescrof = FakeArrayDescr class FakePolicy: def look_inside_graph(self, graph): @@ -163,10 +168,10 @@ def f(n): a = lltype.malloc(TP, n, flavor='raw') - #a[0] = n - #res = a[0] - #lltype.free(a, flavor='raw') - #return res + a[0] = n + res = a[0] + lltype.free(a, flavor='raw') + return res rtyper = support.annotate(f, [35]) jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) @@ -176,3 +181,6 @@ # s = jitdriver_sd.mainjitcode.dump() assert 'residual_call_ir_i $<* fn _ll_1_raw_malloc__Signed>' in s + assert 'setarrayitem_raw_i' in s + assert 'getarrayitem_raw_i' in s + assert 'residual_call_ir_v $<* fn _ll_1_raw_free__arrayPtr>' in s Modified: pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/codewriter/test/test_flatten.py Tue Jul 20 20:55:30 2010 @@ -729,3 +729,108 @@ int_between %i0, %i1, %i2 -> %i3 int_return %i3 """, transform=True) + + def test_force_cast(self): + from pypy.rpython.lltypesystem import rffi + + for FROM, TO, expected in [ + (rffi.SIGNEDCHAR, rffi.SIGNEDCHAR, ""), + (rffi.SIGNEDCHAR, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.SIGNEDCHAR, rffi.SHORT, ""), + (rffi.SIGNEDCHAR, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.SIGNEDCHAR, rffi.LONG, ""), + (rffi.SIGNEDCHAR, rffi.ULONG, ""), + + (rffi.UCHAR, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.UCHAR, rffi.UCHAR, ""), + (rffi.UCHAR, rffi.SHORT, ""), + (rffi.UCHAR, rffi.USHORT, ""), + (rffi.UCHAR, rffi.LONG, ""), + (rffi.UCHAR, rffi.ULONG, ""), + + (rffi.SHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.SHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.SHORT, rffi.SHORT, ""), + (rffi.SHORT, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.SHORT, rffi.LONG, ""), + (rffi.SHORT, rffi.ULONG, ""), + + (rffi.USHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.USHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.USHORT, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.USHORT, rffi.USHORT, ""), + (rffi.USHORT, rffi.LONG, ""), + (rffi.USHORT, rffi.ULONG, ""), + + (rffi.LONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.LONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.LONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.LONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.LONG, rffi.LONG, ""), + (rffi.LONG, rffi.ULONG, ""), + + (rffi.ULONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.ULONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.ULONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.ULONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.ULONG, rffi.LONG, ""), + (rffi.ULONG, rffi.ULONG, ""), + ]: + expected = [s.strip() for s in expected.splitlines()] + check_force_cast(FROM, TO, expected, 42) + check_force_cast(FROM, TO, expected, -42) + expected.append('int_return %i' + str(len(expected))) + expected = '\n'.join(expected) + # + def f(n): + return rffi.cast(TO, n) + self.encoding_test(f, [rffi.cast(FROM, 42)], expected, + transform=True) + + def test_force_cast_pointer(self): + from pypy.rpython.lltypesystem import rffi + def h(p): + return rffi.cast(rffi.VOIDP, p) + self.encoding_test(h, [lltype.nullptr(rffi.CCHARP.TO)], """ + int_return %i0 + """, transform=True) + + +def check_force_cast(FROM, TO, operations, value): + """Check that the test is correctly written...""" + from pypy.rpython.lltypesystem import rffi + import re + r = re.compile('(\w+) \%i\d, \$(-?\d+)') + # + value = rffi.cast(FROM, value) + value = rffi.cast(lltype.Signed, value) + # + expected_value = rffi.cast(TO, value) + expected_value = rffi.cast(lltype.Signed, expected_value) + # + for op in operations: + match = r.match(op) + assert match, "line %r does not match regexp" % (op,) + opname = match.group(1) + if opname == 'int_add': value += int(match.group(2)) + elif opname == 'int_sub': value -= int(match.group(2)) + elif opname == 'int_and': value &= int(match.group(2)) + else: assert 0, opname + # + assert rffi.cast(lltype.Signed, value) == expected_value Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/blackhole.py Tue Jul 20 20:55:30 2010 @@ -990,6 +990,13 @@ bhimpl_getarrayitem_gc_pure_r = bhimpl_getarrayitem_gc_r bhimpl_getarrayitem_gc_pure_f = bhimpl_getarrayitem_gc_f + @arguments("cpu", "i", "d", "i", returns="i") + def bhimpl_getarrayitem_raw_i(cpu, array, arraydescr, index): + return cpu.bh_getarrayitem_raw_i(arraydescr, array, index) + @arguments("cpu", "i", "d", "i", returns="f") + def bhimpl_getarrayitem_raw_f(cpu, array, arraydescr, index): + return cpu.bh_getarrayitem_raw_f(arraydescr, array, index) + @arguments("cpu", "r", "d", "i", "i") def bhimpl_setarrayitem_gc_i(cpu, array, arraydescr, index, newvalue): cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue) @@ -1000,6 +1007,15 @@ def bhimpl_setarrayitem_gc_f(cpu, array, arraydescr, index, newvalue): cpu.bh_setarrayitem_gc_f(arraydescr, array, index, newvalue) + @arguments("cpu", "i", "d", "i", "i") + def bhimpl_setarrayitem_raw_i(cpu, array, arraydescr, index, newvalue): + cpu.bh_setarrayitem_raw_i(arraydescr, array, index, newvalue) + @arguments("cpu", "i", "d", "i", "f") + def bhimpl_setarrayitem_raw_f(cpu, array, arraydescr, index, newvalue): + cpu.bh_setarrayitem_raw_f(arraydescr, array, index, newvalue) + + # note, there is no 'r' here, since it can't happen + @arguments("cpu", "r", "d", returns="i") def bhimpl_arraylen_gc(cpu, array, arraydescr): return cpu.bh_arraylen_gc(arraydescr, array) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/executor.py Tue Jul 20 20:55:30 2010 @@ -90,6 +90,15 @@ else: return BoxInt(cpu.bh_getarrayitem_gc_i(arraydescr, array, index)) +def do_getarrayitem_raw(cpu, _, arraybox, indexbox, arraydescr): + array = arraybox.getref_base() + index = indexbox.getint() + assert not arraydescr.is_array_of_pointers() + if arraydescr.is_array_of_floats(): + return BoxFloat(cpu.bh_getarrayitem_raw_f(arraydescr, array, index)) + else: + return BoxInt(cpu.bh_getarrayitem_raw_i(arraydescr, array, index)) + def do_setarrayitem_gc(cpu, _, arraybox, indexbox, itembox, arraydescr): array = arraybox.getref_base() index = indexbox.getint() @@ -101,6 +110,15 @@ else: cpu.bh_setarrayitem_gc_i(arraydescr, array, index, itembox.getint()) +def do_setarrayitem_raw(cpu, _, arraybox, indexbox, itembox, arraydescr): + array = arraybox.getint() + index = indexbox.getint() + assert not arraydescr.is_array_of_pointers() + if arraydescr.is_array_of_floats(): + cpu.bh_setarrayitem_raw_f(arraydescr, array, index, itembox.getfloat()) + else: + cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint()) + def do_getfield_gc(cpu, _, structbox, fielddescr): struct = structbox.getref_base() if fielddescr.is_pointer_field(): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py Tue Jul 20 20:55:30 2010 @@ -18,6 +18,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.metainterp.history import AbstractDescr, make_hashable_int + def optimize_loop_1(metainterp_sd, loop): """Optimize loop.operations to make it match the input of loop.specnodes and to remove internal overheadish operations. Note that loop.specnodes @@ -992,6 +993,14 @@ self.make_equal_to(op.result, v1) else: self.optimize_default(op) + + def optimize_INT_SUB(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v2.is_constant() and v2.box.getint() == 0: + self.make_equal_to(op.result, v1) + else: + return self.optimize_default(op) optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/pyjitpl.py Tue Jul 20 20:55:30 2010 @@ -378,6 +378,14 @@ opimpl_getarrayitem_gc_f = _opimpl_getarrayitem_gc_any @arguments("box", "descr", "box") + def _opimpl_getarrayitem_raw_any(self, arraybox, arraydescr, indexbox): + return self.execute_with_descr(rop.GETARRAYITEM_RAW, + arraydescr, arraybox, indexbox) + + opimpl_getarrayitem_raw_i = _opimpl_getarrayitem_raw_any + opimpl_getarrayitem_raw_f = _opimpl_getarrayitem_raw_any + + @arguments("box", "descr", "box") def _opimpl_getarrayitem_gc_pure_any(self, arraybox, arraydescr, indexbox): return self.execute_with_descr(rop.GETARRAYITEM_GC_PURE, arraydescr, arraybox, indexbox) @@ -396,6 +404,15 @@ opimpl_setarrayitem_gc_r = _opimpl_setarrayitem_gc_any opimpl_setarrayitem_gc_f = _opimpl_setarrayitem_gc_any + @arguments("box", "descr", "box", "box") + def _opimpl_setarrayitem_raw_any(self, arraybox, arraydescr, + indexbox, itembox): + self.execute_with_descr(rop.SETARRAYITEM_RAW, arraydescr, arraybox, + indexbox, itembox) + + opimpl_setarrayitem_raw_i = _opimpl_setarrayitem_raw_any + opimpl_setarrayitem_raw_f = _opimpl_setarrayitem_raw_any + @arguments("box", "descr") def opimpl_arraylen_gc(self, arraybox, arraydescr): return self.execute_with_descr(rop.ARRAYLEN_GC, arraydescr, arraybox) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/resoperation.py Tue Jul 20 20:55:30 2010 @@ -199,6 +199,7 @@ '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- 'GETARRAYITEM_GC/2d', + 'GETARRAYITEM_RAW/2d', 'GETFIELD_GC/1d', 'GETFIELD_RAW/1d', 'NEW/0d', @@ -209,7 +210,7 @@ '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'SETARRAYITEM_GC/3d', - 'SETARRAYITEM_RAW/3d',#only added by backend.llsupport.gc.rewrite_assembler + 'SETARRAYITEM_RAW/3d', 'SETFIELD_GC/2d', 'SETFIELD_RAW/2d', 'ARRAYCOPY/7d', # removed before it's passed to the backend Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_basic.py Tue Jul 20 20:55:30 2010 @@ -1562,6 +1562,35 @@ res = self.interp_operations(f, [3, 2]) assert res == 1 + def test_raw_malloc_and_access(self): + from pypy.rpython.lltypesystem import rffi + + TP = rffi.CArray(lltype.Signed) + + def f(n): + a = lltype.malloc(TP, n, flavor='raw') + a[0] = n + res = a[0] + lltype.free(a, flavor='raw') + return res + + res = self.interp_operations(f, [10]) + assert res == 10 + + def test_raw_malloc_and_access_float(self): + from pypy.rpython.lltypesystem import rffi + + TP = rffi.CArray(lltype.Float) + + def f(n, f): + a = lltype.malloc(TP, n, flavor='raw') + a[0] = f + res = a[0] + lltype.free(a, flavor='raw') + return res + + res = self.interp_operations(f, [10, 3.5]) + assert res == 3.5 class TestOOtype(BasicTests, OOJitMixin): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py Tue Jul 20 20:55:30 2010 @@ -2051,8 +2051,19 @@ jump(i1, i0) """ self.optimize_loop(ops, 'Not, Not', expected) - - + + def test_fold_partially_constant_ops(self): + ops = """ + [i0] + i1 = int_sub(i0, 0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + # ---------- def make_fail_descr(self): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_recursive.py Tue Jul 20 20:55:30 2010 @@ -523,7 +523,7 @@ def test_trace_from_start(self): def p(pc, code): code = hlstr(code) - return "%s %d %s" % (code, pc, code[pc]) + return "'%s' at %d: %s" % (code, pc, code[pc]) def c(pc, code): return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], @@ -537,9 +537,9 @@ op = code[pc] if op == "+": n += 7 - if op == "-": + elif op == "-": n -= 1 - if op == "c": + elif op == "c": n = f('---', n) elif op == "l": if n > 0: @@ -556,6 +556,7 @@ result = 0 for i in range(m): result += f('+-cl--', i) + g(50) self.meta_interp(g, [50], backendopt=True) self.check_tree_loop_count(3) self.check_history(int_add=1) Modified: pypy/branch/x86-64-jit-backend/pypy/module/_file/interp_file.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_file/interp_file.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_file/interp_file.py Tue Jul 20 20:55:30 2010 @@ -4,6 +4,7 @@ from pypy.rlib.rarithmetic import r_longlong from pypy.module._file.interp_stream import W_AbstractStream from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror +from pypy.module.posix.interp_posix import dispatch_filename from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.typedef import TypeDef, GetSetProperty @@ -81,11 +82,11 @@ # file lock. They don't convert StreamErrors to OperationErrors, too. def direct___init__(self, w_name, mode='r', buffering=-1): - name = self.space.str_w(w_name) self.direct_close() self.w_name = w_name self.check_mode_ok(mode) - stream = streamio.open_file_as_stream(name, mode, buffering) + stream = dispatch_filename(streamio.open_file_as_stream)( + self.space, w_name, mode, buffering) fd = stream.try_to_find_file_descriptor() self.fdopenstream(stream, fd, mode) Modified: pypy/branch/x86-64-jit-backend/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_file/test/test_file.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_file/test/test_file.py Tue Jul 20 20:55:30 2010 @@ -125,6 +125,15 @@ assert type(res) is str f.close() + def test_unicode_filename(self): + import sys + try: + u'\xe9'.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + skip("encoding not good enough") + f = self.file(self.temppath + u'\xe9', "w") + f.close() + def test_oserror_has_filename(self): try: f = self.file("file that is clearly not there") Modified: pypy/branch/x86-64-jit-backend/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_locale/interp_locale.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_locale/interp_locale.py Tue Jul 20 20:55:30 2010 @@ -132,7 +132,13 @@ space.is_true(space.isinstance(w_s2, space.w_str)): s1, s2 = space.str_w(w_s1), space.str_w(w_s2) - return space.wrap(_strcoll(rffi.str2charp(s1), rffi.str2charp(s2))) + s1_c = rffi.str2charp(s1) + s2_c = rffi.str2charp(s2) + try: + return space.wrap(_strcoll(s1_c, s2_c)) + finally: + rffi.free_charp(s1_c) + rffi.free_charp(s2_c) #if not space.is_true(space.isinstance(w_s1, space.w_unicode)) and \ # not space.is_true(space.isinstance(w_s2, space.w_unicode)): @@ -143,7 +149,12 @@ s1_c = rffi.unicode2wcharp(s1) s2_c = rffi.unicode2wcharp(s2) - result = _wcscoll(s1_c, s2_c) + try: + result = _wcscoll(s1_c, s2_c) + finally: + rffi.free_wcharp(s1_c) + rffi.free_wcharp(s2_c) + return space.wrap(result) strcoll.unwrap_spec = [ObjSpace, W_Root, W_Root] @@ -156,13 +167,21 @@ n1 = len(s) + 1 buf = lltype.malloc(rffi.CCHARP.TO, n1, flavor="raw", zero=True) - n2 = _strxfrm(buf, rffi.str2charp(s), n1) + 1 + s_c = rffi.str2charp(s) + try: + n2 = _strxfrm(buf, s_c, n1) + 1 + finally: + rffi.free_charp(s_c) if n2 > n1: # more space needed lltype.free(buf, flavor="raw") buf = lltype.malloc(rffi.CCHARP.TO, intmask(n2), flavor="raw", zero=True) - _strxfrm(buf, rffi.str2charp(s), n2) + s_c = rffi.str2charp(s) + try: + _strxfrm(buf, s_c, n2) + finally: + rffi.free_charp(s_c) val = rffi.charp2str(buf) lltype.free(buf, flavor="raw") @@ -194,7 +213,11 @@ def gettext(space, msg): """gettext(msg) -> string Return translation of msg.""" - return space.wrap(rffi.charp2str(_gettext(rffi.str2charp(msg)))) + msg_c = rffi.str2charp(msg) + try: + return space.wrap(rffi.charp2str(_gettext(msg_c))) + finally: + rffi.free_charp(msg_c) gettext.unwrap_spec = [ObjSpace, str] @@ -205,10 +228,20 @@ Return translation of msg in domain.""" if space.is_w(w_domain, space.w_None): domain = None - result = _dgettext(domain, rffi.str2charp(msg)) + msg_c = rffi.str2charp(msg) + try: + result = _dgettext(domain, msg_c) + finally: + rffi.free_charp(msg_c) else: domain = space.str_w(w_domain) - result = _dgettext(rffi.str2charp(domain), rffi.str2charp(msg)) + domain_c = rffi.str2charp(domain) + msg_c = rffi.str2charp(msg) + try: + result = _dgettext(domain_c, msg_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(msg_c) return space.wrap(rffi.charp2str(result)) @@ -223,12 +256,21 @@ if space.is_w(w_domain, space.w_None): domain = None - result = _dcgettext(domain, rffi.str2charp(msg), - rffi.cast(rffi.INT, category)) + msg_c = rffi.str2charp(msg) + try: + result = _dcgettext(domain, msg_c, rffi.cast(rffi.INT, category)) + finally: + rffi.free_charp(msg_c) else: domain = space.str_w(w_domain) - result = _dcgettext(rffi.str2charp(domain), rffi.str2charp(msg), - rffi.cast(rffi.INT, category)) + domain_c = rffi.str2charp(domain) + msg_c = rffi.str2charp(msg) + try: + result = _dcgettext(domain_c, msg_c, + rffi.cast(rffi.INT, category)) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(msg_c) return space.wrap(rffi.charp2str(result)) @@ -246,7 +288,11 @@ result = _textdomain(domain) else: domain = space.str_w(w_domain) - result = _textdomain(rffi.str2charp(domain)) + domain_c = rffi.str2charp(domain) + try: + result = _textdomain(domain_c) + finally: + rffi.free_charp(domain_c) return space.wrap(rffi.charp2str(result)) @@ -261,11 +307,20 @@ if space.is_w(w_dir, space.w_None): dir = None - dirname = _bindtextdomain(rffi.str2charp(domain), dir) + domain_c = rffi.str2charp(domain) + try: + dirname = _bindtextdomain(domain_c, dir) + finally: + rffi.free_charp(domain_c) else: dir = space.str_w(w_dir) - dirname = _bindtextdomain(rffi.str2charp(domain), - rffi.str2charp(dir)) + domain_c = rffi.str2charp(domain) + dir_c = rffi.str2charp(dir) + try: + dirname = _bindtextdomain(domain_c, dir_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(dir_c) if not dirname: errno = rposix.get_errno() @@ -284,12 +339,20 @@ if space.is_w(w_codeset, space.w_None): codeset = None - result = _bind_textdomain_codeset( - rffi.str2charp(domain), codeset) + domain_c = rffi.str2charp(domain) + try: + result = _bind_textdomain_codeset(domain_c, codeset) + finally: + rffi.free_charp(domain_c) else: codeset = space.str_w(w_codeset) - result = _bind_textdomain_codeset(rffi.str2charp(domain), - rffi.str2charp(codeset)) + domain_c = rffi.str2charp(domain) + codeset_c = rffi.str2charp(codeset) + try: + result = _bind_textdomain_codeset(domain_c, codeset_c) + finally: + rffi.free_charp(domain_c) + rffi.free_charp(codeset_c) if not result: return space.w_None Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/api.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/api.py Tue Jul 20 20:55:30 2010 @@ -104,8 +104,12 @@ for name in ("pypy_decl.h", "pypy_macros.h"): headers.append(udir.join(name)) for header in headers: - header.copy(dstdir) target = dstdir.join(header.basename) + try: + header.copy(dstdir) + except py.error.EACCES: + target.remove() # maybe it was a read-only file + header.copy(dstdir) target.chmod(0444) # make the file read-only, to make sure that nobody # edits it by mistake Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/methodobject.py Tue Jul 20 20:55:30 2010 @@ -100,7 +100,11 @@ return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) def get_doc(space, self): - return space.wrap(rffi.charp2str(self.ml.c_ml_doc)) + doc = self.ml.c_ml_doc + if doc: + return space.wrap(rffi.charp2str(doc)) + else: + return space.w_None class W_PyCMethodObject(W_PyCFunctionObject): Modified: pypy/branch/x86-64-jit-backend/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/marshal/interp_marshal.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/marshal/interp_marshal.py Tue Jul 20 20:55:30 2010 @@ -365,8 +365,8 @@ return self.reader.read(n) def get1(self): - # convince typer to use a char - return chr(ord(self.get(1))) + # the [0] is used to convince the annotator to return a char + return self.get(1)[0] def atom_str(self, typecode): self.start(typecode) Modified: pypy/branch/x86-64-jit-backend/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/posix/interp_posix.py Tue Jul 20 20:55:30 2010 @@ -1,8 +1,9 @@ from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong from pypy.rlib.unroll import unrolling_iterable -from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 from pypy.rpython.module.ll_os import RegisterOs from pypy.rpython.module import ll_os_stat from pypy.rpython.lltypesystem import rffi, lltype @@ -12,15 +13,78 @@ import os, sys _WIN = sys.platform == 'win32' -def open(space, fname, flag, mode=0777): +class FileEncoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_bytes = space.call_method(self.w_obj, 'encode', + getfilesystemencoding(space)) + return space.str_w(w_bytes) + + def as_unicode(self): + return self.space.unicode_w(self.w_obj) + +class FileDecoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + return self.space.str_w(self.w_obj) + + def as_unicode(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_unicode = space.call_method(self.w_obj, 'decode', + getfilesystemencoding(space)) + return space.unicode_w(w_unicode) + + at specialize.memo() +def dispatch_filename(func, tag=0): + def dispatch(space, w_fname, *args): + if space.isinstance_w(w_fname, space.w_unicode): + fname = FileEncoder(space, w_fname) + return func(fname, *args) + else: + fname = space.str_w(w_fname) + return func(fname, *args) + return dispatch + + at specialize.memo() +def dispatch_filename_2(func): + def dispatch(space, w_fname1, w_fname2, *args): + if space.isinstance_w(w_fname1, space.w_unicode): + fname1 = FileEncoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname1 = FileDecoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + return dispatch + +def open(space, w_fname, flag, mode=0777): """Open a file (for low level IO). Return a file descriptor (a small integer).""" - try: - fd = os.open(fname, flag, mode) + try: + fd = dispatch_filename(rposix.open)( + space, w_fname, flag, mode) except OSError, e: - raise wrap_oserror(space, e, fname) + raise wrap_oserror2(space, e, w_fname) return space.wrap(fd) -open.unwrap_spec = [ObjSpace, 'path', "c_int", "c_int"] +open.unwrap_spec = [ObjSpace, W_Root, "c_int", "c_int"] def lseek(space, fd, pos, how): """Set the current position of a file descriptor. Return the new position. @@ -159,7 +223,7 @@ return build_stat_result(space, st) fstat.unwrap_spec = [ObjSpace, "c_int"] -def stat(space, path): +def stat(space, w_path): """Perform a stat system call on the given path. Return an object with (at least) the following attributes: st_mode @@ -175,22 +239,22 @@ """ try: - st = os.stat(path) + st = dispatch_filename(rposix.stat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -stat.unwrap_spec = [ObjSpace, 'path'] +stat.unwrap_spec = [ObjSpace, W_Root] -def lstat(space, path): +def lstat(space, w_path): "Like stat(path), but do no follow symbolic links." try: - st = os.lstat(path) + st = dispatch_filename(rposix.lstat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -lstat.unwrap_spec = [ObjSpace, 'path'] +lstat.unwrap_spec = [ObjSpace, W_Root] class StatState(object): def __init__(self, space): @@ -231,7 +295,7 @@ raise wrap_oserror(space, e) dup2.unwrap_spec = [ObjSpace, "c_int", "c_int"] -def access(space, path, mode): +def access(space, w_path, mode): """ access(path, mode) -> 1 if granted, 0 otherwise @@ -242,12 +306,12 @@ existence, or the inclusive-OR of R_OK, W_OK, and X_OK. """ try: - ok = os.access(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) + ok = dispatch_filename(rposix.access)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) else: return space.wrap(ok) -access.unwrap_spec = [ObjSpace, str, "c_int"] +access.unwrap_spec = [ObjSpace, W_Root, "c_int"] def times(space): @@ -278,32 +342,38 @@ return space.wrap(rc) system.unwrap_spec = [ObjSpace, str] -def unlink(space, path): +def unlink(space, w_path): """Remove a file (same as remove(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -unlink.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +unlink.unwrap_spec = [ObjSpace, W_Root] -def remove(space, path): +def remove(space, w_path): """Remove a file (same as unlink(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -remove.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +remove.unwrap_spec = [ObjSpace, W_Root] -def _getfullpathname(space, path): +def _getfullpathname(space, w_path): """helper for ntpath.abspath """ - posix = __import__(os.name) # nt specific try: - fullpath = posix._getfullpathname(path) + if space.isinstance_w(w_path, space.w_unicode): + path = FileEncoder(space, w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) + else: + path = space.str_w(w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) except OSError, e: - raise wrap_oserror(space, e, path) - else: - return space.wrap(fullpath) -_getfullpathname.unwrap_spec = [ObjSpace, str] + raise wrap_oserror2(space, e, w_path) + else: + return w_fullpath +_getfullpathname.unwrap_spec = [ObjSpace, W_Root] def getcwd(space): """Return the current working directory.""" @@ -315,35 +385,46 @@ return space.wrap(cur) getcwd.unwrap_spec = [ObjSpace] -def getcwdu(space): - """Return the current working directory as a unicode string.""" - # XXX ascii encoding for now - return space.call_method(getcwd(space), 'decode') +if sys.platform == 'win32': + def getcwdu(space): + """Return the current working directory as a unicode string.""" + try: + cur = os.getcwdu() + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(cur) +else: + def getcwdu(space): + """Return the current working directory as a unicode string.""" + filesystemencoding = space.sys.filesystemencoding + return space.call_method(getcwd(space), 'decode', + space.wrap(filesystemencoding)) getcwdu.unwrap_spec = [ObjSpace] -def chdir(space, path): +def chdir(space, w_path): """Change the current working directory to the specified path.""" try: - os.chdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -chdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.chdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chdir.unwrap_spec = [ObjSpace, W_Root] -def mkdir(space, path, mode=0777): +def mkdir(space, w_path, mode=0777): """Create a directory.""" try: - os.mkdir(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -mkdir.unwrap_spec = [ObjSpace, str, "c_int"] + dispatch_filename(rposix.mkdir)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +mkdir.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rmdir(space, path): +def rmdir(space, w_path): """Remove a directory.""" try: - os.rmdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -rmdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.rmdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +rmdir.unwrap_spec = [ObjSpace, W_Root] def strerror(space, errno): """Translate an error code to a message string.""" @@ -410,7 +491,7 @@ unsetenv.unwrap_spec = [ObjSpace, str] -def listdir(space, dirname): +def listdir(space, w_dirname): """Return a list containing the names of the entries in the directory. \tpath: path of directory to list @@ -418,12 +499,18 @@ The list is in arbitrary order. It does not include the special entries '.' and '..' even if they are present in the directory.""" try: - result = os.listdir(dirname) + if space.isinstance_w(w_dirname, space.w_unicode): + dirname = FileEncoder(space, w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] + else: + dirname = space.str_w(w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] except OSError, e: - raise wrap_oserror(space, e, dirname) - result_w = [space.wrap(s) for s in result] + raise wrap_oserror2(space, e, w_dirname) return space.newlist(result_w) -listdir.unwrap_spec = [ObjSpace, str] +listdir.unwrap_spec = [ObjSpace, W_Root] def pipe(space): "Create a pipe. Returns (read_end, write_end)." @@ -434,21 +521,21 @@ return space.newtuple([space.wrap(fd1), space.wrap(fd2)]) pipe.unwrap_spec = [ObjSpace] -def chmod(space, path, mode): +def chmod(space, w_path, mode): "Change the access permissions of a file." - try: - os.chmod(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -chmod.unwrap_spec = [ObjSpace, str, "c_int"] + try: + dispatch_filename(rposix.chmod)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chmod.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rename(space, old, new): +def rename(space, w_old, w_new): "Rename a file or directory." - try: - os.rename(old, new) - except OSError, e: + try: + dispatch_filename_2(rposix.rename)(space, w_old, w_new) + except OSError, e: raise wrap_oserror(space, e) -rename.unwrap_spec = [ObjSpace, str, str] +rename.unwrap_spec = [ObjSpace, W_Root, W_Root] def umask(space, mask): "Set the current numeric umask and return the previous umask." @@ -576,7 +663,7 @@ raise wrap_oserror(space, e) execve.unwrap_spec = [ObjSpace, str, W_Root, W_Root] -def utime(space, path, w_tuple): +def utime(space, w_path, w_tuple): """ utime(path, (atime, mtime)) utime(path, None) @@ -585,10 +672,10 @@ """ if space.is_w(w_tuple, space.w_None): try: - os.utime(path, None) + dispatch_filename(rposix.utime, 1)(space, w_path, None) return except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) try: msg = "utime() arg 2 must be a tuple (atime, mtime) or None" args_w = space.fixedview(w_tuple) @@ -596,14 +683,14 @@ raise OperationError(space.w_TypeError, space.wrap(msg)) actime = space.float_w(args_w[0]) modtime = space.float_w(args_w[1]) - os.utime(path, (actime, modtime)) + dispatch_filename(rposix.utime, 2)(space, w_path, (actime, modtime)) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) except OperationError, e: if not e.match(space, space.w_TypeError): raise raise OperationError(space.w_TypeError, space.wrap(msg)) -utime.unwrap_spec = [ObjSpace, str, W_Root] +utime.unwrap_spec = [ObjSpace, W_Root, W_Root] def setsid(space): """setsid() -> pid Modified: pypy/branch/x86-64-jit-backend/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/posix/test/test_posix2.py Tue Jul 20 20:55:30 2010 @@ -32,6 +32,9 @@ # even when running on top of CPython 2.4. os.stat_float_times(True) + # Initialize sys.filesystemencoding + space.call_method(space.getbuiltinmodule('sys'), 'getfilesystemencoding') + def need_sparse_files(): if sys.platform == 'darwin': py.test.skip("no sparse files on default Mac OS X file system") @@ -706,6 +709,28 @@ except OSError: pass +class AppTestUnicodeFilename: + def setup_class(cls): + ufilename = (unicode(udir.join('test_unicode_filename_')) + + u'\u65e5\u672c.txt') # "Japan" + try: + f = file(ufilename, 'w') + except UnicodeEncodeError: + py.test.skip("encoding not good enough") + f.write("test") + f.close() + cls.space = space + cls.w_filename = space.wrap(ufilename) + cls.w_posix = space.appexec([], GET_POSIX) + + def test_open(self): + fd = self.posix.open(self.filename, self.posix.O_RDONLY) + try: + content = self.posix.read(fd, 50) + finally: + self.posix.close(fd) + assert content == "test" + class TestPexpect(object): # XXX replace with AppExpectTest class as soon as possible Modified: pypy/branch/x86-64-jit-backend/pypy/module/termios/interp_termios.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/termios/interp_termios.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/termios/interp_termios.py Tue Jul 20 20:55:30 2010 @@ -60,8 +60,8 @@ # last one need to be chosen carefully cc_w = [space.wrap(i) for i in cc] if lflag & termios.ICANON: - cc_w[termios.VMIN] = space.wrap(ord(cc[termios.VMIN])) - cc_w[termios.VTIME] = space.wrap(ord(cc[termios.VTIME])) + cc_w[termios.VMIN] = space.wrap(ord(cc[termios.VMIN][0])) + cc_w[termios.VTIME] = space.wrap(ord(cc[termios.VTIME][0])) w_cc = space.newlist(cc_w) l_w.append(w_cc) return space.newlist(l_w) Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/stringobject.py Tue Jul 20 20:55:30 2010 @@ -863,7 +863,7 @@ space.w_TypeError, "ord() expected a character, but string " "of length %d found", len(u_str)) - return space.wrap(ord(u_str)) + return space.wrap(ord(u_str[0])) def getnewargs__String(space, w_str): return space.newtuple([wrapstr(space, w_str._value)]) Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/libffi.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/libffi.py Tue Jul 20 20:55:30 2010 @@ -402,7 +402,7 @@ restype = ffi_type_sint32 elif restype.c_size <= 8: restype = ffi_type_sint64 - + res = c_ffi_prep_cif(self.ll_cif, cc, rffi.cast(rffi.UINT, argnum), restype, self.ll_argtypes) Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rarithmetic.py Tue Jul 20 20:55:30 2010 @@ -33,7 +33,7 @@ """ -import math +import sys, math from pypy.rpython import extregistry from pypy.rlib import objectmodel @@ -54,13 +54,10 @@ NAN = INFINITY / INFINITY def isinf(x): - return x != 0.0 and x / 2 == x + return x == INFINITY or x == -INFINITY -# To get isnan, working x-platform and both on 2.3 and 2.4, is a -# horror. I think this works (for reasons I don't really want to talk -# about), and probably when implemented on top of pypy, too. def isnan(v): - return v != v*1.0 or (v == 1.0 and v == 2.0) + return v != v def intmask(n): if isinstance(n, int): @@ -116,14 +113,23 @@ "NOT_RPYTHON" return _local_ovfcheck(int(long(a) << b)) -FL_MAXINT = float(LONG_TEST-1) -FL_MININT = float(-LONG_TEST) - -def ovfcheck_float_to_int(x): - _, intp = math.modf(x) - if FL_MININT < intp < FL_MAXINT: - return int(intp) - raise OverflowError +# Strange things happening for float to int on 64 bit: +# int(float(i)) != i because of rounding issues. +# These are the minimum and maximum float value that can +# successfully be casted to an int. +if sys.maxint == 2147483647: + def ovfcheck_float_to_int(x): + if -2147483649.0 < x < 2147483648.0: + return int(x) + raise OverflowError +else: + # The following values are not quite +/-sys.maxint. + # Note the "<= x <" here, as opposed to "< x <" above. + # This is justified by test_typed in translator/c/test. + def ovfcheck_float_to_int(x): + if -9223372036854776832.0 <= x < 9223372036854775296.0: + return int(x) + raise OverflowError def compute_restype(self_type, other_type): if self_type is other_type: Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rdynload.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rdynload.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rdynload.py Tue Jul 20 20:55:30 2010 @@ -18,8 +18,6 @@ if _WIN32: from pypy.rlib import rwin32 - -if _WIN32: includes = ['windows.h'] else: includes = ['dlfcn.h'] Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rposix.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rposix.py Tue Jul 20 20:55:30 2010 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rlib.rarithmetic import intmask +from pypy.rlib.objectmodel import specialize class CConstantErrno(CConstant): # these accessors are used when calling get_errno() or set_errno() @@ -42,3 +43,102 @@ os.close(fd) except OSError: pass + +#___________________________________________________________________ +# Wrappers around posix functions, that accept either strings, or +# instances with a "as_bytes()" method. +# - pypy.modules.posix.interp_posix passes an object containing a unicode path +# which can encode itself with sys.filesystemencoding. +# - but pypy.rpython.module.ll_os.py on Windows will replace these functions +# with other wrappers that directly handle unicode strings. + at specialize.argtype(0) +def open(path, flags, mode): + if isinstance(path, str): + return os.open(path, flags, mode) + else: + return os.open(path.as_bytes(), flags, mode) + + at specialize.argtype(0) +def stat(path): + if isinstance(path, str): + return os.stat(path) + else: + return os.stat(path.as_bytes()) + + at specialize.argtype(0) +def lstat(path): + if isinstance(path, str): + return os.lstat(path) + else: + return os.lstat(path.as_bytes()) + + at specialize.argtype(0) +def unlink(path): + if isinstance(path, str): + return os.unlink(path) + else: + return os.unlink(path.as_bytes()) + + at specialize.argtype(0, 1) +def rename(path1, path2): + if isinstance(path1, str): + return os.rename(path1, path2) + else: + return os.rename(path1.as_bytes(), path2.as_bytes()) + + at specialize.argtype(0) +def listdir(dirname): + if isinstance(dirname, str): + return os.listdir(dirname) + else: + return os.listdir(dirname.as_bytes()) + + at specialize.argtype(0) +def access(path, mode): + if isinstance(path, str): + return os.access(path, mode) + else: + return os.access(path.as_bytes(), mode) + + at specialize.argtype(0) +def chmod(path, mode): + if isinstance(path, str): + return os.chmod(path, mode) + else: + return os.chmod(path.as_bytes(), mode) + + at specialize.argtype(0, 1) +def utime(path, times): + if isinstance(path, str): + return os.utime(path, times) + else: + return os.utime(path.as_bytes(), times) + + at specialize.argtype(0) +def chdir(path): + if isinstance(path, str): + return os.chdir(path) + else: + return os.chdir(path.as_bytes()) + + at specialize.argtype(0) +def mkdir(path, mode=0777): + if isinstance(path, str): + return os.mkdir(path, mode) + else: + return os.mkdir(path.as_bytes(), mode) + + at specialize.argtype(0) +def rmdir(path): + if isinstance(path, str): + return os.rmdir(path) + else: + return os.rmdir(path.as_bytes()) + +if os.name == 'nt': + import nt + def _getfullpathname(path): + if isinstance(path, str): + return nt._getfullpathname(path) + else: + return nt._getfullpathname(path.as_bytes()) Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rwin32.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rwin32.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rwin32.py Tue Jul 20 20:55:30 2010 @@ -31,6 +31,7 @@ DWORD = rffi_platform.SimpleType("DWORD", rffi.UINT) BOOL = rffi_platform.SimpleType("BOOL", rffi.LONG) BYTE = rffi_platform.SimpleType("BYTE", rffi.UCHAR) + WCHAR = rffi_platform.SimpleType("WCHAR", rffi.UCHAR) INT = rffi_platform.SimpleType("INT", rffi.INT) LONG = rffi_platform.SimpleType("LONG", rffi.LONG) PLONG = rffi_platform.SimpleType("PLONG", rffi.LONGP) @@ -38,6 +39,8 @@ LPCVOID = rffi_platform.SimpleType("LPCVOID", rffi.VOIDP) LPSTR = rffi_platform.SimpleType("LPSTR", rffi.CCHARP) LPCSTR = rffi_platform.SimpleType("LPCSTR", rffi.CCHARP) + LPWSTR = rffi_platform.SimpleType("LPWSTR", rffi.CWCHARP) + LPCWSTR = rffi_platform.SimpleType("LPCWSTR", rffi.CWCHARP) LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.INTP) SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T) ULONG_PTR = rffi_platform.SimpleType("ULONG_PTR", rffi.ULONG) @@ -87,6 +90,10 @@ GetLastError = winexternal('GetLastError', [], DWORD) SetLastError = winexternal('SetLastError', [DWORD], lltype.Void) + # In tests, the first call to GetLastError is always wrong, because error + # is hidden by operations in ll2ctypes. Call it now. + GetLastError() + LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], rffi.VOIDP) GetProcAddress = winexternal('GetProcAddress', [rffi.VOIDP, rffi.CCHARP], @@ -129,13 +136,29 @@ } return 0; }''') - exename = static_platform.compile( - [cfile], ExternalCompilationInfo(), - outputfilename = "dosmaperr", - standalone=True) - output = os.popen(str(exename)) - errors = dict(map(int, line.split()) - for line in output) + try: + exename = static_platform.compile( + [cfile], ExternalCompilationInfo(), + outputfilename = "dosmaperr", + standalone=True) + except WindowsError: + # Fallback for the mingw32 compiler + errors = { + 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, + 11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, + 22: 13, 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13, + 29: 13, 30: 13, 31: 13, 32: 13, 33: 13, 34: 13, 35: 13, + 36: 13, 53: 2, 65: 13, 67: 2, 80: 17, 82: 13, 83: 13, 89: 11, + 108: 13, 109: 32, 112: 28, 114: 9, 128: 10, 129: 10, 130: 9, + 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17, + 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8, + 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8, + 202: 8, 206: 2, 215: 11, 1816: 12, + } + else: + output = os.popen(str(exename)) + errors = dict(map(int, line.split()) + for line in output) return errors, errno.EINVAL # A bit like strerror... Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/streamio.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/streamio.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/streamio.py Tue Jul 20 20:55:30 2010 @@ -38,7 +38,9 @@ # import os, sys +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong, intmask +from pypy.rlib import rposix from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC O_BINARY = getattr(os, "O_BINARY", 0) @@ -71,6 +73,7 @@ return s.join(string.split(c)) + at specialize.argtype(0) def open_file_as_stream(path, mode="r", buffering=-1): os_flags, universal, reading, writing, basemode, binary = decode_mode(mode) stream = open_path_helper(path, os_flags, basemode == "a") @@ -89,9 +92,10 @@ return construct_stream_tower(stream, buffering, universal, reading, writing, binary) + at specialize.argtype(0) def open_path_helper(path, os_flags, append): # XXX for now always return DiskFile - fd = os.open(path, os_flags, 0666) + fd = rposix.open(path, os_flags, 0666) if append: try: os.lseek(fd, 0, 2) Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_rarithmetic.py Tue Jul 20 20:55:30 2010 @@ -271,26 +271,29 @@ assert ovfcheck_float_to_int(13.0) == 13 assert ovfcheck_float_to_int(-1.0) == -1 assert ovfcheck_float_to_int(-13.0) == -13 - # strange things happening for float to int on 64 bit - maxint32 = 2 ** 31 - 1 - assert ovfcheck_float_to_int(float(maxint32-1)) == maxint32-1 - #assert ovfcheck_float_to_int(float(maxint32)) == maxint32 - assert ovfcheck_float_to_int(float(-maxint32)) == -maxint32 - #assert ovfcheck_float_to_int(float(-maxint32-1)) == -maxint32-1 - - try: - ovfcheck_float_to_int(float(-sys.maxint-1)-1) - except OverflowError: - pass - else: - assert False - - try: - ovfcheck_float_to_int(float(sys.maxint)+1) - except OverflowError: - pass - else: - assert False + + # strange things happening for float to int on 64 bit: + # int(float(i)) != i because of rounding issues + x = sys.maxint + while int(float(x)) > sys.maxint: + x -= 1 + assert ovfcheck_float_to_int(float(x)) == int(float(x)) + + x = sys.maxint + 1 + while int(float(x)) <= sys.maxint: + x += 1 + py.test.raises(OverflowError, ovfcheck_float_to_int, x) + + x = -sys.maxint-1 + while int(float(x)) < -sys.maxint-1: + x += 1 + assert ovfcheck_float_to_int(float(x)) == int(float(x)) + + x = -sys.maxint-1 + while int(float(x)) >= -sys.maxint-1: + x -= 1 + py.test.raises(OverflowError, ovfcheck_float_to_int, x) + def test_abs(): assert type(abs(r_longlong(1))) is r_longlong Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/extfunc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/extfunc.py Tue Jul 20 20:55:30 2010 @@ -52,7 +52,10 @@ return super(ExtRegistryEntry, self).__getattr__(attr) raise exc, exc_inst, tb -def registering(func): +def registering(func, condition=True): + if not condition: + return lambda method: None + def decorator(method): method._registering_func = func return method @@ -63,11 +66,9 @@ func = getattr(ns, name) except AttributeError: condition = False + func = None - if condition: - return registering(func) - else: - return lambda method: None + return registering(func, condition=condition) class LazyRegisteringMeta(type): def __new__(self, _name, _type, _vars): @@ -167,8 +168,6 @@ return signature_args def compute_result_annotation(self, *args_s): - if hasattr(self, 'ann_hook'): - self.ann_hook() self.normalize_args(*args_s) # check arguments return self.signature_result @@ -235,7 +234,6 @@ def register_external(function, args, result=None, export_name=None, llimpl=None, ooimpl=None, llfakeimpl=None, oofakeimpl=None, - annotation_hook=None, sandboxsafe=False): """ function: the RPython function that will be rendered as an external function (e.g.: math.floor) @@ -244,7 +242,6 @@ export_name: the name of the function as it will be seen by the backends llimpl, ooimpl: optional; if provided, these RPython functions are called instead of the target function llfakeimpl, oofakeimpl: optional; if provided, they are called by the llinterpreter - annotationhook: optional; a callable that is called during annotation, useful for genc hacks sandboxsafe: use True if the function performs no I/O (safe for --sandbox) """ @@ -271,8 +268,6 @@ lltypefakeimpl = staticmethod(llfakeimpl) if oofakeimpl: ootypefakeimpl = staticmethod(oofakeimpl) - if annotation_hook: - ann_hook = staticmethod(annotation_hook) if export_name: FunEntry.__name__ = export_name Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rffi.py Tue Jul 20 20:55:30 2010 @@ -176,6 +176,15 @@ # XXX leaks if a str2charp() fails with MemoryError # and was not the first in this function freeme = arg + elif TARGET == CWCHARP: + if arg is None: + arg = lltype.nullptr(CWCHARP.TO) # None => (wchar_t*)NULL + freeme = arg + elif isinstance(arg, unicode): + arg = unicode2wcharp(arg) + # XXX leaks if a unicode2wcharp() fails with MemoryError + # and was not the first in this function + freeme = arg elif _isfunctype(TARGET) and not _isllptr(arg): # XXX pass additional arguments if invoke_around_handlers: @@ -786,6 +795,8 @@ return r_wchar_t.BITS/8 if tp is lltype.Float: return 8 + if tp is lltype.SingleFloat: + return 4 assert isinstance(tp, lltype.Number) if tp is lltype.Signed: return ULONG._type.BITS/8 Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/test/test_rffi.py Tue Jul 20 20:55:30 2010 @@ -707,6 +707,7 @@ lltype.UniChar: ctypes.c_wchar, lltype.Char: ctypes.c_ubyte, DOUBLE: ctypes.c_double, + FLOAT: ctypes.c_float, SIGNEDCHAR: ctypes.c_byte, UCHAR: ctypes.c_ubyte, SHORT: ctypes.c_short, Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/memory/lltypelayout.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/memory/lltypelayout.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/memory/lltypelayout.py Tue Jul 20 20:55:30 2010 @@ -111,6 +111,8 @@ elif isinstance(offset, llmemory.ItemOffset): return sizeof(offset.TYPE) * offset.repeat elif isinstance(offset, llmemory.ArrayItemsOffset): + if offset.TYPE._hints.get('nolength', None): + return 0 return get_fixed_size(lltype.Signed) elif isinstance(offset, llmemory.GCHeaderOffset): return sizeof(offset.gcheaderbuilder.HDR) Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os.py Tue Jul 20 20:55:30 2010 @@ -6,12 +6,15 @@ # might be found in doc/rffi.txt import os, sys, errno +import py from pypy.rpython.module.support import ll_strcpy, OOSupport -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rlib.rarithmetic import r_longlong -from pypy.rpython.extfunc import BaseLazyRegistering +from pypy.rpython.extfunc import ( + BaseLazyRegistering, lazy_register, register_external) from pypy.rpython.extfunc import registering, registering_if, extdef -from pypy.annotation.model import SomeInteger, SomeString, SomeTuple, SomeFloat +from pypy.annotation.model import ( + SomeInteger, SomeString, SomeTuple, SomeFloat, SomeUnicodeString) from pypy.annotation.model import s_ImpossibleValue, s_None, s_Bool from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype @@ -26,7 +29,99 @@ from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.annlowlevel import llstr from pypy.rlib import rgc -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import keepalive_until_here, specialize + +def monkeypatch_rposix(posixfunc, unicodefunc, signature): + func_name = posixfunc.__name__ + + if hasattr(signature, '_default_signature_'): + signature = signature._default_signature_ + arglist = ['arg%d' % (i,) for i in range(len(signature))] + transformed_arglist = arglist[:] + for i, arg in enumerate(signature): + if arg is unicode: + transformed_arglist[i] = transformed_arglist[i] + '.as_unicode()' + + args = ', '.join(arglist) + transformed_args = ', '.join(transformed_arglist) + main_arg = 'arg%d' % (signature.index(unicode),) + + source = py.code.Source(""" + def %(func_name)s(%(args)s): + if isinstance(%(main_arg)s, str): + return posixfunc(%(args)s) + else: + return unicodefunc(%(transformed_args)s) + """ % locals()) + miniglobals = {'posixfunc' : posixfunc, + 'unicodefunc': unicodefunc, + '__name__': __name__, # for module name propagation + } + exec source.compile() in miniglobals + new_func = miniglobals[func_name] + specialized_args = [i for i in range(len(signature)) + if signature[i] in (unicode, None)] + new_func = specialize.argtype(*specialized_args)(new_func) + + # Monkeypatch the function in pypy.rlib.rposix + setattr(rposix, func_name, new_func) + +class StringTraits: + str = str + CHAR = rffi.CHAR + CCHARP = rffi.CCHARP + charp2str = staticmethod(rffi.charp2str) + str2charp = staticmethod(rffi.str2charp) + free_charp = staticmethod(rffi.free_charp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_' + name + +class UnicodeTraits: + str = unicode + CHAR = rffi.WCHAR_T + CCHARP = rffi.CWCHARP + charp2str = staticmethod(rffi.wcharp2unicode) + str2charp = staticmethod(rffi.unicode2wcharp) + free_charp = staticmethod(rffi.free_wcharp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + 'w' + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_w' + name + +def registering_str_unicode(posixfunc, condition=True): + if not condition: + return registering(None, condition=False) + + func_name = posixfunc.__name__ + + def register_posixfunc(self, method): + val = method(self, StringTraits()) + register_external(posixfunc, *val.def_args, **val.def_kwds) + + if sys.platform == 'win32': + val = method(self, UnicodeTraits()) + @func_renamer(func_name + "_unicode") + def unicodefunc(*args): + return posixfunc(*args) + register_external(unicodefunc, *val.def_args, **val.def_kwds) + signature = val.def_args[0] + monkeypatch_rposix(posixfunc, unicodefunc, signature) + + def decorator(method): + decorated = lambda self: register_posixfunc(self, method) + decorated._registering_func = posixfunc + return decorated + return decorator posix = __import__(os.name) @@ -282,8 +377,8 @@ return extdef([int, int], s_None, llimpl=dup2_llimpl, export_name="ll_os.ll_os_dup2") - @registering(os.utime) - def register_os_utime(self): + @registering_str_unicode(os.utime) + def register_os_utime(self, traits): UTIMBUFP = lltype.Ptr(self.UTIMBUF) os_utime = self.llexternal('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT) @@ -336,6 +431,9 @@ # tp is known to be None, and one version where it is known # to be a tuple of 2 floats. if not _WIN32: + assert traits.str is str + + @specialize.argtype(1) def os_utime_llimpl(path, tp): if tp is None: error = os_utime(path, lltype.nullptr(UTIMBUFP.TO)) @@ -346,85 +444,13 @@ if error == -1: raise OSError(rposix.get_errno(), "os_utime failed") else: - from pypy.rlib import rwin32 - from pypy.rpython.module.ll_os_stat import time_t_to_FILE_TIME - - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'], - ) - - FILE_WRITE_ATTRIBUTES = platform.ConstantInteger( - 'FILE_WRITE_ATTRIBUTES') - OPEN_EXISTING = platform.ConstantInteger( - 'OPEN_EXISTING') - FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger( - 'FILE_FLAG_BACKUP_SEMANTICS') - globals().update(platform.configure(CConfig)) - - CreateFile = rffi.llexternal( - 'CreateFileA', - [rwin32.LPCSTR, rwin32.DWORD, rwin32.DWORD, - rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD, - rwin32.HANDLE], - rwin32.HANDLE, - calling_conv='win') - - GetSystemTime = rffi.llexternal( - 'GetSystemTime', - [lltype.Ptr(rwin32.SYSTEMTIME)], - lltype.Void, - calling_conv='win') - - SystemTimeToFileTime = rffi.llexternal( - 'SystemTimeToFileTime', - [lltype.Ptr(rwin32.SYSTEMTIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv='win') - - SetFileTime = rffi.llexternal( - 'SetFileTime', - [rwin32.HANDLE, - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv = 'win') + from pypy.rpython.module.ll_win32file import make_utime_impl + os_utime_llimpl = make_utime_impl(traits) - def os_utime_llimpl(path, tp): - hFile = CreateFile(path, - FILE_WRITE_ATTRIBUTES, 0, - None, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, 0) - if hFile == rwin32.INVALID_HANDLE_VALUE: - raise rwin32.lastWindowsError() - ctime = lltype.nullptr(rwin32.FILETIME) - atime = lltype.malloc(rwin32.FILETIME, flavor='raw') - mtime = lltype.malloc(rwin32.FILETIME, flavor='raw') - try: - if tp is None: - now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw') - try: - GetSystemTime(now) - if (not SystemTimeToFileTime(now, atime) or - not SystemTimeToFileTime(now, mtime)): - raise rwin32.lastWindowsError() - finally: - lltype.free(now, flavor='raw') - else: - actime, modtime = tp - time_t_to_FILE_TIME(actime, atime) - time_t_to_FILE_TIME(modtime, mtime) - if not SetFileTime(hFile, ctime, atime, mtime): - raise rwin32.lastWindowsError() - finally: - rwin32.CloseHandle(hFile) - lltype.free(atime, flavor='raw') - lltype.free(mtime, flavor='raw') - os_utime_llimpl._annspecialcase_ = 'specialize:argtype(1)' - - s_string = SomeString() + if traits.str is str: + s_string = SomeString() + else: + s_string = SomeUnicodeString() s_tuple_of_2_floats = SomeTuple([SomeFloat(), SomeFloat()]) def os_utime_normalize_args(s_path, s_times): @@ -445,12 +471,12 @@ else: raise Exception("os.utime() arg 2 must be None or a tuple of " "2 floats, got %s" % (s_times,)) + os_utime_normalize_args._default_signature_ = [traits.str, None] return extdef(os_utime_normalize_args, s_None, "ll_os.ll_os_utime", llimpl=os_utime_llimpl) - @registering(os.times) def register_os_times(self): if sys.platform.startswith('win'): @@ -687,22 +713,21 @@ def register_os_setsid(self): return self.extdef_for_os_function_returning_int('setsid') - @registering(os.open) - def register_os_open(self): - os_open = self.llexternal(underscore_on_windows+'open', - [rffi.CCHARP, rffi.INT, rffi.MODE_T], + @registering_str_unicode(os.open) + def register_os_open(self, traits): + os_open = self.llexternal(traits.posix_function_name('open'), + [traits.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT) - def os_open_llimpl(path, flags, mode): result = rffi.cast(rffi.LONG, os_open(path, flags, mode)) if result == -1: raise OSError(rposix.get_errno(), "os_open failed") return result - def os_open_oofakeimpl(o_path, flags, mode): - return os.open(o_path._str, flags, mode) + def os_open_oofakeimpl(path, flags, mode): + return os.open(OOSupport.from_rstr(path), flags, mode) - return extdef([str, int, int], int, "ll_os.ll_os_open", + return extdef([traits.str, int, int], int, traits.ll_os_name('open'), llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) # ------------------------------- os.read ------------------------------- @@ -862,10 +887,10 @@ llimpl=fdatasync_llimpl, export_name="ll_os.ll_os_fdatasync") - @registering(os.access) - def register_os_access(self): - os_access = self.llexternal(underscore_on_windows + 'access', - [rffi.CCHARP, rffi.INT], + @registering_str_unicode(os.access) + def register_os_access(self, traits): + os_access = self.llexternal(traits.posix_function_name('access'), + [traits.CCHARP, rffi.INT], rffi.INT) if sys.platform.startswith('win'): @@ -882,44 +907,22 @@ def os_access_oofakeimpl(path, mode): return os.access(OOSupport.from_rstr(path), mode) - return extdef([str, int], s_Bool, llimpl=access_llimpl, - export_name="ll_os.ll_os_access", + return extdef([traits.str, int], s_Bool, llimpl=access_llimpl, + export_name=traits.ll_os_name("access"), oofakeimpl=os_access_oofakeimpl) - @registering_if(posix, '_getfullpathname') - def register_posix__getfullpathname(self): - from pypy.rlib import rwin32 + @registering_str_unicode(getattr(posix, '_getfullpathname', None), + condition=sys.platform=='win32') + def register_posix__getfullpathname(self, traits): # this nt function is not exposed via os, but needed # to get a correct implementation of os.abspath - # XXX why do we ignore WINAPI conventions everywhere? - LPSTRP = rffi.CArrayPtr(rwin32.LPSTR) - # XXX unicode? - GetFullPathName = self.llexternal( - 'GetFullPathNameA', - [rwin32.LPCSTR, - rwin32.DWORD, - rwin32.LPSTR, - rffi.CArrayPtr(rwin32.LPSTR)], - rwin32.DWORD) - - def _getfullpathname_llimpl(lpFileName): - nBufferLength = rwin32.MAX_PATH + 1 - lpBuffer = lltype.malloc(rwin32.LPSTR.TO, nBufferLength, flavor='raw') - try: - res = GetFullPathName( - lpFileName, rffi.cast(rwin32.DWORD, nBufferLength), - lpBuffer, lltype.nullptr(LPSTRP.TO)) - if res == 0: - raise rwin32.lastWindowsError("_getfullpathname failed") - result = rffi.charp2str(lpBuffer) - return result - finally: - lltype.free(lpBuffer, flavor='raw') + from pypy.rpython.module.ll_win32file import make_getfullpathname_impl + getfullpathname_llimpl = make_getfullpathname_impl(traits) - return extdef([str], # a single argument which is a str - str, # returns a string - "ll_os.posix__getfullpathname", - llimpl=_getfullpathname_llimpl) + return extdef([traits.str], # a single argument which is a str + traits.str, # returns a string + traits.ll_os_name('_getfullpathname'), + llimpl=getfullpathname_llimpl) @registering(os.getcwd) def register_os_getcwd(self): @@ -953,71 +956,42 @@ "ll_os.ll_os_getcwd", llimpl=os_getcwd_llimpl, oofakeimpl=os_getcwd_oofakeimpl) - @registering(os.listdir) - def register_os_listdir(self): - # we need a different approach on Windows and on Posix - if sys.platform.startswith('win'): - from pypy.rlib import rwin32 - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'] - ) - WIN32_FIND_DATA = platform.Struct('struct _WIN32_FIND_DATAA', - [('cFileName', lltype.FixedSizeArray(rffi.CHAR, 1))]) - ERROR_FILE_NOT_FOUND = platform.ConstantInteger( - 'ERROR_FILE_NOT_FOUND') - ERROR_NO_MORE_FILES = platform.ConstantInteger( - 'ERROR_NO_MORE_FILES') + @registering(os.getcwdu, condition=sys.platform=='win32') + def register_os_getcwdu(self): + os_wgetcwd = self.llexternal(underscore_on_windows + 'wgetcwd', + [rffi.CWCHARP, rffi.SIZE_T], + rffi.CWCHARP) - config = platform.configure(CConfig) - WIN32_FIND_DATA = config['WIN32_FIND_DATA'] - ERROR_FILE_NOT_FOUND = config['ERROR_FILE_NOT_FOUND'] - ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] - LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) - - FindFirstFile = self.llexternal('FindFirstFile', - [rwin32.LPCSTR, LPWIN32_FIND_DATA], - rwin32.HANDLE) - FindNextFile = self.llexternal('FindNextFile', - [rwin32.HANDLE, LPWIN32_FIND_DATA], - rwin32.BOOL) - FindClose = self.llexternal('FindClose', - [rwin32.HANDLE], - rwin32.BOOL) + def os_getcwd_llimpl(): + bufsize = 256 + while True: + buf = lltype.malloc(rffi.CWCHARP.TO, bufsize, flavor='raw') + res = os_wgetcwd(buf, rffi.cast(rffi.SIZE_T, bufsize)) + if res: + break # ok + error = rposix.get_errno() + lltype.free(buf, flavor='raw') + if error != errno.ERANGE: + raise OSError(error, "getcwd failed") + # else try again with a larger buffer, up to some sane limit + bufsize *= 4 + if bufsize > 1024*1024: # xxx hard-coded upper limit + raise OSError(error, "getcwd result too large") + result = rffi.wcharp2unicode(res) + lltype.free(buf, flavor='raw') + return result - def os_listdir_llimpl(path): - if path and path[-1] not in ('/', '\\', ':'): - path += '/' - path += '*.*' - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - try: - result = [] - hFindFile = FindFirstFile(path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - error = rwin32.GetLastError() - if error == ERROR_FILE_NOT_FOUND: - return result - else: - raise WindowsError(error, "FindFirstFile failed") - while True: - name = rffi.charp2str(rffi.cast(rffi.CCHARP, - filedata.c_cFileName)) - if name != "." and name != "..": # skip these - result.append(name) - if not FindNextFile(hFindFile, filedata): - break - # FindNextFile sets error to ERROR_NO_MORE_FILES if - # it got to the end of the directory - error = rwin32.GetLastError() - FindClose(hFindFile) - if error == ERROR_NO_MORE_FILES: - return result - else: - raise WindowsError(error, "FindNextFile failed") - finally: - lltype.free(filedata, flavor='raw') + return extdef([], unicode, + "ll_os.ll_os_wgetcwd", llimpl=os_getcwd_llimpl) + @registering_str_unicode(os.listdir) + def register_os_listdir(self, traits): + # we need a different approach on Windows and on Posix + if sys.platform.startswith('win'): + from pypy.rpython.module.ll_win32file import make_listdir_impl + os_listdir_llimpl = make_listdir_impl(traits) else: + assert traits.str is str compilation_info = ExternalCompilationInfo( includes = ['sys/types.h', 'dirent.h'] ) @@ -1057,9 +1031,9 @@ raise OSError(error, "os_readdir failed") return result - return extdef([str], # a single argument which is a str - [str], # returns a list of strings - "ll_os.ll_os_listdir", + return extdef([traits.str], # a single argument which is a str + [traits.str], # returns a list of strings + traits.ll_os_name('listdir'), llimpl=os_listdir_llimpl) @registering(os.pipe) @@ -1234,38 +1208,40 @@ return extdef([str], int, llimpl=system_llimpl, export_name="ll_os.ll_os_system") - @registering(os.unlink) - def register_os_unlink(self): - os_unlink = self.llexternal(underscore_on_windows+'unlink', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.unlink) + def register_os_unlink(self, traits): + os_unlink = self.llexternal(traits.posix_function_name('unlink'), + [traits.CCHARP], rffi.INT) def unlink_llimpl(pathname): res = rffi.cast(lltype.Signed, os_unlink(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_unlink failed") - return extdef([str], s_None, llimpl=unlink_llimpl, - export_name="ll_os.ll_os_unlink") + return extdef([traits.str], s_None, llimpl=unlink_llimpl, + export_name=traits.ll_os_name('unlink')) - @registering(os.chdir) - def register_os_chdir(self): - os_chdir = self.llexternal(underscore_on_windows+'chdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.chdir) + def register_os_chdir(self, traits): + os_chdir = self.llexternal(traits.posix_function_name('chdir'), + [traits.CCHARP], rffi.INT) def chdir_llimpl(path): res = rffi.cast(lltype.Signed, os_chdir(path)) if res < 0: raise OSError(rposix.get_errno(), "os_chdir failed") - return extdef([str], s_None, llimpl=chdir_llimpl, - export_name="ll_os.ll_os_chdir") + return extdef([traits.str], s_None, llimpl=chdir_llimpl, + export_name=traits.ll_os_name('chdir')) - @registering(os.mkdir) - def register_os_mkdir(self): + @registering_str_unicode(os.mkdir) + def register_os_mkdir(self, traits): if os.name == 'nt': ARG2 = [] # no 'mode' argument on Windows - just ignored else: ARG2 = [rffi.MODE_T] - os_mkdir = self.llexternal(underscore_on_windows+'mkdir', - [rffi.CCHARP]+ARG2, rffi.INT) + os_mkdir = self.llexternal(traits.posix_function_name('mkdir'), + [traits.CCHARP] + ARG2, rffi.INT) IGNORE_MODE = len(ARG2) == 0 def mkdir_llimpl(pathname, mode): @@ -1277,46 +1253,47 @@ if res < 0: raise OSError(rposix.get_errno(), "os_mkdir failed") - return extdef([str, int], s_None, llimpl=mkdir_llimpl, - export_name="ll_os.ll_os_mkdir") + return extdef([traits.str, int], s_None, llimpl=mkdir_llimpl, + export_name=traits.ll_os_name('mkdir')) - @registering(os.rmdir) - def register_os_rmdir(self): - os_rmdir = self.llexternal(underscore_on_windows+'rmdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.rmdir) + def register_os_rmdir(self, traits): + os_rmdir = self.llexternal(traits.posix_function_name('rmdir'), + [traits.CCHARP], rffi.INT) def rmdir_llimpl(pathname): res = rffi.cast(lltype.Signed, os_rmdir(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_rmdir failed") - return extdef([str], s_None, llimpl=rmdir_llimpl, - export_name="ll_os.ll_os_rmdir") + return extdef([traits.str], s_None, llimpl=rmdir_llimpl, + export_name=traits.ll_os_name('rmdir')) - @registering(os.chmod) - def register_os_chmod(self): - os_chmod = self.llexternal(underscore_on_windows+'chmod', [rffi.CCHARP, rffi.MODE_T], - rffi.INT) + @registering_str_unicode(os.chmod) + def register_os_chmod(self, traits): + os_chmod = self.llexternal(traits.posix_function_name('chmod'), + [traits.CCHARP, rffi.MODE_T], rffi.INT) def chmod_llimpl(path, mode): res = rffi.cast(lltype.Signed, os_chmod(path, rffi.cast(rffi.MODE_T, mode))) if res < 0: raise OSError(rposix.get_errno(), "os_chmod failed") - return extdef([str, int], s_None, llimpl=chmod_llimpl, - export_name="ll_os.ll_os_chmod") + return extdef([traits.str, int], s_None, llimpl=chmod_llimpl, + export_name=traits.ll_os_name('chmod')) - @registering(os.rename) - def register_os_rename(self): - os_rename = self.llexternal('rename', [rffi.CCHARP, rffi.CCHARP], - rffi.INT) + @registering_str_unicode(os.rename) + def register_os_rename(self, traits): + os_rename = self.llexternal(traits.posix_function_name('rename'), + [traits.CCHARP, traits.CCHARP], rffi.INT) def rename_llimpl(oldpath, newpath): res = rffi.cast(lltype.Signed, os_rename(oldpath, newpath)) if res < 0: raise OSError(rposix.get_errno(), "os_rename failed") - return extdef([str, str], s_None, llimpl=rename_llimpl, - export_name="ll_os.ll_os_rename") + return extdef([traits.str, traits.str], s_None, llimpl=rename_llimpl, + export_name=traits.ll_os_name('rename')) @registering(os.umask) def register_os_umask(self): @@ -1425,17 +1402,17 @@ @registering(os.fstat) def register_os_fstat(self): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('fstat') + return ll_os_stat.register_stat_variant('fstat', StringTraits()) - @registering(os.stat) - def register_os_stat(self): + @registering_str_unicode(os.stat) + def register_os_stat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('stat') + return ll_os_stat.register_stat_variant('stat', traits) - @registering(os.lstat) - def register_os_lstat(self): + @registering_str_unicode(os.lstat) + def register_os_lstat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('lstat') + return ll_os_stat.register_stat_variant('lstat', traits) # ------------------------------- os.W* --------------------------------- Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os_stat.py Tue Jul 20 20:55:30 2010 @@ -5,13 +5,14 @@ import os, sys from pypy.annotation import model as annmodel from pypy.tool.pairtype import pairtype -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rpython import extregistry -from pypy.rpython.extfunc import register_external +from pypy.rpython.extfunc import register_external, extdef from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.annlowlevel import hlstr @@ -211,13 +212,27 @@ return make_stat_result(result) -def register_stat_variant(name): - if sys.platform.startswith('win'): - _functions = {'stat': '_stati64', - 'fstat': '_fstati64', - 'lstat': '_stati64'} # no lstat on Windows - c_func_name = _functions[name] - elif sys.platform.startswith('linux'): +def register_stat_variant(name, traits): + if name != 'fstat': + arg_is_path = True + s_arg = traits.str + ARG1 = traits.CCHARP + else: + arg_is_path = False + s_arg = int + ARG1 = rffi.INT + + if sys.platform == 'win32': + # See Win32 implementation below + posix_stat_llimpl = make_win32_stat_impl(name, traits) + + return extdef( + [s_arg], s_StatResult, traits.ll_os_name(name), + llimpl=posix_stat_llimpl) + + assert traits.str is str + + if sys.platform.startswith('linux'): # because we always use _FILE_OFFSET_BITS 64 - this helps things work that are not a c compiler _functions = {'stat': 'stat64', 'fstat': 'fstat64', @@ -226,22 +241,26 @@ else: c_func_name = name - arg_is_path = (name != 'fstat') + posix_mystat = rffi.llexternal(c_func_name, + [ARG1, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info) + @func_renamer('os_%s_llimpl' % (name,)) def posix_stat_llimpl(arg): stresult = lltype.malloc(STAT_STRUCT.TO, flavor='raw') try: if arg_is_path: - arg = rffi.str2charp(arg) + arg = traits.str2charp(arg) error = rffi.cast(rffi.LONG, posix_mystat(arg, stresult)) if arg_is_path: - rffi.free_charp(arg) + traits.free_charp(arg) if error != 0: raise OSError(rposix.get_errno(), "os_?stat failed") return build_stat_result(stresult) finally: lltype.free(stresult, flavor='raw') + @func_renamer('os_%s_fake' % (name,)) def posix_fakeimpl(arg): if s_arg == str: arg = hlstr(arg) @@ -259,40 +278,17 @@ setattr(ll_tup, 'item%d' % i, val) return ll_tup - if arg_is_path: - s_arg = str - ARG1 = rffi.CCHARP - else: - s_arg = int - ARG1 = rffi.INT + return extdef( + [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), + llimpl=posix_stat_llimpl, llfakeimpl=posix_fakeimpl) - if sys.platform != 'win32': - posix_mystat = rffi.llexternal(c_func_name, - [ARG1, STAT_STRUCT], rffi.INT, - compilation_info=compilation_info) - - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(posix_stat_llimpl, - 'os_%s_llimpl' % (name,)), - llfakeimpl=func_with_new_name(posix_fakeimpl, - 'os_%s_fake' % (name,)), - ) - else: - # See Win32 implementation below - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(globals()['win32_%s_llimpl' % (name,)], - 'os_%s_llimpl' % (name,)), - ) +def make_win32_stat_impl(name, traits): + from pypy.rlib import rwin32 + from pypy.rpython.module.ll_win32file import make_win32_traits + win32traits = make_win32_traits(traits) -# ____________________________________________________________ -if sys.platform == 'win32': # The CRT of Windows has a number of flaws wrt. its stat() implementation: - # - for when we implement subsecond resolution in RPython, time stamps - # would be restricted to second resolution + # - time stamps are restricted to second resolution # - file modification times suffer from forth-and-back conversions between # UTC and local time # Therefore, we implement our own stat, based on the Win32 API directly. @@ -302,122 +298,18 @@ assert len(STAT_FIELDS) == 10 # no extra fields on Windows - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h', 'winbase.h', 'sys/stat.h'], - ) - - GetFileExInfoStandard = platform.ConstantInteger( - 'GetFileExInfoStandard') - FILE_ATTRIBUTE_DIRECTORY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_DIRECTORY') - FILE_ATTRIBUTE_READONLY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_READONLY') - ERROR_SHARING_VIOLATION = platform.ConstantInteger( - 'ERROR_SHARING_VIOLATION') - _S_IFDIR = platform.ConstantInteger('_S_IFDIR') - _S_IFREG = platform.ConstantInteger('_S_IFREG') - _S_IFCHR = platform.ConstantInteger('_S_IFCHR') - _S_IFIFO = platform.ConstantInteger('_S_IFIFO') - FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN') - FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') - FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') - - WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( - 'WIN32_FILE_ATTRIBUTE_DATA', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - BY_HANDLE_FILE_INFORMATION = platform.Struct( - 'BY_HANDLE_FILE_INFORMATION', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('nNumberOfLinks', rwin32.DWORD), - ('nFileIndexHigh', rwin32.DWORD), - ('nFileIndexLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - WIN32_FIND_DATA = platform.Struct( - 'WIN32_FIND_DATAA', - # Only interesting fields - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - globals().update(platform.configure(CConfig)) - GET_FILEEX_INFO_LEVELS = rffi.ULONG # an enumeration - - GetFileAttributesEx = rffi.llexternal( - 'GetFileAttributesExA', - [rffi.CCHARP, GET_FILEEX_INFO_LEVELS, - lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)], - rwin32.BOOL, - calling_conv='win') - - GetFileInformationByHandle = rffi.llexternal( - 'GetFileInformationByHandle', - [rwin32.HANDLE, lltype.Ptr(BY_HANDLE_FILE_INFORMATION)], - rwin32.BOOL, - calling_conv='win') - - GetFileType = rffi.llexternal( - 'GetFileType', - [rwin32.HANDLE], - rwin32.DWORD, - calling_conv='win') - - FindFirstFile = rffi.llexternal( - 'FindFirstFileA', - [rffi.CCHARP, lltype.Ptr(WIN32_FIND_DATA)], - rwin32.HANDLE, - calling_conv='win') - - FindClose = rffi.llexternal( - 'FindClose', - [rwin32.HANDLE], - rwin32.BOOL, - calling_conv='win') - def attributes_to_mode(attributes): m = 0 - if attributes & FILE_ATTRIBUTE_DIRECTORY: - m |= _S_IFDIR | 0111 # IFEXEC for user,group,other + if attributes & win32traits.FILE_ATTRIBUTE_DIRECTORY: + m |= win32traits._S_IFDIR | 0111 # IFEXEC for user,group,other else: - m |= _S_IFREG - if attributes & FILE_ATTRIBUTE_READONLY: + m |= win32traits._S_IFREG + if attributes & win32traits.FILE_ATTRIBUTE_READONLY: m |= 0444 else: m |= 0666 return m - def make_longlong(high, low): - return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) - - # Seconds between 1.1.1601 and 1.1.1970 - secs_between_epochs = lltype.r_longlong(11644473600) - - def FILE_TIME_to_time_t_nsec(filetime): - ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) - # FILETIME is in units of 100 nsec - nsec = (ft % 10000000) * 100 - time = (ft / 10000000) - secs_between_epochs - return time, nsec - - def time_t_to_FILE_TIME(time, filetime): - ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) - filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) - filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) - def attribute_data_to_stat(info): st_mode = attributes_to_mode(info.c_dwFileAttributes) st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) @@ -456,65 +348,94 @@ return make_stat_result(result) def attributes_from_dir(l_path, data): - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - hFindFile = FindFirstFile(l_path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - return 0 - FindClose(hFindFile) - data.c_dwFileAttributes = filedata.c_dwFileAttributes - rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) - rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) - rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) - data.c_nFileSizeHigh = filedata.c_nFileSizeHigh - data.c_nFileSizeLow = filedata.c_nFileSizeLow - return 1 + filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') + try: + hFindFile = win32traits.FindFirstFile(l_path, filedata) + if hFindFile == rwin32.INVALID_HANDLE_VALUE: + return 0 + win32traits.FindClose(hFindFile) + data.c_dwFileAttributes = filedata.c_dwFileAttributes + rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) + rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) + rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) + data.c_nFileSizeHigh = filedata.c_nFileSizeHigh + data.c_nFileSizeLow = filedata.c_nFileSizeLow + return 1 + finally: + lltype.free(filedata, flavor='raw') def win32_stat_llimpl(path): - data = lltype.malloc(WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') + data = lltype.malloc(win32traits.WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') try: - l_path = rffi.str2charp(path) - res = GetFileAttributesEx(l_path, GetFileExInfoStandard, data) + l_path = traits.str2charp(path) + res = win32traits.GetFileAttributesEx(l_path, win32traits.GetFileExInfoStandard, data) errcode = rwin32.GetLastError() if res == 0: - if errcode == ERROR_SHARING_VIOLATION: + if errcode == win32traits.ERROR_SHARING_VIOLATION: res = attributes_from_dir(l_path, data) errcode = rwin32.GetLastError() - rffi.free_charp(l_path) + traits.free_charp(l_path) if res == 0: raise WindowsError(errcode, "os_stat failed") return attribute_data_to_stat(data) finally: lltype.free(data, flavor='raw') - win32_lstat_llimpl = win32_stat_llimpl def win32_fstat_llimpl(fd): handle = rwin32._get_osfhandle(fd) - filetype = GetFileType(handle) - if filetype == FILE_TYPE_CHAR: + filetype = win32traits.GetFileType(handle) + if filetype == win32traits.FILE_TYPE_CHAR: # console or LPT device - return make_stat_result((_S_IFCHR, + return make_stat_result((win32traits._S_IFCHR, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_PIPE: + elif filetype == win32traits.FILE_TYPE_PIPE: # socket or named pipe - return make_stat_result((_S_IFIFO, + return make_stat_result((win32traits._S_IFIFO, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_UNKNOWN: + elif filetype == win32traits.FILE_TYPE_UNKNOWN: error = rwin32.GetLastError() if error != 0: raise WindowsError(error, "os_fstat failed") # else: unknown but valid file # normal disk file (FILE_TYPE_DISK) - info = lltype.malloc(BY_HANDLE_FILE_INFORMATION, flavor='raw', - zero=True) + info = lltype.malloc(win32traits.BY_HANDLE_FILE_INFORMATION, + flavor='raw', zero=True) try: - res = GetFileInformationByHandle(handle, info) + res = win32traits.GetFileInformationByHandle(handle, info) if res == 0: raise WindowsError(rwin32.GetLastError(), "os_fstat failed") return by_handle_info_to_stat(info) finally: lltype.free(info, flavor='raw') + if name == 'fstat': + return win32_fstat_llimpl + else: + return win32_stat_llimpl + + +#__________________________________________________ +# Helper functions for win32 + +def make_longlong(high, low): + return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) + +# Seconds between 1.1.1601 and 1.1.1970 +secs_between_epochs = lltype.r_longlong(11644473600) + +def FILE_TIME_to_time_t_nsec(filetime): + ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) + # FILETIME is in units of 100 nsec + nsec = (ft % 10000000) * 100 + time = (ft / 10000000) - secs_between_epochs + return time, nsec + +def time_t_to_FILE_TIME(time, filetime): + ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) + filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) + filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) + Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_termios.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_termios.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_termios.py Tue Jul 20 20:55:30 2010 @@ -85,7 +85,7 @@ c_struct.c_c_lflag, ispeed, ospeed, cc = attributes try: for i in range(NCCS): - c_struct.c_c_cc[i] = rffi.r_uchar(ord(cc[i])) + c_struct.c_c_cc[i] = rffi.r_uchar(ord(cc[i][0])) error = c_cfsetispeed(c_struct, ispeed) if error == -1: raise termios.error(error, 'tcsetattr failed') Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/module/test/test_ll_os_stat.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/module/test/test_ll_os_stat.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/module/test/test_ll_os_stat.py Tue Jul 20 20:55:30 2010 @@ -1,4 +1,4 @@ -from pypy.rpython.module import ll_os_stat +from pypy.rpython.module import ll_os_stat, ll_os import sys, os import py @@ -8,14 +8,18 @@ py.test.skip("win32 specific tests") def test_stat(self): - stat = ll_os_stat.win32_stat_llimpl + stat = ll_os_stat.make_win32_stat_impl('stat', ll_os.StringTraits()) + wstat = ll_os_stat.make_win32_stat_impl('stat', ll_os.UnicodeTraits()) def check(f): - assert stat(f).st_mtime == os.stat(f).st_mtime + expected = os.stat(f).st_mtime + assert stat(f).st_mtime == expected + assert wstat(unicode(f)).st_mtime == expected check('c:/') check('c:/temp') check('c:/pagefile.sys') def test_fstat(self): - stat = ll_os_stat.win32_fstat_llimpl(0) # stdout + fstat = ll_os_stat.make_win32_stat_impl('fstat', ll_os.StringTraits()) + stat = fstat(0) # stdout assert stat.st_mode != 0 Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/rstr.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/rstr.py Tue Jul 20 20:55:30 2010 @@ -80,17 +80,6 @@ # defaults to checking the length return super(AbstractStringRepr, self).rtype_is_true(hop) - def rtype_ord(self, hop): - string_repr = hop.args_r[0].repr - v_str, = hop.inputargs(string_repr) - c_zero = inputconst(Signed, 0) - v_chr = hop.gendirectcall(self.ll.ll_stritem_nonneg, v_str, c_zero) - if string_repr is hop.rtyper.type_system.rstr.string_repr: - return hop.genop('cast_char_to_int', [v_chr], resulttype=Signed) - else: - assert string_repr is hop.rtyper.type_system.rstr.unicode_repr - return hop.genop('cast_unichar_to_int', [v_chr], resulttype=Signed) - def rtype_method_startswith(self, hop): str1_repr, str2_repr = self._str_reprs(hop) v_str, v_value = hop.inputargs(str1_repr, str2_repr) @@ -299,7 +288,11 @@ if not hop.args_s[1].is_constant(): raise TyperError("encoding must be constant") encoding = hop.args_s[1].const - v_self = hop.inputarg(self.repr, 0) + if encoding == "ascii": + expect = self.lowleveltype # can be a UniChar + else: + expect = self.repr # must be a regular unicode string + v_self = hop.inputarg(expect, 0) hop.exception_is_here() if encoding == "ascii": return hop.gendirectcall(self.ll_str, v_self) @@ -426,7 +419,17 @@ sourcevars.append((v_item, r_arg)) return r_str.ll.do_stringformat(hop, sourcevars) - + + +class __extend__(AbstractCharRepr): + def ll_str(self, ch): + return self.ll.ll_chr2str(ch) + +class __extend__(AbstractUniCharRepr): + def ll_str(self, ch): + # xxx suboptimal, maybe + return str(unicode(ch)) + class __extend__(AbstractCharRepr, AbstractUniCharRepr): @@ -444,9 +447,6 @@ get_ll_fasthash_function = get_ll_hash_function - def ll_str(self, ch): - return self.ll.ll_chr2str(ch) - def rtype_len(_, hop): return hop.inputconst(Signed, 1) Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/test/test_extfunc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/test/test_extfunc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/test/test_extfunc.py Tue Jul 20 20:55:30 2010 @@ -6,150 +6,164 @@ from pypy.annotation.policy import AnnotatorPolicy from pypy.rpython.test.test_llinterp import interpret -def b(x): - return eval("x+40") +class TestExtFuncEntry: -class BTestFuncEntry(ExtFuncEntry): - _about_ = b - name = 'b' - signature_args = [annmodel.SomeInteger()] - signature_result = annmodel.SomeInteger() - -def test_annotation_b(): - def f(): - return b(1) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def test_rtyping_b(): - def f(): - return b(2) - - res = interpret(f, []) - assert res == 42 - -def c(y, x): - yyy - -class CTestFuncEntry(ExtFuncEntry): - _about_ = c - name = 'ccc' - signature_args = [annmodel.SomeInteger()] * 2 - signature_result = annmodel.SomeInteger() - - def lltypeimpl(y, x): - return y + x - lltypeimpl = staticmethod(lltypeimpl) - -def test_interp_c(): - def f(): - return c(3, 4) - - res = interpret(f, []) - assert res == 7 - -def d(y): - return eval("y()") - -class DTestFuncEntry(ExtFuncEntry): - _about_ = d - name = 'd' - signature_args = [annmodel.SomeGenericCallable(args=[], result= - annmodel.SomeFloat())] - signature_result = annmodel.SomeFloat() - -def test_callback(): - def callback(): - return 2.5 - - def f(): - return d(callback) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeFloat) - assert a.translator._graphof(callback) - -def dd(): - pass - -register_external(dd, [int], int) - -def test_register_external_signature(): - def f(): - return dd(3) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - - -def function_with_tuple_arg(): - """ - Dummy function which is declared via register_external to take a tuple as - an argument so that register_external's behavior for tuple-taking functions - can be verified. - """ -register_external(function_with_tuple_arg, [(int,)], int) - -def test_register_external_tuple_args(): - """ - Verify the annotation of a registered external function which takes a tuple - argument. - """ - def f(): - return function_with_tuple_arg((1,)) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - - # Not a very good assertion, but at least it means _something_ happened. - assert isinstance(s, annmodel.SomeInteger) - -def function_with_list(): - pass -register_external(function_with_list, [[int]], int) - -def function_returning_list(): - pass -register_external(function_returning_list, [], [int]) - -def test_register_external_return_goes_back(): - """ - Check whether it works to pass the same list from one external - fun to another - [bookkeeper and list joining issues] - """ - def f(): - return function_with_list(function_returning_list()) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def function_withspecialcase(arg): - return repr(arg) -register_external(function_withspecialcase, args=None, result=str) - -def test_register_external_specialcase(): - def f(): - x = function_withspecialcase - return x(33) + x("aaa") + x([]) + "\n" - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeString) + def test_basic(self): + """ + A ExtFuncEntry provides an annotation for a function, no need to flow + its graph. + """ + def b(x): + "NOT_RPYTHON" + return eval("x+40") + + class BTestFuncEntry(ExtFuncEntry): + _about_ = b + name = 'b' + signature_args = [annmodel.SomeInteger()] + signature_result = annmodel.SomeInteger() + + def f(): + return b(2) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + res = interpret(f, []) + assert res == 42 + + def test_lltypeimpl(self): + """ + interpret() calls lltypeimpl instead of of the function/ + """ + def c(y, x): + yyy + + class CTestFuncEntry(ExtFuncEntry): + _about_ = c + name = 'ccc' + signature_args = [annmodel.SomeInteger()] * 2 + signature_result = annmodel.SomeInteger() + + def lltypeimpl(y, x): + return y + x + lltypeimpl = staticmethod(lltypeimpl) + + def f(): + return c(3, 4) + + res = interpret(f, []) + assert res == 7 + + def test_callback(self): + """ + Verify annotation when a callback function is in the arguments list. + """ + def d(y): + return eval("y()") + + class DTestFuncEntry(ExtFuncEntry): + _about_ = d + name = 'd' + signature_args = [annmodel.SomeGenericCallable(args=[], result= + annmodel.SomeFloat())] + signature_result = annmodel.SomeFloat() + + def callback(): + return 2.5 + + def f(): + return d(callback) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeFloat) + assert a.translator._graphof(callback) + + def test_register_external_signature(self): + """ + Test the standard interface for external functions. + """ + def dd(): + pass + register_external(dd, [int], int) + + def f(): + return dd(3) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_tuple_args(self): + """ + Verify the annotation of a registered external function which takes a + tuple argument. + """ + + def function_with_tuple_arg(): + """ + Dummy function which is declared via register_external to take a + tuple as an argument so that register_external's behavior for + tuple-taking functions can be verified. + """ + register_external(function_with_tuple_arg, [(int,)], int) + + def f(): + return function_with_tuple_arg((1,)) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + + # Not a very good assertion, but at least it means _something_ happened. + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_return_goes_back(self): + """ + Check whether it works to pass the same list from one external + fun to another + [bookkeeper and list joining issues] + """ + def function_with_list(): + pass + register_external(function_with_list, [[int]], int) + + def function_returning_list(): + pass + register_external(function_returning_list, [], [int]) + + def f(): + return function_with_list(function_returning_list()) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_specialcase(self): + """ + When args=None, the external function accepts any arguments unmodified. + """ + def function_withspecialcase(arg): + return repr(arg) + register_external(function_withspecialcase, args=None, result=str) + + def f(): + x = function_withspecialcase + return x(33) + x("aaa") + x([]) + "\n" + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeString) Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/test/test_rstr.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/test/test_rstr.py Tue Jul 20 20:55:30 2010 @@ -131,7 +131,7 @@ s = c * mul res = 0 for i in range(len(s)): - res = res*10 + ord(const(s[i])) - ord(const('0')) + res = res*10 + ord(const(s[i])[0]) - ord(const('0')[0]) c2 = c c2 *= mul res = 10 * res + (c2 == s) @@ -577,7 +577,7 @@ sum = 0 for num in l: if len(num): - sum += ord(num) - ord(const('0')) + sum += ord(num[0]) - ord(const('0')[0]) return sum + len(l) * 100 for i in range(5): res = self.interpret(fn, [i]) @@ -863,6 +863,24 @@ res = self.interpret(f, [1]) assert self.ll_to_string(res) == "hello" + def test_str_unichar(self): + def f(i): + c = u"abc" + return str(c[i])[0] + assert self.interpret(f, [1]) == "b" + + def test_encode_char(self): + def f(i): + c = u"abc" + return c[i].encode("ascii") + assert self.ll_to_string(self.interpret(f, [0])) == "a" + + def test_encode_char_latin1(self): + def f(i): + c = u"abc" + return c[i].encode("latin-1") + assert self.ll_to_string(self.interpret(f, [0])) == "a" + def FIXME_test_str_to_pystringobj(): def f(n): if n >= 0: Modified: pypy/branch/x86-64-jit-backend/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/c/test/test_typed.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/c/test/test_typed.py Tue Jul 20 20:55:30 2010 @@ -789,3 +789,37 @@ return u'hello' + unichr(i) f = self.getcompiled(func, [int]) assert f(0x1234) == u'hello\u1234' + + def test_ovfcheck_float_to_int(self): + from pypy.rlib.rarithmetic import ovfcheck_float_to_int + + def func(fl): + try: + return ovfcheck_float_to_int(fl) + except OverflowError: + return -666 + f = self.getcompiled(func, [float]) + assert f(-123.0) == -123 + + for frac in [0.0, 0.01, 0.99]: + # strange things happening for float to int on 64 bit: + # int(float(i)) != i because of rounding issues + x = sys.maxint + while int(x + frac) > sys.maxint: + x -= 1 + assert f(x + frac) == int(x + frac) + + x = sys.maxint + while int(x - frac) <= sys.maxint: + x += 1 + assert f(x - frac) == -666 + + x = -sys.maxint-1 + while int(x - frac) < -sys.maxint-1: + x += 1 + assert f(x - frac) == int(x - frac) + + x = -sys.maxint-1 + while int(x + frac) >= -sys.maxint-1: + x -= 1 + assert f(x + frac) == -666 Modified: pypy/branch/x86-64-jit-backend/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/platform/__init__.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/platform/__init__.py Tue Jul 20 20:55:30 2010 @@ -161,7 +161,7 @@ return (library_dirs + self.link_flags + export_flags + link_files + list(eci.link_extra) + libraries) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if eci.export_symbols: raise ValueError("This platform does not support export symbols") return [] Modified: pypy/branch/x86-64-jit-backend/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/platform/darwin.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/platform/darwin.py Tue Jul 20 20:55:30 2010 @@ -56,7 +56,7 @@ include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -65,6 +65,9 @@ for sym in eci.export_symbols: f.write("_%s\n" % (sym,)) f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["-Wl,-exported_symbols_list,%s" % (response_file,)] class Darwin_i386(Darwin): Modified: pypy/branch/x86-64-jit-backend/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/platform/posix.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/platform/posix.py Tue Jul 20 20:55:30 2010 @@ -39,7 +39,7 @@ def _link_args_from_eci(self, eci, standalone): return Platform._link_args_from_eci(self, eci, standalone) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -50,6 +50,9 @@ f.write("%s;\n" % (sym,)) f.write("};") f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["-Wl,--export-dynamic,--version-script=%s" % (response_file,)] def _link(self, cc, ofiles, link_args, standalone, exe_name): @@ -90,7 +93,7 @@ if shared: linkflags = self._args_for_shared(linkflags) - linkflags += self._exportsymbols_link_flags(eci) + linkflags += self._exportsymbols_link_flags(eci, relto=path) if shared: libname = exe_name.new(ext='').basename Modified: pypy/branch/x86-64-jit-backend/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/platform/windows.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/platform/windows.py Tue Jul 20 20:55:30 2010 @@ -141,7 +141,7 @@ # Windows needs to resolve all symbols even for DLLs return super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -150,6 +150,9 @@ for sym in eci.export_symbols: f.write("/EXPORT:%s\n" % (sym,)) f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["@%s" % (response_file,)] def _compile_c_file(self, cc, cfile, compile_args): @@ -219,7 +222,7 @@ if shared: linkflags = self._args_for_shared(linkflags) + [ '/EXPORT:$(PYPY_MAIN_FUNCTION)'] - linkflags += self._exportsymbols_link_flags(eci) + linkflags += self._exportsymbols_link_flags(eci, relto=path) if shared: so_name = exe_name.new(purebasename='lib' + exe_name.purebasename, From hakanardo at codespeak.net Tue Jul 20 21:02:59 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 20 Jul 2010 21:02:59 +0200 (CEST) Subject: [pypy-svn] r76294 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100720190259.9F357282BF2@codespeak.net> Author: hakanardo Date: Tue Jul 20 21:02:41 2010 New Revision: 76294 Added: pypy/branch/interplevel-array/pypy/module/array/interp_array_wrapped.py (props changed) - copied unchanged from r76278, pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/__init__.py Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: started a multimethod implementation of the array class Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/__init__.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/__init__.py Tue Jul 20 21:02:41 2010 @@ -1,13 +1,16 @@ -# Package initialisation from pypy.interpreter.mixedmodule import MixedModule +from pypy.module.array.interp_array import types, W_ArrayBase +from pypy.objspace.std.model import registerimplementation +for mytype in types.values(): + print mytype.w_class + registerimplementation(mytype.w_class) class Module(MixedModule): - appleveldefs = { + interpleveldefs = { + 'array' : 'interp_array.W_ArrayBase', } - interpleveldefs = { - 'array': 'interp_array.W_WrappedArray', - 'simple_array': 'interp_simple.simple_array', + appleveldefs = { } Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Jul 20 21:02:41 2010 @@ -1,4 +1,3 @@ -from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.rpython.lltypesystem import lltype, rffi @@ -8,29 +7,69 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rstruct.runpack import runpack from pypy.interpreter.argument import Arguments, Signature +from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable +from pypy.objspace.std.stdtypedef import SMM, StdTypeDef +from pypy.objspace.std.register_all import register_all +from pypy.objspace.std.model import W_Object -import py -app_array = py.path.local(__file__).dirpath().join('app_array.py') -app = ApplevelClass(file(str(app_array)).read()) - -def appmethod(n,allappfn={}): - if not allappfn.has_key(n): - #exec 'import %s as mod'%f.__module__ - #src=file(re.sub('.pyc$', '.py', mod.__file__)).read() - #app = ApplevelClass(src) - import app_array - f = getattr(app_array,n) - args = f.func_code.co_varnames[0:f.func_code.co_argcount] - args = ', '.join(['space'] + ['w_'+s for s in args]) - appfn = app.interphook(n) - exec """def descr(%s): - return appfn(%s)"""%(args, args) in locals() - descr.__name__='descr_appmethod_%s'%n - allappfn[n]=interp2app(descr) - return allappfn[n] +def w_array(space, typecode, w_initializer=None): + if len(typecode) != 1: + msg = 'array() argument 1 must be char, not str' + raise OperationError(space.w_TypeError, space.wrap(msg)) + typecode=typecode[0] + + for tc in unroll_typecodes: + if typecode == tc: + a = types[tc].w_class(space) + if not space.is_w(w_initializer, space.w_None): + if space.type(w_initializer) is space.w_str: + space.call_method(a, 'fromstring', w_initializer) + elif space.type(w_initializer) is space.w_unicode: + space.call_method(a, 'fromunicode', w_initializer) + elif space.type(w_initializer) is space.w_list: + space.call_method(a, 'fromlist', w_initializer) + else: + space.call_method(a, 'extend', w_initializer) + break + else: + msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' + raise OperationError(space.w_ValueError, space.wrap(msg)) -class W_ArrayBase(Wrappable): - pass + return a +w_array.unwrap_spec = (ObjSpace, str, W_Root) + +def w_array_sub(space, w_cls, typecode, w_initializer=None): + return w_array(space, typecode, w_initializer) +w_array_sub.unwrap_spec = (ObjSpace, W_Root, str, W_Root) + + +array_append = SMM('append', 2) +array_extend = SMM('extend', 2) + +array_tolist = SMM('tolist', 1) +array_fromlist = SMM('fromlist', 2) + + +def descr_itemsize(space, self): + return space.wrap(self.itemsize) +def descr_typecode(space, self): + return space.wrap(self.typecode) + + +type_typedef = StdTypeDef( + 'array', + __new__ = interp2app(w_array_sub), + itemsize = GetSetProperty(descr_itemsize), + typecode = GetSetProperty(descr_typecode), + ) +type_typedef.registermethods(globals()) + + +class W_ArrayBase(W_Object): + typedef = type_typedef + @staticmethod + def register(typeorder): + typeorder[W_ArrayBase] = [] class TypeCode(object): def __init__(self, itemtype, unwrap, canoverflow=False, signed=False): @@ -92,10 +131,13 @@ return val class W_Array(W_ArrayBase): - itemsize = mytype.bytes typecode = mytype.typecode - + + @staticmethod + def register(typeorder): + typeorder[W_Array] = [] + def __init__(self, space): self.space = space self.len = 0 @@ -138,9 +180,11 @@ raise OperationError(space.w_OverflowError, space.wrap(msg)) return rffi.cast(mytype.itemtype, item) + def __del__(self): self.setlen(0) + def setlen(self, size): if size > 0: if size > self.allocated or size < self.allocated/2: @@ -165,56 +209,9 @@ lltype.free(self.buffer, flavor='raw') self.buffer = new_buffer self.len = size - - setlen.unwrap_spec = ['self', int] - - def descr_len(self): - return self.space.wrap(self.len) - descr_len.unwrap_spec = ['self'] - def descr_getslice(self, w_idx): - space = self.space - start, stop, step = space.decode_index(w_idx, self.len) - if step < 0: - w_lst = self.descr_tolist() - w_lst = space.getitem(w_lst, w_idx) - w_a=mytype.w_class(self.space) - w_a.descr_fromsequence(w_lst) - else: - size = (stop - start) / step - if (stop - start) % step > 0: size += 1 - if size < 0: size = 0 - w_a=mytype.w_class(self.space) - w_a.setlen(size) - j=0 - for i in range(start, stop, step): - w_a.buffer[j]=self.buffer[i] - j+=1 - return w_a - - def descr_getitem(self, w_idx): - space=self.space - start, stop, step = space.decode_index(w_idx, self.len) - if step == 0: - idx = start - item = self.buffer[idx] - tc=mytype.typecode - if tc == 'b' or tc == 'B' or tc == 'h' or tc == 'H' or tc == 'i' or tc == 'l': - item = rffi.cast(lltype.Signed, item) - elif mytype.typecode == 'f': - item = float(item) - return self.space.wrap(item) - else: - return self.descr_getslice(w_idx) - descr_getitem.unwrap_spec = ['self', W_Root] - - def descr_append(self, w_x): - x = self.item_w(w_x) - self.setlen(self.len + 1) - self.buffer[self.len - 1] = x - descr_append.unwrap_spec = ['self', W_Root] - def descr_fromsequence(self, w_seq): + def fromsequence(self, w_seq): space = self.space oldlen = self.len try: @@ -239,472 +236,82 @@ self.setlen(oldlen + i) raise self.setlen(oldlen + i) - - descr_fromsequence.unwrap_spec = ['self', W_Root] - - - def descr_extend(self, w_iterable): - space=self.space - if isinstance(w_iterable, W_ArrayBase): - if mytype.typecode != w_iterable.typecode: - msg = "can only extend with array of same kind" - raise OperationError(space.w_TypeError, space.wrap(msg)) - if isinstance(w_iterable, W_Array): - oldlen = self.len - new = w_iterable.len - self.setlen(self.len + new) - for i in range(new): - self.buffer[oldlen + i] = w_iterable.buffer[i] - else: - assert False - else: - self.descr_fromsequence(w_iterable) - descr_extend.unwrap_spec = ['self', W_Root] - - def descr_setslice(self, w_idx, w_item): - space=self.space - start, stop, step = self.space.decode_index(w_idx, self.len) - if isinstance(w_item, W_Array): # Implies mytype.typecode == w_item.typecode - size = (stop - start) / step - if (stop - start) % step > 0: size += 1 - if w_item.len != size or step < 0: - w_lst = self.descr_tolist() - w_item = space.call_method(w_item, 'tolist') - space.setitem(w_lst, w_idx, w_item) - self.setlen(0) - self.descr_fromsequence(w_lst) - else: - j=0 - for i in range(start, stop, step): - self.buffer[i]=w_item.buffer[j] - j+=1 - return - msg='can only assign array to array slice' - raise OperationError(self.space.w_TypeError, self.space.wrap(msg)) - def descr_setitem(self, w_idx, w_item): - start, stop, step = self.space.decode_index(w_idx, self.len) - if step == 0: - idx = start - item = self.item_w(w_item) - self.buffer[idx] = item - else: - self.descr_setslice(w_idx, w_item) - descr_setitem.unwrap_spec = ['self', W_Root, W_Root] - - def charbuf(self): - return rffi.cast(rffi.CCHARP, self.buffer) - - def descr_fromstring(self, s): - if len(s)%mytype.bytes !=0: - msg = 'string length not a multiple of item size' - raise OperationError(self.space.w_ValueError, self.space.wrap(msg)) - oldlen = self.len - new = len(s) / mytype.bytes - self.setlen(oldlen + new) - if False: - for i in range(new): - p = i * mytype.bytes - item=runpack(mytype.typecode, s[p:p + mytype.bytes]) - #self.buffer[oldlen + i]=self.item_w(self.space.wrap(item)) - self.buffer[oldlen + i]=rffi.cast(mytype.itemtype, item) - else: - pbuf =self.charbuf() - for i in range(len(s)): - pbuf[oldlen * mytype.bytes + i] = s[i] - - descr_fromstring.unwrap_spec = ['self', str] - - def descr_tolist(self): - w_l=self.space.newlist([]) - for i in range(self.len): - w_l.append(self.descr_getitem(self.space.wrap(i))) - return w_l - #return self.space.newlist([self.space.wrap(i) for i in self.buffer]) - descr_tolist.unwrap_spec = ['self'] - - def descr_tostring(self): - pbuf = self.charbuf() - s = '' - i=0 - while i < self.len * mytype.bytes: - s += pbuf[i] - i+=1 - return self.space.wrap(s) - - def descr_buffer(self): - from pypy.interpreter.buffer import StringLikeBuffer - space = self.space - return space.wrap(StringLikeBuffer(space, self.descr_tostring())) - descr_buffer.unwrap_spec = ['self'] - - def descr_isarray(self, w_other): - return self.space.wrap(isinstance(w_other, W_ArrayBase)) - def descr_reduce(self): - space=self.space - if self.len>0: - args=[space.wrap(self.typecode), self.descr_tostring()] - else: - args=[space.wrap(self.typecode)] - from pypy.interpreter.mixedmodule import MixedModule - w_mod = space.getbuiltinmodule('array') - mod = space.interp_w(MixedModule, w_mod) - w_new_inst = mod.get('array') - return space.newtuple([w_new_inst, space.newtuple(args)]) - - def descr_pop(self, i=-1): - if i < 0: i += self.len - if i < 0 or i >= self.len: - msg = 'pop index out of range' - raise OperationError(self.space.w_IndexError, self.space.wrap(msg)) - val = self.descr_getitem(self.space.wrap(i)) - while i < self.len - 1: - self.buffer[i] = self.buffer[i + 1] - i += 1 - self.setlen(self.len-1) - return val - descr_pop.unwrap_spec = ['self', int] + def len__Array(space, self): + return space.wrap(self.len) - def descr_copy(self): - w_a = mytype.w_class(self.space) - w_a.descr_fromsequence(self) - return w_a + def getitem__Array_ANY(space, self, w_idx): + idx = space.int_w(w_idx) + item = self.buffer[idx] + tc=mytype.typecode + if (tc == 'b' or tc == 'B' or tc == 'h' or tc == 'H' or + tc == 'i' or tc == 'l'): + item = rffi.cast(lltype.Signed, item) + elif mytype.typecode == 'f': + item = float(item) + return self.space.wrap(item) + + def setitem__Array_ANY_ANY(space, self, w_idx, w_item): + idx = space.int_w(w_idx) + item = self.item_w(w_item) + self.buffer[idx] = item + + def array_append__Array_ANY(space, self, w_x): + x = self.item_w(w_x) + self.setlen(self.len + 1) + self.buffer[self.len - 1] = x - def descr_buffer_info(self): - space = self.space - w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer)) - w_len = space.wrap(self.len) - return space.newtuple([w_ptr, w_len]) - - def descr_byteswap(self): - if mytype.bytes not in [1, 2, 4, 8]: - msg="byteswap not supported for this array" - raise OperationError(self.space.w_RuntimeError, self.space.wrap(msg)) - if self.len == 0: return - bytes = self.charbuf() - tmp = [bytes[0]] * mytype.bytes - for start in range(0, self.len * mytype.bytes, mytype.bytes): - stop = start + mytype.bytes - 1 - for i in range(mytype.bytes): - tmp[i] = bytes[start + i] - for i in range(mytype.bytes): - bytes[stop - i] = tmp[i] - - def descr_add(self, w_other): - other = self.space.interp_w(W_Array, w_other) - w_a = mytype.w_class(self.space) - w_a.setlen(self.len + other.len) - for i in range(self.len): - w_a.buffer[i] = self.buffer[i] - for i in range(other.len): - w_a.buffer[i + self.len] = other.buffer[i] - return w_a - descr_add.unwrap_spec = ['self', W_Root] - def descr_iadd(self, w_other): - other = self.space.interp_w(W_Array, w_other) + def array_extend__Array_ANY(space, self, w_iterable): + if isinstance(w_iterable, W_Array): oldlen = self.len - otherlen = other.len - self.setlen(oldlen + otherlen) - for i in range(otherlen): - self.buffer[oldlen + i] = other.buffer[i] - return self - descr_add.unwrap_spec = ['self', W_Root] - - def descr_mul(self, repeat): - w_a = mytype.w_class(self.space) - w_a.setlen(self.len * repeat) - for r in range(repeat): - for i in range(self.len): - w_a.buffer[r * self.len + i] = self.buffer[i] - return w_a - descr_mul.unwrap_spec = ['self', int] - - def descr_imul(self, repeat): - oldlen=self.len - self.setlen(self.len * repeat) - for r in range(1,repeat): - for i in range(oldlen): - self.buffer[r * oldlen + i] = self.buffer[i] - return self - descr_imul.unwrap_spec = ['self', int] - - def descr_delitem(self, w_idx): - space=self.space - w_lst = self.descr_tolist() - space.delitem(w_lst, w_idx) - self.setlen(0) - self.descr_fromsequence(w_lst) - descr_delitem.unwrap_spec = ['self', W_Root] - - def descr_iter(self): - return W_ArrayIter(self) - + new = w_iterable.len + self.setlen(self.len + new) + for i in range(new): + if oldlen + i < self.len: + self.buffer[oldlen + i] = w_iterable.buffer[i] + else: + self.setlen(oldlen + i + 1) + self.buffer[oldlen + i] = w_iterable.buffer[i] + self.setlen(oldlen + i + 1) + elif isinstance(w_iterable, W_ArrayBase): + msg = "can only extend with array of same kind" + raise OperationError(space.w_TypeError, space.wrap(msg)) + else: + self.fromsequence(w_iterable) - def descr_itemsize(space, self): - return space.wrap(mytype.bytes) - def descr_typecode(space, self): - return space.wrap(mytype.typecode) - - W_ArrayIter.__name__ = 'W_ArrayIterType_'+mytype.typecode - W_ArrayIter.typedef = TypeDef( - 'ArrayIterType_'+mytype.typecode, - __iter__ = interp2app(W_ArrayIter.iter_w), - next = interp2app(W_ArrayIter.next_w), - ) + def array_tolist__Array(space, self): + w_l=space.newlist([]) + for i in range(self.len): + w_l.append(getitem__Array_ANY(space, self, space.wrap(i))) + return w_l + + def array_fromlist__Array_ANY(space, self, w_lst): + if space.type(w_lst) is not space.w_list: + msg = "arg must be list" + raise OperationError(space.w_TypeError, space.wrap(msg)) + s = self.len + try: + self.fromsequence(w_lst) + except OperationError: + self.setlen(s) + raise + - W_Array.__name__ = 'W_ArrayType_'+mytype.typecode - W_Array.typedef = TypeDef( - 'ArrayType_'+mytype.typecode, - append = interp2app(W_Array.descr_append), - __len__ = interp2app(W_Array.descr_len), - __getitem__ = interp2app(W_Array.descr_getitem), - __setitem__ = interp2app(W_Array.descr_setitem), - __delitem__ = interp2app(W_Array.descr_delitem), - __getslice__ = appmethod('__getslice__'), - __setslice__ = appmethod('__setslice__'), - __delslice__ = appmethod('__delslice__'), - - itemsize = GetSetProperty(descr_itemsize, cls=W_Array), - typecode = GetSetProperty(descr_typecode, cls=W_Array), - extend = interp2app(W_Array.descr_extend), - - _fromsequence= interp2app(W_Array.descr_fromsequence), - fromstring = interp2app(W_Array.descr_fromstring), - fromunicode = appmethod('fromunicode'), - fromfile = appmethod('fromfile'), - read = appmethod('fromfile'), - fromlist = appmethod('fromlist'), - - tolist = interp2app(W_Array.descr_tolist), - tounicode = appmethod('tounicode'), - tofile = appmethod('tofile'), - write = appmethod('tofile'), - tostring = interp2app(W_Array.descr_tostring), - - _setlen = interp2app(W_Array.setlen), - __buffer__ = interp2app(W_Array.descr_buffer), - - __repr__ = appmethod('__repr__'), - count = appmethod('count'), - index = appmethod('index'), - remove = appmethod('remove'), - reverse = appmethod('reverse'), - insert = appmethod('insert'), - pop = interp2app(W_Array.descr_pop), - - __eq__ = appmethod('__eq__'), - __ne__ = appmethod('__ne__'), - __lt__ = appmethod('__lt__'), - __gt__ = appmethod('__gt__'), - __le__ = appmethod('__le__'), - __ge__ = appmethod('__ge__'), + def cmp__Array_ANY(space, self, other): + if isinstance(other, W_ArrayBase): + w_lst1 = array_tolist__Array(space, self) + w_lst2 = array_tolist__Array(space, other) + return space.cmp(w_lst1, w_lst2) + else: + raise OperationError(space.w_NotImplementedError, space.wrap('')) - _isarray = interp2app(W_Array.descr_isarray), - __reduce__ = interp2app(W_Array.descr_reduce), - __copy__ = interp2app(W_Array.descr_copy), - - __add__ = interp2app(W_Array.descr_add), - __iadd__ = interp2app(W_Array.descr_iadd), - __mul__ = interp2app(W_Array.descr_mul), - __rmul__ = interp2app(W_Array.descr_mul), - __imul__ = interp2app(W_Array.descr_imul), - - buffer_info = interp2app(W_Array.descr_buffer_info), - byteswap = interp2app(W_Array.descr_byteswap), - - __iter__ = interp2app(W_Array.descr_iter), - __contains__ = appmethod('__contains__'), - ) - + W_Array.__name__ = 'W_ArrayType_'+mytype.typecode mytype.w_class = W_Array + register_all(locals(), globals()) for mytype in types.values(): make_array(mytype) - -initiate=app.interphook('initiate') -def array(space, typecode, w_initializer=None): - if len(typecode) != 1: - msg = 'array() argument 1 must be char, not str' - raise OperationError(space.w_TypeError, space.wrap(msg)) - typecode=typecode[0] - - for tc in unroll_typecodes: - if typecode == tc: - a = types[tc].w_class(space) - initiate(space, a, w_initializer) - ## if w_initializer is not None: - ## if not space.is_w(w_initializer, space.w_None): - ## a.descr_fromsequence(w_initializer) - break - else: - msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' - raise OperationError(space.w_ValueError, space.wrap(msg)) - - - return a -array.unwrap_spec = (ObjSpace, str, W_Root) - -class W_WrappedArray(Wrappable): - _immutable_fields_ = ['_array'] - - def __init__(self, space, a): - self.space = space - self._array = a - - def descr_init(self, typecode, w_initializer=None): - pass - descr_init.unwrap_spec = ['self', str, W_Root] - - def descr_pop(self, w_i=-1): - space = self.space - return space.call_method(self._array, 'pop', w_i) - descr_pop.unwrap_spec = ['self', W_Root] - - def descr_getitem(self, w_i): - space = self.space - w_item = space.getitem(self._array, w_i) - if isinstance(w_item, W_ArrayBase): - return W_WrappedArray(space, w_item) - return w_item - descr_getitem.unwrap_spec = ['self', W_Root] - - def descr_iadd(self, w_i): - space = self.space - w_i = space.interp_w(W_WrappedArray, w_i) - w_item = space.call_method(self._array, '__iadd__', w_i._array) - return self - descr_iadd.unwrap_spec = ['self', W_Root] - - def descr_imul(self, w_i): - space = self.space - w_item = space.call_method(self._array, '__imul__', w_i) - return self - descr_imul.unwrap_spec = ['self', W_Root] - - def descr_reduce(self): - space=self.space - if space.int_w(space.len(self._array)) > 0: - w_s = space.call_method(self._array, 'tostring') - args = [space.wrap(self._array.typecode), w_s] - else: - args = [space.wrap(self._array.typecode)] - return space.newtuple([space.type(self), space.newtuple(args)]) - - -def unwrap_array(w_a): - if isinstance(w_a, W_WrappedArray): - return w_a._array - return w_a - -def make_descr(fn, nargs, wrp): - if nargs == 0: - def descr(space, self): - ret = space.call_method(self._array, fn) - if (wrp): return W_WrappedArray(space, ret) - return ret - - elif nargs == 1: - def descr(space, self, w_a): - ret = space.call_method(self._array, fn, - unwrap_array(w_a)) - if (wrp): return W_WrappedArray(space, ret) - return ret - - elif nargs == 2: - def descr(space, self, w_a, w_b): - ret = space.call_method(self._array, fn, - unwrap_array(w_a), unwrap_array(w_b)) - if (wrp): return W_WrappedArray(space, ret) - return ret - - elif nargs == 3: - def descr(space, self, w_a, w_b, w_c): - ret = space.call_method(self._array, fn, - unwrap_array(w_a), unwrap_array(w_b), unwrap_array(w_c)) - if (wrp): return W_WrappedArray(space, ret) - return ret - - else: - raise NotImplementedError - - descr.unwrap_spec = (ObjSpace, W_WrappedArray) + (W_Root,) * nargs - descr.__name__ = 'W_WrappedArray_descr_' + fn - return interp2app(descr) - -typedefs={} -for fn, nargs, wrp in (('append', 1, False), - ('__len__', 0, False), - ('__setitem__', 2, False), - ('__delitem__', 1, False), - ('__getslice__', 2, True), - ('__setslice__', 3, False), - ('__delslice__', 2, False), - ('extend', 1, False), - ('fromstring', 1, False), - ('fromunicode', 1, False), - ('fromfile', 2, False), - ('read', 2, False), - ('fromlist', 1, False), - ('tolist', 0, False), - ('tounicode', 0, False), - ('tofile', 1, False), - ('write', 1, False), - ('tostring', 0, False), - ('__buffer__', 0, False), - ('__repr__', 0, False), - ('count', 1, False), - ('index', 1, False), - ('remove', 1, False), - ('reverse', 0, False), - ('insert', 2, False), - ('__eq__', 1, False), - ('__ne__', 1, False), - ('__lt__', 1, False), - ('__gt__', 1, False), - ('__le__', 1, False), - ('__ge__', 1, False), - ('__copy__', 0, True), - ('__add__', 1, True), - ('__mul__', 1, True), - ('__rmul__', 1, True), - ('buffer_info', 0, False), - ('byteswap', 0, False), - ('__iter__', 0, False), - ('__contains__', 1, False), - ): - typedefs[fn] = make_descr(fn, nargs, wrp) - -def new_array(space, w_cls, typecode, w_initializer=None, w_args=None): - if len(w_args.arguments_w) > 0: - msg = 'array() takes at most 2 arguments' - raise OperationError(space.w_TypeError, space.wrap(msg)) - w_array = space.allocate_instance(W_WrappedArray, w_cls) - w_array.__init__(space, array(space, typecode, w_initializer)) - return w_array -new_array.unwrap_spec = (ObjSpace, W_Root, str, W_Root, Arguments) - -def descr_wrapped_itemsize(space, self): - return space.wrap(self._array.itemsize) - -def descr_wrapped_typecode(space, self): - return space.wrap(self._array.typecode) - - -W_WrappedArray.typedef = TypeDef( - 'array', - __module__ = 'array', - __new__ = interp2app(new_array), - __init__ = interp2app(W_WrappedArray.descr_init), - pop = interp2app(W_WrappedArray.descr_pop), - __getitem__ = interp2app(W_WrappedArray.descr_getitem), - __iadd__ = interp2app(W_WrappedArray.descr_iadd), - __imul__ = interp2app(W_WrappedArray.descr_imul), - __reduce__ = interp2app(W_WrappedArray.descr_reduce), - itemsize = GetSetProperty(descr_wrapped_itemsize, cls=W_WrappedArray), - typecode = GetSetProperty(descr_wrapped_typecode, cls=W_WrappedArray), - - **typedefs -) + print mytype, mytype.w_class Added: pypy/branch/interplevel-array/pypy/module/array/test/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/test/__init__.py Tue Jul 20 21:02:41 2010 @@ -0,0 +1 @@ +# Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Tue Jul 20 21:02:41 2010 @@ -2,22 +2,25 @@ import py -class AppTestSimpleArray: - def setup_class(cls): - cls.space = gettestobjspace(usemodules=('array',)) - cls.w_simple_array = cls.space.appexec([], """(): - import array - return array.simple_array - """) - - def test_simple(self): - a = self.simple_array(10) - a[5] = 7.42 - assert a[5] == 7.42 +## class AppTestSimpleArray: +## def setup_class(cls): +## cls.space = gettestobjspace(usemodules=('array',)) +## cls.w_simple_array = cls.space.appexec([], """(): +## import array +## return array.simple_array +## """) + +## def test_simple(self): +## a = self.simple_array(10) +## a[5] = 7.42 +## assert a[5] == 7.42 class BaseArrayTests: def test_ctor(self): + assert len(self.array('c')) == 0 + assert len(self.array('i')) == 0 + raises(TypeError, self.array, 'hi') raises(TypeError, self.array, 1) raises(ValueError, self.array, 'q') @@ -45,9 +48,17 @@ assert len(a) == 3 b = self.array('c', a) + assert len(b) == 3 assert a == b raises(TypeError, self.array, 'i', a) + a = self.array('i', (1,2,3)) + b = self.array('h', (1,2,3)) + assert a == b + + for tc in 'bhilBHILfd': + assert self.array(tc).typecode == tc + def test_value_range(self): values = (-129, 128, -128, 127, 0, 255, -1, 256, -32768, 32767, -32769, 32768, 65535, 65536, From hakanardo at codespeak.net Tue Jul 20 22:23:06 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 20 Jul 2010 22:23:06 +0200 (CEST) Subject: [pypy-svn] r76295 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100720202306.DCED5282BF2@codespeak.net> Author: hakanardo Date: Tue Jul 20 22:23:04 2010 New Revision: 76295 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: fromfile Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Jul 20 22:23:04 2010 @@ -48,6 +48,10 @@ array_tolist = SMM('tolist', 1) array_fromlist = SMM('fromlist', 2) +array_tostring = SMM('tostring', 1) +array_fromstring = SMM('fromstring', 2) +array_tofile = SMM('tofile', 1) +array_fromfile = SMM('fromfile', 3) def descr_itemsize(space, self): @@ -236,8 +240,12 @@ self.setlen(oldlen + i) raise self.setlen(oldlen + i) - + def charbuf(self): + return rffi.cast(rffi.CCHARP, self.buffer) + + + # List interface def len__Array(space, self): return space.wrap(self.len) @@ -282,6 +290,9 @@ else: self.fromsequence(w_iterable) + + # Convertions + def array_tolist__Array(space, self): w_l=space.newlist([]) for i in range(self.len): @@ -298,7 +309,37 @@ except OperationError: self.setlen(s) raise - + + def array_fromstring__Array_ANY(space, self, w_s): + s = space.str_w(w_s) + if len(s)%mytype.bytes !=0: + msg = 'string length not a multiple of item size' + raise OperationError(self.space.w_ValueError, self.space.wrap(msg)) + oldlen = self.len + new = len(s) / mytype.bytes + self.setlen(oldlen + new) + cbuf =self.charbuf() + for i in range(len(s)): + cbuf[oldlen * mytype.bytes + i] = s[i] + + def array_fromfile__Array_ANY_ANY(space, self, w_f, w_n): + if space.type(w_f).name != 'file': # FIXME: this cant be the right way? + msg = "arg1 must be open file" + raise OperationError(space.w_TypeError, space.wrap(msg)) + n = space.int_w(w_n) + + size = self.itemsize * n + w_item = space.call_method(w_f, 'read', space.wrap(size)) + item = space.str_w(w_item) + if len(item) < size: + n = len(item) % self.itemsize + if n != 0: item = item[0:-(len(item) % self.itemsize)] + w_item = space.wrap(item) + array_fromstring__Array_ANY(space, self, w_item) + msg = "not enough items in file" + raise OperationError(space.w_EOFError, space.wrap(msg)) + array_fromstring__Array_ANY(space, self, w_item) + def cmp__Array_ANY(space, self, other): if isinstance(other, W_ArrayBase): From jcreigh at codespeak.net Wed Jul 21 15:14:54 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 21 Jul 2010 15:14:54 +0200 (CEST) Subject: [pypy-svn] r76296 - pypy/branch/x86-64-jit-backend/pypy/config Message-ID: <20100721131454.01129282B90@codespeak.net> Author: jcreigh Date: Wed Jul 21 15:14:53 2010 New Revision: 76296 Modified: pypy/branch/x86-64-jit-backend/pypy/config/translationoption.py Log: use boehm GC by default when translating 64-bit JIT Modified: pypy/branch/x86-64-jit-backend/pypy/config/translationoption.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/config/translationoption.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/config/translationoption.py Wed Jul 21 15:14:53 2010 @@ -342,6 +342,10 @@ 'jit': 'hybrid extraopts jit', } +# For now, 64-bit JIT requires boehm +if IS_64_BITS: + OPT_TABLE['jit'] = 'boehm extraopts jit' + def set_opt_level(config, level): """Apply optimization suggestions on the 'config'. The optimizations depend on the selected level and possibly on the backend. From hakanardo at codespeak.net Wed Jul 21 15:29:29 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 21 Jul 2010 15:29:29 +0200 (CEST) Subject: [pypy-svn] r76297 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100721132929.4E4A4282B90@codespeak.net> Author: hakanardo Date: Wed Jul 21 15:29:27 2010 New Revision: 76297 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: some typechecking passed over to the multimethods and slice support Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Wed Jul 21 15:29:27 2010 @@ -50,6 +50,8 @@ array_fromlist = SMM('fromlist', 2) array_tostring = SMM('tostring', 1) array_fromstring = SMM('fromstring', 2) +array_tounicode = SMM('tounicode', 1) +array_fromunicode = SMM('fromunicode', 2) array_tofile = SMM('tofile', 1) array_fromfile = SMM('fromfile', 3) @@ -231,10 +233,9 @@ else: myiter = space.listview for w_i in myiter(w_seq): - if oldlen + i < self.len: - self.buffer[oldlen + i] = self.item_w(w_i) - else: - self.descr_append(w_i) + if oldlen + i >= self.len: + self.setlen(oldlen + i + 1) + self.buffer[oldlen + i] = self.item_w(w_i) i += 1 except OperationError: self.setlen(oldlen + i) @@ -251,7 +252,8 @@ return space.wrap(self.len) def getitem__Array_ANY(space, self, w_idx): - idx = space.int_w(w_idx) + idx, stop, step = space.decode_index(w_idx, self.len) + assert step == 0 item = self.buffer[idx] tc=mytype.typecode if (tc == 'b' or tc == 'B' or tc == 'h' or tc == 'H' or @@ -261,10 +263,50 @@ item = float(item) return self.space.wrap(item) + def getitem__Array_Slice(space, self, w_slice): + start, stop, step = space.decode_index(w_slice, self.len) + if step < 0: + w_lst = array_tolist__Array(space, self) + w_lst = space.getitem(w_lst, w_idx) + w_a=mytype.w_class(self.space) + w_a.fromsequence(w_lst) + else: + size = (stop - start) / step + if (stop - start) % step > 0: size += 1 + w_a=mytype.w_class(self.space) + w_a.setlen(size) + for j,i in enumerate(range(start, stop, step)): + w_a.buffer[j]=self.buffer[i] + return w_a + + def getslice__Array_ANY_ANY(space, self, w_i, w_j): + return space.getitem(self, space.newslice(w_i, w_j, space.w_None)) + def setitem__Array_ANY_ANY(space, self, w_idx, w_item): - idx = space.int_w(w_idx) + idx, stop, step = space.decode_index(w_idx, self.len) + if step != 0: + msg='can only assign array to array slice' + raise OperationError(self.space.w_TypeError, self.space.wrap(msg)) item = self.item_w(w_item) self.buffer[idx] = item + + def setitem__Array_Slice_Array(space, self, w_idx, w_item): + start, stop, step = self.space.decode_index(w_idx, self.len) + size = (stop - start) / step + if (stop - start) % step > 0: size += 1 + if w_item.len != size or step < 0: + w_lst = array_tolist__Array(space, self) + w_item = space.call_method(w_item, 'tolist') + space.setitem(w_lst, w_idx, w_item) + self.setlen(0) + self.fromsequence(w_lst) + else: + for j,i in enumerate(range(start, stop, step)): + self.buffer[i]=w_item.buffer[j] + + def setslice__Array_ANY_ANY_ANY(space, self, w_i, w_j, w_x): + space.setitem(self, space.newslice(w_i, w_j, space.w_None), w_x) + def array_append__Array_ANY(space, self, w_x): x = self.item_w(w_x) @@ -278,11 +320,9 @@ new = w_iterable.len self.setlen(self.len + new) for i in range(new): - if oldlen + i < self.len: - self.buffer[oldlen + i] = w_iterable.buffer[i] - else: + if oldlen + i >= self.len: self.setlen(oldlen + i + 1) - self.buffer[oldlen + i] = w_iterable.buffer[i] + self.buffer[oldlen + i] = w_iterable.buffer[i] self.setlen(oldlen + i + 1) elif isinstance(w_iterable, W_ArrayBase): msg = "can only extend with array of same kind" @@ -299,10 +339,7 @@ w_l.append(getitem__Array_ANY(space, self, space.wrap(i))) return w_l - def array_fromlist__Array_ANY(space, self, w_lst): - if space.type(w_lst) is not space.w_list: - msg = "arg must be list" - raise OperationError(space.w_TypeError, space.wrap(msg)) + def array_fromlist__Array_List(space, self, w_lst): s = self.len try: self.fromsequence(w_lst) @@ -339,6 +376,20 @@ msg = "not enough items in file" raise OperationError(space.w_EOFError, space.wrap(msg)) array_fromstring__Array_ANY(space, self, w_item) + + def array_fromunicode__Array_Unicode(space, self, w_ustr): + if mytype.typecode != 'u': + msg = "fromunicode() may only be called on type 'u' arrays" + raise OperationError(space.w_ValueError, space.wrap(msg)) + + # XXX the following probable bug is not emulated: + # CPython accepts a non-unicode string or a buffer, and then + # behaves just like fromstring(), except that it strangely truncate + # string arguments at multiples of the unicode byte size. + # Let's only accept unicode arguments for now. + self.fromsequence(w_ustr) + + def cmp__Array_ANY(space, self, other): @@ -349,10 +400,32 @@ else: raise OperationError(space.w_NotImplementedError, space.wrap('')) + + def repr__Array(space, self): + if self.len == 0: + return "array('%s')" % self.typecode + elif self.typecode == "c": + r = space.repr(space.call_method(self, 'tostring')) + s = "array('%s', %s)" % (self.typecode, space.str_w(r)) + return space.wrap(s) + elif self.typecode == "u": + r = space.repr(space.call_method(self, 'tounicode')) + s = "array('%s', %s)" % (self.typecode, space.str_w(r)) + return space.wrap(s) + else: + r = space.repr(space.call_method(self, 'tolist')) + s = "array('%s', %s)" % (self.typecode, space.str_w(r)) + return space.wrap(s) + + W_Array.__name__ = 'W_ArrayType_'+mytype.typecode mytype.w_class = W_Array + + from pypy.objspace.std.sliceobject import W_SliceObject + from pypy.objspace.std.listobject import W_ListObject + from pypy.objspace.std.unicodeobject import W_UnicodeObject register_all(locals(), globals()) + for mytype in types.values(): make_array(mytype) - print mytype, mytype.w_class From jcreigh at codespeak.net Wed Jul 21 15:41:45 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 21 Jul 2010 15:41:45 +0200 (CEST) Subject: [pypy-svn] r76298 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100721134145.4FA0D282BDB@codespeak.net> Author: jcreigh Date: Wed Jul 21 15:41:43 2010 New Revision: 76298 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Log: rx86: clean up creation of all_instructions list Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Wed Jul 21 15:41:43 2010 @@ -635,13 +635,8 @@ # ____________________________________________________________ -# FIXME: What about 32-bit only or 64-bit only instructions? -# This is used to build the MachineCodeBlockWrapper. Missing -# some instructions could possibly lead to subtle bugs. +_classes = (AbstractX86CodeBuilder, X86_64_CodeBuilder, X86_32_CodeBuilder) -# FIXME: hack hack hack -all_instructions = ([name for name in AbstractX86CodeBuilder.__dict__ - if name.split('_')[0].isupper()] + - [name for name in X86_64_CodeBuilder.__dict__ - if name.split('_')[0].isupper()]) -all_instructions.sort() +# Used to build the MachineCodeBlockWrapper +all_instructions = sorted(name for cls in _classes for name in cls.__dict__ + if name.split('_')[0].isupper()) From jcreigh at codespeak.net Wed Jul 21 16:06:19 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 21 Jul 2010 16:06:19 +0200 (CEST) Subject: [pypy-svn] r76299 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100721140619.74619282B90@codespeak.net> Author: jcreigh Date: Wed Jul 21 16:06:15 2010 New Revision: 76299 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Log: remove baseless scary comments Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Wed Jul 21 16:06:15 2010 @@ -50,7 +50,6 @@ def location_code(self): return 'b' - # FIXME: This definition of assembler sufficient? def assembler(self): return repr(self) @@ -84,13 +83,11 @@ else: return 'r' - # FIXME: This definition of assembler sufficient? def assembler(self): return '%' + repr(self) class ImmedLoc(AssemblerLocation): _immutable_ = True - # XXX: Does this even make sense for an immediate? width = WORD def __init__(self, value): from pypy.rpython.lltypesystem import rffi, lltype From jcreigh at codespeak.net Wed Jul 21 16:23:55 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 21 Jul 2010 16:23:55 +0200 (CEST) Subject: [pypy-svn] r76300 - pypy/branch/x86-64-jit-backend/pypy/config Message-ID: <20100721142355.62F17282B90@codespeak.net> Author: jcreigh Date: Wed Jul 21 16:23:47 2010 New Revision: 76300 Modified: pypy/branch/x86-64-jit-backend/pypy/config/translationoption.py Log: select boehm with 64-bit JIT in a less fragile way Modified: pypy/branch/x86-64-jit-backend/pypy/config/translationoption.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/config/translationoption.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/config/translationoption.py Wed Jul 21 16:23:47 2010 @@ -344,7 +344,7 @@ # For now, 64-bit JIT requires boehm if IS_64_BITS: - OPT_TABLE['jit'] = 'boehm extraopts jit' + OPT_TABLE['jit'] = OPT_TABLE['jit'].replace('hybrid', 'boehm') def set_opt_level(config, level): """Apply optimization suggestions on the 'config'. From jcreigh at codespeak.net Wed Jul 21 16:31:54 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 21 Jul 2010 16:31:54 +0200 (CEST) Subject: [pypy-svn] r76301 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100721143154.94418282B90@codespeak.net> Author: jcreigh Date: Wed Jul 21 16:31:52 2010 New Revision: 76301 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Log: rx86: add comment describing the operand codes Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Wed Jul 21 16:31:52 2010 @@ -394,6 +394,22 @@ # ____________________________________________________________ +# Method names take the form of +# +# _ +# +# For example, the method name for "mov reg, immed" is MOV_ri. Operand order +# is Intel-style, with the destination first. +# +# The operand type codes are: +# r - register +# b - ebp/rbp offset +# s - esp/rsp offset +# j - address +# i - immediate +# x - XMM register +# a - 4-tuple: (base_register, scale_register, scale, offset) +# m - 2-tuple: (base_register, offset) class AbstractX86CodeBuilder(object): """Abstract base class.""" From fijal at codespeak.net Wed Jul 21 23:36:29 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 21 Jul 2010 23:36:29 +0200 (CEST) Subject: [pypy-svn] r76306 - pypy/trunk/pypy/jit/metainterp Message-ID: <20100721213629.31E6B282BF6@codespeak.net> Author: fijal Date: Wed Jul 21 23:36:27 2010 New Revision: 76306 Modified: pypy/trunk/pypy/jit/metainterp/executor.py Log: (hopefully) fix test_basic in x86 backend Modified: pypy/trunk/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/executor.py (original) +++ pypy/trunk/pypy/jit/metainterp/executor.py Wed Jul 21 23:36:27 2010 @@ -91,7 +91,7 @@ return BoxInt(cpu.bh_getarrayitem_gc_i(arraydescr, array, index)) def do_getarrayitem_raw(cpu, _, arraybox, indexbox, arraydescr): - array = arraybox.getref_base() + array = arraybox.getint() index = indexbox.getint() assert not arraydescr.is_array_of_pointers() if arraydescr.is_array_of_floats(): From getxsick at codespeak.net Thu Jul 22 01:38:51 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 22 Jul 2010 01:38:51 +0200 (CEST) Subject: [pypy-svn] r76307 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100721233851.5EB8C282BD4@codespeak.net> Author: getxsick Date: Thu Jul 22 01:38:49 2010 New Revision: 76307 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: don't use dynamic classes for CallDescr this is an initial commit. cache should be added, extra tests etc Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Thu Jul 22 01:38:49 2010 @@ -1,5 +1,6 @@ from pypy.rlib import rdynload from pypy.rpython.lltypesystem import rffi, lltype +from pypy.jit.backend.llsupport import descr, symbolic from pypy.jit.backend.x86.runner import CPU from pypy.jit.metainterp.history import LoopToken, BasicFailDescr from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr, NULLBOX @@ -36,6 +37,7 @@ assert isinstance(args_type, list) self.args_type = args_type self.res_type = res_type + self.res = None self.cpu = cpu lib = lib.handler bargs = [] @@ -65,24 +67,21 @@ raise ValueError(arg) if self.res_type == 'i': - res = lltype.Signed + self.res = lltype.Signed bres = BoxInt() elif self.res_type == 'f': - res = lltype.Float + self.res = lltype.Float bres = BoxFloat() elif self.res_type == 'p': - res = lltype.Signed + self.res = lltype.Signed bres = BoxPtr() elif self.res_type == 'v': - res = lltype.Void + self.res = lltype.Void bres = NULLBOX else: raise ValueError(self.res_type) - FPTR = lltype.Ptr(lltype.FuncType(args, res)) - FUNC = deref(FPTR) - calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) - + calldescr = self.gen_calldescr() # XXX add cache self.looptoken = LoopToken() oplist = [ResOperation(rop.CALL, bargs, bres, descr=calldescr), ResOperation(rop.FINISH, [bres], None, @@ -93,6 +92,26 @@ cache[self.res_type] = { tuple(self.args_type) : self.looptoken } self.setup_stack() + def gen_calldescr(self): + arg_classes = ''.join(self.args_type) + gccache = self.cpu.gc_ll_descr + + if self.res_type == 'i': + cls = SignedCallDescr + elif self.res_type == 'f': + cls = descr.FloatCallDescr + elif self.res_type == 'p': + cls = descr.NonGcPtrCallDescr + elif self.res_type == 'v': + cls = descr.VoidCallDescr + else: + raise NotImplementedError('Unknown type of descr: %s' + % self.res_type) + + calldescr = cls(arg_classes) + calldescr.create_call_stub(gccache.rtyper, self.res) + return calldescr + def call(self, push_result): res = self.cpu.execute_token(self.looptoken) @@ -126,3 +145,12 @@ def push_ref(self, value): self.cpu.set_future_value_ref(self.esp, value) self.esp += 1 + +# ____________________________________________________________ +# CallDescrs + +class SignedCallDescr(descr.BaseIntCallDescr): + _clsname = 'SignedCallDescr' + def get_result_size(self, translate_support_code): + return symbolic.get_size(lltype.Signed, translate_support_code) + From dan at codespeak.net Thu Jul 22 01:45:26 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Thu, 22 Jul 2010 01:45:26 +0200 (CEST) Subject: [pypy-svn] r76308 - in pypy/branch/micronumpy/pypy/module/micronumpy: . test Message-ID: <20100721234526.17009282BD4@codespeak.net> Author: dan Date: Thu Jul 22 01:45:25 2010 New Revision: 76308 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/array.py pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Log: Cleaning up and adding tests. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/array.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/array.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/array.py Thu Jul 22 01:45:25 2010 @@ -4,16 +4,21 @@ from pypy.interpreter.gateway import NoneNotWrapped from pypy.interpreter.gateway import interp2app -def iterable_type(space, xs): - raise NotImplementedError("Stub") - -def get_dtype(space, t): - raise NotImplementedError("Stub") - -def retrieve_dtype(space, t): - raise NotImplementedError("Stub") - -from pypy.rpython.lltypesystem import lltype +def stride_row(shape, i): + assert i >= 0 + stride = 1 + ndim = len(shape) + for s in shape[i + 1:]: + stride *= s + return stride + +def stride_column(shape, i): + i -= 1 + stride = 1 + while i >= 0: + stride *= shape[i] + i -= 1 + return stride def validate_index(array, space, w_i): index_dimensionality = space.int_w(space.len(w_i)) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Thu Jul 22 01:45:25 2010 @@ -14,6 +14,8 @@ from pypy.module.micronumpy.array import construct_array, infer_shape from pypy.module.micronumpy.dtype import null_data +from pypy.module.micronumpy.array import stride_row, stride_column + def size_from_shape(shape): size = 1 for dimension in shape: @@ -23,29 +25,11 @@ def index_w(space, w_index): return space.int_w(space.index(w_index)) -def stride_row(shape, i): - assert i >= 0 - stride = 1 - ndim = len(shape) - for s in shape[i + 1:]: - stride *= s - return stride - -def stride_column(shape, i): - i -= 1 - if i < 0: - return 1 - elif i == 0: - return shape[0] - else: - stride = 1 - for s in shape[:i]: - stride *= s - return stride - class MicroIter(Wrappable): + _immutable_fields_ = ['array', 'stride'] def __init__(self, array): self.array = array + self.stride = array.stride(0) self.i = 0 def descr_iter(self, space): @@ -53,14 +37,37 @@ descr_iter.unwrap_spec = ['self', ObjSpace] def descr_next(self, space): - if self.i < space.int_w(space.len(self.array)): - next = self.array.getitem(space, self.i) # FIXME: wrong for multi dimensional! (would be easy applevel) - self.i += 1 - return next + if self.i < self.array.opposite_stride(1): #will change for row/column + if len(self.array.shape) > 1: + ar = MicroArray(self.array.shape[1:], # ok for column major? + self.array.dtype, + self.array, + offset = self.array.offset + self.i * self.stride) + self.i += 1 + return space.wrap(ar) + elif len(self.array.shape) == 1: + next = self.array.getitem(space, self.i * self.stride) + self.i += 1 + return next + else: + raise OperationError(space.w_ValueError, + space.wrap("Something is horribly wrong with this array's shape. Has %d dimensions." % len(self.array.shape))) else: raise OperationError(space.w_StopIteration, space.wrap("")) descr_next.unwrap_spec = ['self', ObjSpace] + def __str__(self): + from pypy.rlib.rStringIO import RStringIO as StringIO + out = StringIO() + out.write('iterator(i=') + out.write(str(self.i)) + out.write(', array=') + out.write(repr(self.array)) + out.write(')') + result = out.getvalue() + out.close() + return result + MicroIter.typedef = TypeDef('iterator', __iter__ = interp2app(MicroIter.descr_iter), next = interp2app(MicroIter.descr_next), @@ -116,27 +123,29 @@ for i in range(len(index)): if index[i] < 0: index[i] = self.shape[i] + index[i] - return self.flatten_index(space, index) + return self.flatten_index(index) - def create_flatten_index(stride_function): - def flatten_index(self, index): - offset = 0 - for i in range(len(index)): - stride = stride_function(self.shape, i) - offset += index[i] * stride - return offset - return flatten_index - - flatten_index_r = create_flatten_index(stride_row) - flatten_index_c = create_flatten_index(stride_column) - - # FIXME: when different types are supported - # this function will change - def flatten_index(self, space, index): + def flatten_index(self, index): + offset = 0 + for i in range(len(index)): + stride = self.stride(i) + offset += index[i] * stride + return offset + + def stride(self, i): if self.order == 'C': - return self.flatten_index_r(index) # row order for C + return stride_row(self.shape, i) # row order for C elif self.order == 'F': - return self.flatten_index_c(index) # + return stride_column(self.shape, i) # + else: + raise OperationError(space.w_NotImplementedError, + space.wrap("Unknown order: '%s'" % self.order)) + + def opposite_stride(self, i): + if self.order == 'C': # C for C not Column, but this is opposite + return stride_column(self.shape, i) + elif self.order == 'F': + return stride_row(self.shape, i) else: raise OperationError(space.w_NotImplementedError, space.wrap("Unknown order: '%s'" % self.order)) @@ -204,6 +213,11 @@ return app_descr_repr(space, space.wrap(self)) descr_repr.unwrap_spec = ['self', ObjSpace] + # FIXME: Can I use app_descr_str directly somehow? + def descr_str(self, space): + return app_descr_str(space, space.wrap(self)) + descr_str.unwrap_spec = ['self', ObjSpace] + def descr_iter(self, space): return space.wrap(MicroIter(self)) descr_iter.unwrap_spec = ['self', ObjSpace] @@ -217,13 +231,40 @@ app_formatting = gateway.applevel(""" from StringIO import StringIO - def stringify(out, array): + def stringify(out, array, prefix='', commas=False, first=False, last=False, depth=0): + if depth > 0 and not first: + out.write('\\n') + out.write(prefix) # indenting for array( + out.write(' ' * depth) + out.write("[") if len(array.shape) > 1: - out.write(',\\n'.join([str(x) for x in array])) + i = 1 + for x in array: + if i == 1: + stringify(out, x, + first=True, commas=commas, prefix=prefix, depth=depth + 1) + elif i >= len(array): + stringify(out, x, + last=True, commas=commas, prefix=prefix, depth=depth + 1) + else: + stringify(out, x, + commas=commas, prefix=prefix, depth=depth + 1) + + i += 1 + out.write("]") else: - out.write(', '.join([str(x) for x in array])) - out.write("]") + if commas: + separator = ', ' + else: + separator = ' ' + + out.write(separator.join([str(x) for x in array])) + + if commas and not last: + out.write("],") + else: + out.write("]") def descr_str(self): out = StringIO() @@ -235,7 +276,7 @@ def descr_repr(self): out = StringIO() out.write("array(") - stringify(out, self) + stringify(out, self, commas=True, prefix=' ') out.write(")") result = out.getvalue() out.close() @@ -243,6 +284,7 @@ """) app_descr_repr = app_formatting.interphook('descr_repr') +app_descr_str = app_formatting.interphook('descr_str') # Getters, strange GetSetProperty behavior # forced them out of the class @@ -273,6 +315,7 @@ __setitem__ = interp2app(MicroArray.descr_setitem), __len__ = interp2app(MicroArray.descr_len), __repr__ = interp2app(MicroArray.descr_repr), + __str__ = interp2app(MicroArray.descr_str), __iter__ = interp2app(MicroArray.descr_iter), ) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py Thu Jul 22 01:45:25 2010 @@ -5,10 +5,6 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable from pypy.rlib.debug import make_sure_not_resized -from pypy.module.micronumpy.sdarray import sdresult - -from pypy.module.micronumpy.mdarray import mdresult - def unpack_shape(space, w_shape): if space.is_true(space.isinstance(w_shape, space.w_int)): return [space.int_w(w_shape)] @@ -18,82 +14,17 @@ def infer_shape(space, w_values): return [space.int_w(space.len(w_values))] #TODO: handle multi-dimensional arrays... -class ndarray(Wrappable): - def __init__(self, space, w_values, w_shape=NoneNotWrapped, w_dtype=NoneNotWrapped): - self.array = None - self.space = space - if w_dtype is None and not space.is_w(w_values, space.w_None): - #TODO: infer type from w_values (better than this) - w_dtype = space.type(space.fixedview(w_values)[0]) #FIXME: Allocates an entire array and throws it away! - - self.dtype = w_dtype - shape_w = None - if w_shape is None: - try: - shape_w = infer_shape(space, w_values) - except OperationError, e: - if e.match(space, space.w_TypeError): pass - else: raise - else: - shape_w = unpack_shape(space, w_shape) - - if not shape_w is None and not w_dtype is None: - try: - if len(shape_w) == 1: - length = shape_w[0] - self.array = sdresult(space, w_dtype)(space, length) - else: - self.array = mdresult(space, w_dtype)(space, shape_w) - except KeyError, e: - raise OperationError(space.w_NotImplementedError, - space.wrap("Haven't implemented generic array yet!")) - - if not w_values is None and not space.is_w(w_values, space.w_None): - self.array.load_iterable(w_values) #TODO: implement loading for multi-dimensional arrays - - def validate_index(self, space, w_i): - try: - index_dimensionality = space.int_w(space.len(w_i)) - array_dimensionality = len(self.array.shape) - if index_dimensionality > array_dimensionality: - raise OperationError(space.w_IndexError, - space.wrap("Index dimensionality (%d) greater than array dimensionality (%d)." % (index_dimensionality, array_dimensionality))) - except OperationError, e: - if e.match(space, space.w_TypeError): pass - else: raise - - # Math Operations - descr_mul = create_math_operation(mul_scalar) - descr_div = create_math_operation(div_scalar) - descr_add = create_math_operation(add_scalar) - descr_sub = create_math_operation(sub_scalar) - - def descr_iter(self): - space = self.space - return space.wrap(ArrayIter(space, self, 0)) - descr_iter.unwrap_spec = ['self'] +def validate_index(self, space, w_i): + try: + index_dimensionality = space.int_w(space.len(w_i)) + array_dimensionality = len(self.array.shape) + if index_dimensionality > array_dimensionality: + raise OperationError(space.w_IndexError, + space.wrap("Index dimensionality (%d) greater than array dimensionality (%d)." % (index_dimensionality, array_dimensionality))) + except OperationError, e: + if e.match(space, space.w_TypeError): pass + else: raise - def descr_getitem(self, space, w_i): - self.validate_index(space, w_i) - return self.array.getitem(w_i) - descr_getitem.unwrap_spec = ['self', ObjSpace, W_Root] - - def descr_setitem(self, space, w_i, w_x): - self.validate_index(space, w_i) - self.array.setitem(w_i, w_x) - descr_setitem.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] - - def descr_len(self, space): - return self.array.len() - descr_len.unwrap_spec = ['self', ObjSpace] - - def descr_str(self, space): - return space.wrap("[%s]" % self.array.str()) - descr_str.unwrap_spec = ['self', ObjSpace] - - def descr_repr(self, space): - return space.wrap("array([%s])" % self.array.str()) - descr_repr.unwrap_spec = ['self', ObjSpace] def descr_new(space, w_cls, w_shape, w_buffer=NoneNotWrapped, w_offset=NoneNotWrapped, @@ -103,27 +34,3 @@ descr_new.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root, W_Root, W_Root, str] - -ndarray.typedef = TypeDef( - 'ndarray', - #__init__ = interp2app(descr_init), #FIXME - __new__ = interp2app(descr_new), - __iter__ = interp2app(ndarray.descr_iter), - __mul__ = interp2app(ndarray.descr_mul), - __div__ = interp2app(ndarray.descr_div), - __add__ = interp2app(ndarray.descr_add), - __sub__ = interp2app(ndarray.descr_sub), - __str__ = interp2app(ndarray.descr_str), - __repr__ = interp2app(ndarray.descr_repr), - __getitem__ = interp2app(ndarray.descr_getitem), - __setitem__ = interp2app(ndarray.descr_setitem), - __len__ = interp2app(ndarray.descr_len), -) - -def array(space, w_values, w_shape=NoneNotWrapped, w_dtype=NoneNotWrapped): - return ndarray(space, w_values, w_shape, w_dtype) -array.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root] - -def zeros(space, w_shape, w_dtype=NoneNotWrapped): - return ndarray(space, None, w_shape, w_dtype) -zeros.unwrap_spec = [ObjSpace, W_Root, W_Root] Modified: pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Thu Jul 22 01:45:25 2010 @@ -411,6 +411,14 @@ assert stride_column(shape, 0) == 1 assert stride_column(shape, 1) == 2 + shape = (5, 4, 3) + assert stride_row(shape, 0) == 12 + assert stride_row(shape, 1) == 3 + assert stride_row(shape, 2) == 1 + + assert stride_column(shape, 0) == 1 + assert stride_column(shape, 1) == 5 + assert stride_column(shape, 2) == 20 # TODO: more, N-dimensional too def test_memory_layout(self, space): From magcius at codespeak.net Thu Jul 22 02:36:41 2010 From: magcius at codespeak.net (magcius at codespeak.net) Date: Thu, 22 Jul 2010 02:36:41 +0200 (CEST) Subject: [pypy-svn] r76309 - in pypy/branch/avm/pypy/translator: avm1/test avm2 avm2/intrinsic avm2/test Message-ID: <20100722003641.9354F282BD4@codespeak.net> Author: magcius Date: Thu Jul 22 02:36:39 2010 New Revision: 76309 Removed: pypy/branch/avm/pypy/translator/avm2/intrinsic/ Modified: pypy/branch/avm/pypy/translator/avm1/test/harness.py pypy/branch/avm/pypy/translator/avm2/__init__.py pypy/branch/avm/pypy/translator/avm2/class_.py pypy/branch/avm/pypy/translator/avm2/conftest.py pypy/branch/avm/pypy/translator/avm2/constant.py pypy/branch/avm/pypy/translator/avm2/database.py pypy/branch/avm/pypy/translator/avm2/entrypoint.py pypy/branch/avm/pypy/translator/avm2/function.py pypy/branch/avm/pypy/translator/avm2/genavm.py pypy/branch/avm/pypy/translator/avm2/library.py pypy/branch/avm/pypy/translator/avm2/metavm.py pypy/branch/avm/pypy/translator/avm2/opcodes.py pypy/branch/avm/pypy/translator/avm2/record.py pypy/branch/avm/pypy/translator/avm2/runtime.py pypy/branch/avm/pypy/translator/avm2/test/browsertest.py pypy/branch/avm/pypy/translator/avm2/test/entrypoint.py pypy/branch/avm/pypy/translator/avm2/test/runtest.py pypy/branch/avm/pypy/translator/avm2/test/test_int.py pypy/branch/avm/pypy/translator/avm2/test/test_string.py pypy/branch/avm/pypy/translator/avm2/types_.py Log: Fix a lot more testcases Modified: pypy/branch/avm/pypy/translator/avm1/test/harness.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm1/test/harness.py (original) +++ pypy/branch/avm/pypy/translator/avm1/test/harness.py Thu Jul 22 02:36:39 2010 @@ -1,20 +1,23 @@ -from pypy.translator.avm1.test import browsertest as b -from mech.fusion.swf import swfdata as s, tags as t, records as r -from mech.fusion import avm1 as a +from pypy.translator.avm1.test.browsertest import browsertest +from mech.fusion.swf.swfdata import SwfData +from mech.fusion.swf.records import Rect, RGBA +from mech.fusion.swf import tags +from mech.fusion.avm1.avm1gen import AVM1Gen +from mech.fusion.avm1.actions import ActionGetURL2 class TestHarness(object): def __init__(self, name): self.testname = name - self.swf = s.SwfData() - self.swf.add_tag(t.SetBackgroundColor(0x333333)) - self.swf.add_tag(t.DefineEditText(r.Rect(0, 0, 0, 0), "txt", - "Testing %s." % (name,), color=r.RGBA(0xFFFFFF))) - self.swf.add_tag(t.PlaceObject2(1, 2)) - self.actions = g.AVM1Gen(t.DoAction()) + self.swf = SwfData() + self.swf.add_tag(tags.SetBackgroundColor(0x333333)) + self.swf.add_tag(tags.DefineEditText(Rect(0, 0, 0, 0), "txt", + "Testing %s." % (name,), color=RGBA(0xFFFFFF))) + self.swf.add_tag(tags.PlaceObject2(1, 1)) + self.actions = AVM1Gen(tags.DoAction()) self.swf.add_tag(self.actions.block) - self.swf.add_tag(t.ShowFrame()) - self.swf.add_tag(t.End()) + self.swf.add_tag(tags.ShowFrame()) + self.swf.add_tag(tags.End()) self.start_test() def print_text(self, text): @@ -54,9 +57,9 @@ self.actions.set_variable() self.print_var("Got: ", "result") self.actions.push_const("/test.result", "") # URL, target - self.actions.action(a.ActionGetURL2("POST", True, True)) + self.actions.action(ActionGetURL2("POST", True, True)) self.actions.push_const("javascript:window.close()", "") # Close the window. - self.actions.action(a.ActionGetURL2("")) + self.actions.action(ActionGetURL2("")) def do_test(self, debug=False): self.finish_test() @@ -65,4 +68,4 @@ f = open("test.swf", "w") f.write(self.swf.serialize()) f.close() - return b.browsertest(self.testname, self.swf) + return browsertest(self.testname, self.swf) Modified: pypy/branch/avm/pypy/translator/avm2/__init__.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/__init__.py (original) +++ pypy/branch/avm/pypy/translator/avm2/__init__.py Thu Jul 22 02:36:39 2010 @@ -1 +0,0 @@ - Modified: pypy/branch/avm/pypy/translator/avm2/class_.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/class_.py (original) +++ pypy/branch/avm/pypy/translator/avm2/class_.py Thu Jul 22 02:36:39 2010 @@ -1,56 +1,63 @@ + +from py.builtin import set + from pypy.rpython.ootypesystem import ootype -from pypy.translator.cli.node import Node +from pypy.translator.avm2.node import ClassNodeBase +from pypy.translator.avm2.types_ import types from pypy.translator.oosupport.constant import push_constant -from mech.fusion.avm2 import constants, traits -from pypy.translator.avm2 import types_ as types - -try: - set -except NameError: - from sets import Set as set +from mech.fusion.avm2.constants import QName, packagedQName +from mech.fusion.avm2.interfaces import IMultiname -class Class(Node): +class Class(ClassNodeBase): def __init__(self, db, INSTANCE, namespace, name): self.db = db self.cts = db.genoo.TypeSystem(db) self.INSTANCE = INSTANCE + print INSTANCE, INSTANCE._superclass self.exception = ootype.isSubclass(self.INSTANCE, self.db.genoo.EXCEPTION) self.namespace = namespace self.name = name - def dependencies(self): - if not self.is_root(self.INSTANCE): - self.db.pending_class(self.INSTANCE._superclass) - def __hash__(self): return hash(self.INSTANCE) def __eq__(self, other): - return self.INSTANCE == other.INSTANCE + return isinstance(other, type(self)) and self.INSTANCE == other.INSTANCE def __ne__(self, other): return not self == other - def is_root(INSTANCE): - return INSTANCE._superclass is None - is_root = staticmethod(is_root) + def __repr__(self): + return '' % self.name + + def dependencies(self): + if self.INSTANCE._superclass._superclass: + self.db.pending_class(self.INSTANCE._superclass) + + def get_fields(self): + return self.INSTANCE._fields.iteritems() def get_name(self): return self.name - def __repr__(self): - return '' % self.name + def get_full_name(self): + if self.namespace is None: + return self.name + else: + return '%s::%s' % (self.namespace, self.name) + + def get_type(self): + return packagedQName(self.namespace, self.name) def get_base_class(self): base_class = self.INSTANCE._superclass if self.INSTANCE is self.db.genoo.EXCEPTION: - return constants.QName("Error") - if self.is_root(base_class): - return constants.QName("Object") - else: + return QName("Error") + elif self.INSTANCE._superclass._superclass: ns, name = self.db.class_name(base_class).rsplit('::', 1) - return constants.packagedQName(ns, name) + return packagedQName(ns, name) + return QName("Object") def is_abstract(self): return False # XXX @@ -75,22 +82,25 @@ return False - def render(self, ilasm): - if self.is_root(self.INSTANCE): - return - - self.ilasm = ilasm - - ilasm.begin_class(constants.packagedQName(self.namespace, self.name), self.get_base_class()) - for f_name, (f_type, f_default) in self.INSTANCE._fields.iteritems(): - cts_type = self.cts.lltype_to_cts(f_type) + def render_ctor(self, ilasm): + ilasm.begin_constructor() + # set default values for fields + default_values = self.INSTANCE._fields.copy() + default_values.update(self.INSTANCE._overridden_defaults) + for f_name, (F_TYPE, f_default) in default_values.iteritems(): + if getattr(F_TYPE, '_is_value_type', False): + continue # we can't set it to null + # INSTANCE_DEF, _ = self.INSTANCE._lookup_field(f_name) + cts_type = self.cts.lltype_to_cts(F_TYPE) f_name = self.cts.escape_name(f_name) - if cts_type != types.types.void: - ilasm.context.add_instance_trait(traits.AbcSlotTrait(constants.QName(f_name), cts_type.multiname())) - - self._ctor() - self._toString() + if cts_type != types.void: + ilasm.push_this() + push_constant(self.db, F_TYPE, f_default, ilasm) + # class_name = self.db.class_name(INSTANCE_DEF) + ilasm.set_field(f_name) + ilasm.end_constructor() + def render_methods(self, ilasm): for m_name, m_meth in self.INSTANCE._methods.iteritems(): if hasattr(m_meth, 'graph'): # if the first argument's type is not a supertype of @@ -119,40 +129,25 @@ if ARG is not ootype.Void] returntype = self.cts.lltype_to_cts(METH.RESULT) ilasm.begin_method(m_name, arglist, returntype) - ilasm.emit('findpropstrict', constants.QName("Error")) - ilasm.push_const("Abstract method %s::%s called" % (self.name, m_name)) - ilasm.emit('constructprop', constants.QName("Error"), 1) + ilasm.emit('findpropstrict', QName("Error")) + ilasm.load("Abstract method %s::%s called" % (self.name, m_name)) + ilasm.emit('constructprop', QName("Error"), 1) ilasm.throw() ilasm.exit_context() - ilasm.exit_context() - - def _ctor(self): - self.ilasm.begin_constructor() - # set default values for fields - default_values = self.INSTANCE._fields.copy() - default_values.update(self.INSTANCE._overridden_defaults) - for f_name, (F_TYPE, f_default) in default_values.iteritems(): - if getattr(F_TYPE, '_is_value_type', False): - continue # we can't set it to null - # INSTANCE_DEF, _ = self.INSTANCE._lookup_field(f_name) - cts_type = self.cts.lltype_to_cts(F_TYPE) - f_name = self.cts.escape_name(f_name) - if cts_type != types.types.void: - self.ilasm.push_this() - push_constant(self.db, F_TYPE, f_default, self.gen) - # class_name = self.db.class_name(INSTANCE_DEF) - self.ilasm.set_field(f_name) - self.ilasm.end_constructor() + self.render_getName(ilasm) - def _toString(self): - if self.is_root(self.INSTANCE._superclass): - override = False - else: - override = True - print self.exception + def render_toString(self, ilasm): + override = self.INSTANCE._superclass._superclass is not None wrapper = "Exception" if self.exception else "Instance" - self.ilasm.begin_method('toString', [], types.types.string, override=override) - self.ilasm.load("%sWrapper('%s')" % (wrapper, self.name)) - self.ilasm.return_value() - self.ilasm.end_method() + ilasm.begin_method('toString', [], types.string, override=override) + ilasm.load("%sWrapper('%s')" % (wrapper, self.name)) + ilasm.return_value() + ilasm.end_method() + + def render_getName(self, ilasm): + override = self.INSTANCE._superclass._superclass is not None + ilasm.begin_method('getName', [], types.string, override=override) + ilasm.load(self.name) + ilasm.return_value() + ilasm.end_method() Modified: pypy/branch/avm/pypy/translator/avm2/conftest.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/conftest.py (original) +++ pypy/branch/avm/pypy/translator/avm2/conftest.py Thu Jul 22 02:36:39 2010 @@ -1,8 +1,8 @@ def pytest_addoption(parser): group = parser.getgroup("pypy-tamarin options") - group.addoption('--swf', action="store_const", const="swf", dest="tamtarget", default="swf", + group.addoption('--swf', action="store_true", dest="browsertest", default=False, help="generate a .swf and .abc and use a browsertest and Flash Player to run") - group.addoption('--tamarin', action="store_const", const="tamarin", dest="tamtarget", + group.addoption('--tamarin', action="store", dest="tamexec", default="avmshell", help="generate an abc that uses Tamarin") group.addoption('--no-mf-optimize', action="store_false", default=True, dest="mf_optim", help="don't do simple MF optimizations") Modified: pypy/branch/avm/pypy/translator/avm2/constant.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/constant.py (original) +++ pypy/branch/avm/pypy/translator/avm2/constant.py Thu Jul 22 02:36:39 2010 @@ -7,9 +7,14 @@ from pypy.translator.avm2 import types_ as types from pypy.rpython.lltypesystem import lltype -from mech.fusion.avm2 import constants, traits +from mech.fusion.avm2.traits import AbcConstTrait +from mech.fusion.avm2.constants import QName, packagedQName +from mech.fusion.avm2.interfaces import IMultiname -CONST_CLASS = constants.packagedQName("pypy.runtime", "Constants") +from zope.interface import implementer +from zope.component import adapter, provideAdapter + +CONST_CLASS = packagedQName("pypy.runtime", "Constants") # ______________________________________________________________________ # Constant Generators @@ -41,11 +46,11 @@ self.ilasm.exit_context() def _declare_const(self, gen, const): - self.ctx.add_static_trait(traits.AbcConstTrait(constants.QName(const.name), const.get_type().multiname())) + self.ctx.add_static_trait(AbcConstTrait(IMultiname(const.name), IMultiname(const))) def downcast_constant(self, gen, const, EXPECTED_TYPE): type = self.cts.lltype_to_cts(EXPECTED_TYPE) - gen.emit('coerce', type.multiname()) + gen.emit('coerce', IMultiname(type)) def _get_key_for_const(self, value): if isinstance(value, ootype._view) and isinstance(value._inst, ootype._record): @@ -54,22 +59,23 @@ def push_constant(self, gen, const): gen.emit('getlex', CONST_CLASS) - gen.emit('getproperty', constants.QName(const.name)) + gen.emit('getproperty', IMultiname(const.name)) def _push_constant_during_init(self, gen, const): gen.push_this() - gen.emit('getproperty', constants.QName(const.name)) + gen.emit('getproperty', IMultiname(const.name)) def _pre_store_constant(self, gen, const): gen.push_this() def _store_constant(self, gen, const): - gen.emit('initproperty', constants.QName(const.name)) + gen.emit('initproperty', IMultiname(const.name)) def _initialize_data(self, gen, all_constants): """ Iterates through each constant, initializing its data. """ for const in all_constants: - self._consider_step(gen) + if const._do_not_initialize(): + continue self._push_constant_during_init(gen, const) self.current_const = const if not const.initialize_data(self, gen): @@ -113,6 +119,13 @@ assert self.is_null() gen.ilasm.push_null() + at adapter(Avm2BaseConstMixin) + at implementer(IMultiname) +def _adapter(self): + return IMultiname(self.get_type()) + +provideAdapter(_adapter) + # class Avm2DictMixin(Avm2BaseConstMixin): # def _check_for_void_dict(self, gen): # KEYTYPE = self.value._TYPE._KEYTYPE @@ -153,9 +166,8 @@ # the generator interface in Tamarin. class Avm2RecordConst(Avm2BaseConstMixin, RecordConst): - def create_pointer(self, gen): - self.db.const_count.inc('Record') - super(Avm2RecordConst, self).create_pointer(gen) + def _do_not_initialize(self): + return False def initialize_data(self, constgen, gen): assert not self.is_null() @@ -168,10 +180,8 @@ gen.set_field(f_name) class Avm2InstanceConst(Avm2BaseConstMixin, InstanceConst): - def create_pointer(self, gen): - self.db.const_count.inc('Instance') - self.db.const_count.inc('Instance', self.OOTYPE()) - super(Avm2InstanceConst, self).create_pointer(gen) + def _do_not_initialize(self): + return not self.value._TYPE._fields def initialize_data(self, constgen, gen): assert not self.is_null() @@ -198,49 +208,52 @@ def is_inline(self): return True + def _do_not_initialize(self): + return True + def push_inline(self, gen, EXPECTED_TYPE): if not self.is_null(): - if hasattr(self.value, '_FUNC'): - FUNC = self.value._FUNC - classname = self.db.record_delegate(FUNC) - else: - INSTANCE = self.value._INSTANCE - classname = self.db.class_name(INSTANCE) - ns, name = classname.rsplit('::', 1) - gen.emit('getlex', constants.packagedQName(ns, name)) - return - super(Avm2ClassConst, self).push_inline(gen, EXPECTED_TYPE) + INSTANCE = self.value._INSTANCE + classname = self.cts.instance_to_qname(INSTANCE) + return gen.load(classname) +class Avm2ArrayListConst(Avm2BaseConstMixin): + def get_value(self): + pass -class Avm2ArrayListConst(Avm2BaseConstMixin, ListConst): + def get_length(self): + return len(self.get_value()) def _do_not_initialize(self): # Check if it is a list of all zeroes: try: - if self.value._list == [0] * len(self.value._list): - return True + return self.get_value() == [0] * self.get_length() except: - pass - return super(Avm2ListConst, self)._do_not_initialize() - + return False + def create_pointer(self, gen): - llen = len(self.value._list) - self.db.const_count.inc('List') - self.db.const_count.inc('List', self.value._TYPE.ITEM) - self.db.const_count.inc('List', llen) - gen.oonewarray(self.value._TYPE, llen) + gen.oonewarray(self.value._TYPE, self.get_length()) def initialize_data(self, constgen, gen): assert not self.is_null() - + ITEM = self.value._TYPE.ITEM + # check for special cases and avoid initialization if self._do_not_initialize(): return - - for idx, item in enumerate(self.value._array): + + for idx, item in enumerate(self.get_value()): gen.dup() - gen.emit('setproperty', constants.Multiname( - str(idx), constants.PROP_NAMESPACE_SET)) + push_constant(self.db, ITEM, item, gen) + gen.set_field(idx) + +class Avm2ListConst (Avm2ArrayListConst, ListConst): + def get_value(self): + return self.value._list + +class Avm2ArrayConst(Avm2ArrayListConst, ArrayConst): + def get_value(self): + return self.value._array # class CLIDictConst(CLIDictMixin, DictConst): # def create_pointer(self, gen): Modified: pypy/branch/avm/pypy/translator/avm2/database.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/database.py (original) +++ pypy/branch/avm/pypy/translator/avm2/database.py Thu Jul 22 02:36:39 2010 @@ -1,7 +1,6 @@ import string -#from pypy.translator.avm.class_ import Class from pypy.rpython.ootypesystem import ootype -from pypy.translator.avm2 import runtime, types_ as types, class_ as c, record +from pypy.translator.avm2 import types_ as types, class_ as c, record from pypy.translator.oosupport.database import Database as OODatabase try: @@ -30,7 +29,7 @@ def __init__(self, genoo): OODatabase.__init__(self, genoo) self._pending_nodes = [] - self.classes = {} # INSTANCE --> class_name + self.classes = {} # INSTANCE --> class nodes self.classnames = set() # (namespace, name) self.functions = {} # graph --> function_name self.methods = {} # graph --> method_name @@ -82,24 +81,20 @@ def pending_class(self, INSTANCE): try: - return self.classes[INSTANCE] + return self.pending_node(self.classes[INSTANCE]).get_full_name() except KeyError: pass - - if isinstance(INSTANCE, runtime.NativeInstance): + + if 0: #isinstance(INSTANCE, runtime.NativeInstance): self.classes[INSTANCE] = INSTANCE._name return INSTANCE._name else: namespace, name = self._default_class_name(INSTANCE) name = self.get_unique_class_name(namespace, name) - if namespace is None: - full_name = name - else: - full_name = '%s::%s' % (namespace, name) - self.classes[INSTANCE] = full_name cls = c.Class(self, INSTANCE, namespace, name) self.pending_node(cls) - return full_name + self.classes[INSTANCE] = cls + return cls.get_full_name() def record_function(self, graph, name): self.functions[graph] = name @@ -124,23 +119,28 @@ NATIVE_INSTANCE = INSTANCE._hints['NATIVE_INSTANCE'] return NATIVE_INSTANCE._name except KeyError: - return self.classes[INSTANCE] + return self.classes[INSTANCE].get_full_name() - def record_delegate(self, TYPE): - try: - return self.delegates[TYPE] - except KeyError: - name = 'StaticMethod__%d' % len(self.delegates) - self.delegates[TYPE] = name - self.pending_node(Delegate(self, TYPE, name)) - return name - + ## def record_delegate(self, TYPE): + ## try: + ## return self.delegates[TYPE] + ## except KeyError: + ## name = 'StaticMethod__%d' % len(self.delegates) + ## self.delegates[TYPE] = name + ## self.pending_node(Delegate(self, TYPE, name)) + ## return name + def pending_node(self, node): """ Adds a node to the worklist, so long as it is not already there and has not already been rendered. """ - assert not self.locked # sanity check - if node in self._pending_nodes or node in self._rendered_nodes: - return + # assert not self.locked # sanity check + if node in self._rendered_nodes: + return node + # sometimes a dependency will already + # be there, but needs to be rendered + # before others + if node in self._pending_nodes: + self._pending_nodes.remove(node) self._pending_nodes.append(node) node.dependencies() - + return node Modified: pypy/branch/avm/pypy/translator/avm2/entrypoint.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/entrypoint.py (original) +++ pypy/branch/avm/pypy/translator/avm2/entrypoint.py Thu Jul 22 02:36:39 2010 @@ -1,7 +1,7 @@ from pypy.translator.avm2.types_ import Avm2TypeSystem from pypy.translator.avm2.database import LowLevelDatabase -from pypy.translator.cli.node import Node +from pypy.translator.avm2.node import Node from pypy.rpython.ootypesystem import ootype def get_entrypoint(graph): @@ -16,7 +16,9 @@ return TestEntryPoint(graph) class BaseEntryPoint(Node): - + def create_assembler(self, gen): + pass + def set_db(self, db): self.db = db self.cts = Avm2TypeSystem(db) @@ -28,28 +30,23 @@ """ def __init__(self, graph_to_call): - self.graph = graph_to_call + self._graph = graph_to_call def get_name(self): return 'main' def render(self, ilasm): try: - ARG0 = self.graph.getargs()[0].concretetype + ARG0 = self._graph.getargs()[0].concretetype except IndexError: ARG0 = None assert isinstance(ARG0, ootype.List) and ARG0.ITEM is ootype.String,\ 'Wrong entry point signature: List(String) expected' - qname = self.cts.graph_to_qname(self.graph) + qname = self.cts.graph_to_qname(self._graph) ilasm.emit('findpropstrict', qname) ilasm.emit('callpropvoid', qname, 0) - ## ilasm.opcode('pop') # XXX: return this value, if it's an int32 - - ## ilasm.call('void [pypylib]pypy.runtime.DebugPrint::close_file()') - ## ilasm.opcode('ret') - ## ilasm.end_function() - self.db.pending_function(self.graph) + self.db.pending_function(self._graph) class LibraryEntryPoint(BaseEntryPoint): def __init__(self, name, graphs): Modified: pypy/branch/avm/pypy/translator/avm2/function.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/function.py (original) +++ pypy/branch/avm/pypy/translator/avm2/function.py Thu Jul 22 02:36:39 2010 @@ -1,17 +1,25 @@ -from functools import partial +from itertools import chain + +from py.builtin import set from pypy.objspace.flow import model as flowmodel from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.lltype import Void from pypy.translator.oosupport.function import Function as OOFunction -from pypy.translator.cli.node import Node + +from pypy.translator.avm2.node import Node from mech.fusion.avm2 import constants -class Function(OOFunction, Node): +class CompiledABCNode(Node): + def __init__(self, abc): + self.abc = abc - auto_propagate_exceptions = True + def render(self, ilasm): + ilasm.abc.merge(self.abc) +class Function(OOFunction, Node): + auto_propagate_exceptions = True def __init__(self, db, graph, name=None, is_method=False, is_entrypoint=False): OOFunction.__init__(self, db, graph, name, is_method, is_entrypoint) @@ -58,12 +66,12 @@ self.generator.begin_method(self.name, self.args, returntype, static=not self.is_method, override=self.override) - self.declare_locals() + # self.declare_locals() def end_render(self): + self.generator.end_method() if self.classname: - self.generator.exit_context() - self.generator.exit_context() + self.generator.end_class() def render_return_block(self, block): return_var = block.inputargs[0] @@ -76,13 +84,39 @@ def set_label(self, label): return self.generator.set_label(label) - def declare_locals(self): - for TYPE, name in set(self.locals): - TYPE.load_default(self.generator) - self.generator.store_var(name) + def get_block_locals(self, block, exits=False): + if not block.operations: + return set() + all_locals = set(arg.name for op in block.operations for arg in op.args if isinstance(arg, flowmodel.Variable)) + all_locals |= set(op.result.name for op in block.operations) + all_locals -= set(arg.name for arg in self.graph.getargs() if isinstance(arg, flowmodel.Variable)) + if exits: + all_locals -= set(arg.name for exit in block.exits + for arg in exit.args + exit.target.inputargs + if isinstance(arg, flowmodel.Variable)) + if isinstance(block.exitswitch, flowmodel.Variable) and block.exitswitch.name in all_locals: + all_locals.remove(block.exitswitch.name) + return all_locals + + def get_block_links_args(self, link): + return set(arg.name for exit in link.prevblock.exits for arg in exit.args if arg not in link.args if isinstance(arg, flowmodel.Variable)) + + def end_block(self, block): + L = self.get_block_locals(block, True) + for var in self.get_block_locals(block, True): + if self.generator.HL(var): + self.generator.KL(var) + + def __eq__(self, other): + return type(self) == type(other) and self.graph == other.graph + + ## def declare_locals(self): + ## for TYPE, name in set(self.locals): + ## TYPE.load_default(self.generator) + ## self.generator.store_var(name) def _trace_enabled(self): - return True + return False def _trace(self, s, writeline=False): print "TRACE:", s @@ -94,33 +128,36 @@ print "Rendering op:", op super(Function, self)._render_op(op) - ## def _setup_link(self, link): - ## target = link.target - ## linkvars = [] - ## for to_load, to_store in zip(link.args, target.inputargs): - ## if isinstance(to_load, flowmodel.Variable) and to_load.name == to_store.name: - ## continue - ## if to_load.concretetype is ootype.Void: - ## continue - ## linkvars.append((to_load, to_store)) - - ## # after SSI_to_SSA it can happen to have to_load = [a, b] and - ## # to_store = [b, c]. If we store each variable sequentially, - ## # 'b' would be overwritten before being read. To solve, we - ## # first load all the values on the stack, then store in the - ## # appropriate places. - - ## if self._trace_enabled(): - ## self._trace('link', writeline=True) - ## for to_load, to_store in linkvars: - ## self._trace_value('%s <-- %s' % (to_store, to_load), to_load) - ## self._trace('', writeline=True) + def _setup_link(self, link): + target = link.target + linkvars = [] + locals = self.get_block_locals(link.prevblock) | self.get_block_links_args(link) + for to_load, to_store in zip(link.args, target.inputargs): + if isinstance(to_load, flowmodel.Variable) and to_load.name == to_store.name: + continue + if to_load.concretetype is ootype.Void: + continue + linkvars.append((to_load, to_store)) + + # after SSI_to_SSA it can happen to have to_load = [a, b] and + # to_store = [b, c]. If we store each variable sequentially, + # 'b' would be overwritten before being read. To solve, we + # first load all the values on the stack, then store in the + # appropriate places. + + if self._trace_enabled(): + self._trace('link', writeline=True) + for to_load, to_store in linkvars: + self._trace_value('%s <-- %s' % (to_store, to_load), to_load) + self._trace('', writeline=True) + + for to_load, to_store in linkvars: + self.generator.load(to_load) + if isinstance(to_load, flowmodel.Variable) and to_load.name in locals: + self.generator.KL(to_load.name) - ## for to_load, to_store in linkvars: - ## self.generator.load(to_load) - - ## for to_load, to_store in reversed(linkvars): - ## self.generator.store(to_store) + for to_load, to_store in reversed(linkvars): + self.generator.store(to_store) def begin_try(self, cond): if cond: @@ -134,7 +171,7 @@ def begin_catch(self, llexitcase): ll_meta_exc = llexitcase ll_exc = ll_meta_exc._INSTANCE - self.ilasm.begin_catch(ll_exc) + self.ilasm.begin_catch(self.cts.instance_to_qname(ll_exc)) def end_catch(self, target_label): self.ilasm.end_catch() @@ -150,20 +187,67 @@ # the exception value is on the stack, use it as the 2nd target arg assert len(link.args) == 2 assert len(link.target.inputargs) == 2 - self.store(link.target.inputargs[1]) + self.generator.store(link.target.inputargs[1]) else: # the exception value is on the stack, store it in the proper place if isinstance(link.last_exception, flowmodel.Variable): - self.ilasm.opcode('dup') - self.store(link.last_exc_value) - self.ilasm.emit('convert_o') - self.ilasm.get_field('prototype') - self.ilasm.get_field('constructor') - self.store(link.last_exception) + self.generator.emit('dup') + self.generator.store(link.last_exc_value) + self.generator.emit('convert_o') + self.generator.get_field('prototype') + self.generator.get_field('constructor') + self.generator.store(link.last_exception) else: - self.store(link.last_exc_value) + self.generator.store(link.last_exc_value) self._setup_link(link) + def render_normal_block(self, block): + for op in block.operations: + self._render_op(op) + + self.end_block(block) + + if block.exitswitch is None: + assert len(block.exits) == 1 + link = block.exits[0] + target_label = self._get_block_name(link.target) + self._setup_link(link) + self.generator.branch_unconditionally(target_label) + elif block.exitswitch.concretetype is ootype.Bool: + self.render_bool_switch(block) + elif block.exitswitch.concretetype in (ootype.Signed, ootype.SignedLongLong, + ootype.Unsigned, ootype.UnsignedLongLong, + ootype.Char, ootype.UniChar): + self.render_numeric_switch(block) + else: + assert False, 'Unknonw exitswitch type: %s' % block.exitswitch.concretetype + + def render_bool_switch(self, block): + assert len(block.exits) == 2 + for link in block.exits: + if link.exitcase: + link_true = link + else: + link_false = link + + true_label = self.next_label('link_true') + self.generator.load(block.exitswitch) + self.generator.KL(block.exitswitch.name, True) + self.generator.branch_conditionally(True, true_label) + self._follow_link(link_false) # if here, the exitswitch is false + self.set_label(true_label) + self._follow_link(link_true) # if here, the exitswitch is true + + def _follow_link(self, link): + target_label = self._get_block_name(link.target) + allowed = self.get_block_locals(link.prevblock) | self.get_block_links_args(link) + for arg in chain(*(exit.args for exit in link.prevblock.exits)): + if isinstance(arg, flowmodel.Variable) and arg.name in allowed and arg not in link.args and self.generator.HL(arg.name): + self.generator.KL(arg.name) + self._setup_link(link) + self.generator.branch_unconditionally(target_label) + + # def render_numeric_switch(self, block): # if block.exitswitch.concretetype in (ootype.SignedLongLong, ootype.UnsignedLongLong): # # TODO: it could be faster to check is the values fit in @@ -189,16 +273,5 @@ # for link, lbl in cases.itervalues(): # self.render_switch_case(link, lbl) - # def call_oostring(self, ARGTYPE): - # if isinstance(ARGTYPE, ootype.Instance): - # argtype = self.cts.types.object - # else: - # argtype = self.cts.lltype_to_cts(ARGTYPE) - # self.call_signature('string [pypylib]pypy.runtime.Utils::OOString(%s, int32)' % argtype) - - # def call_oounicode(self, ARGTYPE): - # argtype = self.cts.lltype_to_cts(ARGTYPE) - # self.call_signature('string [pypylib]pypy.runtime.Utils::OOUnicode(%s)' % argtype) - # Those parts of the generator interface that are function # specific Modified: pypy/branch/avm/pypy/translator/avm2/genavm.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/genavm.py (original) +++ pypy/branch/avm/pypy/translator/avm2/genavm.py Thu Jul 22 02:36:39 2010 @@ -1,18 +1,20 @@ import py from mech.fusion.avm2.abc_ import AbcFile +from pypy.conftest import option from pypy.translator.oosupport.genoo import GenOO -from pypy.translator.avm2.avm2gen import PyPyAvm2ilasm -from pypy.translator.avm2.constant import Avm2ConstGenerator, Avm2ClassConst, Avm2InstanceConst, Avm2RecordConst, Avm2ArrayListConst +from pypy.translator.avm2.codegen import PyPyCodeGenerator +from pypy.translator.avm2.constant import (Avm2ConstGenerator, Avm2ClassConst, + Avm2InstanceConst, Avm2RecordConst, Avm2ListConst, Avm2ArrayConst) from pypy.translator.avm2.database import LowLevelDatabase -from pypy.translator.avm2.function import Function as TamarinFunction +from pypy.translator.avm2.function import Function from pypy.translator.avm2.opcodes import opcodes from pypy.translator.avm2.types_ import Avm2TypeSystem class GenAVM2(GenOO): opcodes = opcodes - Function = TamarinFunction + Function = Function Database = LowLevelDatabase TypeSystem = Avm2TypeSystem @@ -21,14 +23,16 @@ ClassConst = Avm2ClassConst InstanceConst = Avm2InstanceConst RecordConst = Avm2RecordConst - ListConst = Avm2ArrayListConst - ArrayConst = Avm2ArrayListConst + ListConst = Avm2ListConst + ArrayConst = Avm2ArrayConst def __init__(self, tmpdir, translator, entrypoint, config=None, exctrans=False): GenOO.__init__(self, tmpdir, translator, entrypoint, config, exctrans) self.const_stat = str(tmpdir.join('const_stat')) self.ilasm = None self.abc = None + + # Get the base exception type for conversion. rtyper = translator.rtyper bk = rtyper.annotator.bookkeeper clsdef = bk.getuniqueclassdef(Exception) @@ -37,7 +41,7 @@ def create_assembler(self): self.abc = AbcFile() - return PyPyAvm2ilasm(self.db, self.abc, True) + return PyPyCodeGenerator(self.db, self.abc, option.mf_optim) def generate_source(self): if self.ilasm is None: Modified: pypy/branch/avm/pypy/translator/avm2/library.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/library.py (original) +++ pypy/branch/avm/pypy/translator/avm2/library.py Thu Jul 22 02:36:39 2010 @@ -1,80 +1,59 @@ +import functools + from pypy.rpython.ootypesystem import ootype -from mech.fusion.avm2.constants import QName, packagedQName, TYPE_MULTINAME_TypeName -from mech.fusion.avm2.query import ClassDesc -from mech.fusion.avm2.library import Library -from mech.fusion.avm2.playerglobal.flash.utils import Vector - -from pypy.translator.avm2.types_ import vec_qname - -## Monkey Patching! - -ClassDesc._nativeclass = None - -class PyPyLibrary(Library): - def resolve_class(self, TYPE): - if self.has_type(TYPE): - return self.get_type(TYPE) - if playerglobal_lib.has_type(TYPE): - return self.get_type(TYPE) - if TYPE.KIND == TYPE_MULTINAME_TypeName and TYPE.name == vec_qname: - assert len(TYPE.types) == 1 - return Vector[TYPE.types[0]] - if getattr(TYPE, "multiname", None): - return TYPE.multiname() - - def convert_classdesc(self, classdesc): - resolve = self.resolve_class - from pypy.translator.avm2.runtime import NativeClass, NativeInstance - from pypy.translator.avm2.runtime import _overloaded_static_meth, _static_meth - - if classdesc._nativeclass is not None: - return classdesc._nativeclass - - TYPE = NativeInstance(classdesc.Package, classdesc.ShortName, None, {}, {}) - Class = NativeClass(TYPE, {}, {}) - classdesc._nativeclass = Class - if classdesc.FullName == QName('Object'): - TYPE._set_superclass(ootype.ROOT) - else: - BASETYPE = resolve(classdesc.BaseType) - TYPE._set_superclass(BASETYPE) - - TYPE._isArray = classdesc.IsArray - if classdesc.IsArray: - TYPE._ELEMENT = resolve(classdesc.ElementType) - - # add both static and instance methods, and static fields - static_meths = self.group_methods(classdesc.StaticMethods, - _overloaded_static_meth, _static_meth, ootype.StaticMethod) - meths = self.group_methods(classdesc.Methods, ootype.overload, - ootype.meth, ootype.Meth) - Class._add_methods(static_meths) - Class._add_static_fields(dict((name, - resolve(t)) for name, t in classdesc.StaticFields])) - Class._add_static_fields(dict((name, - resolve(t)) for name, t, g, s in classdesc.StaticProperties)) - TYPE._add_methods(meths) - TYPE._add_fields(dict((name, resolve(t)) for name, t in classdesc.Fields)) - TYPE._add_fields(dict((name, resolve(t)) for name, t, g, s in classdesc.Properties)) - return Class - - def group_methods(self, methods, overload, meth, Meth): - from pypy.translator.avm2.runtime import OverloadingResolver - groups = {} - for name, args, result, AS3 in methods: - groups[name] = args, result, AS3 - - res = {} - attrs = dict(resolver=OverloadingResolver) - for name, methlist in groups.iteritems(): - meths = [meth(Meth([self.resolve_class(arg) for arg in args], - self.resolve_class(result))) for (args, result) in methlist] - res[name] = overload(*meths, **attrs) - return res +from mech.fusion.avm2 import playerglobal +from mech.fusion.avm2.interfaces import IMultiname +from mech.fusion.avm2.constants import QName, TypeName, undefined +from mech.fusion.avm2.library import make_package + +from pypy.translator.avm2.runtime import AVM2Class, AVM2Instance, _static_meth + +def ClassDescConverter(classdesc, _cache={}): + T = classdesc.Library.get_type + C = lambda t: ClassDescConverter(t)._INSTANCE + def resolve(TYPE): + if TYPE in classdesc.Library: + return C(T(TYPE)) + + if type(TYPE) == object or TYPE in (undefined, None): # interfaces and root objects + return ootype.ROOT + name = IMultiname(TYPE) + + if isinstance(name, TypeName): + return C(T(name.name).specialize(name.types)) + + return name + + def create_methods(methods, meth, Meth): + return dict((name, meth(Meth([resolve(arg) for arg in args], resolve(result)))) + for name, args, result in methods) + + if classdesc in _cache: + return _cache[classdesc] + + TYPE = AVM2Instance(classdesc.Package, classdesc.ShortName, None) + Class = AVM2Class(TYPE, {}, {}) + _cache[classdesc] = Class + + if classdesc.FullName == QName('Object'): + TYPE._set_superclass(ootype.ROOT) + else: + TYPE._set_superclass(resolve(classdesc.BaseType)) + + # add both static and instance methods, and static fields + static_meths = create_methods(classdesc.StaticMethods, _static_meth, ootype.StaticMethod) + meths = create_methods(classdesc.Methods, ootype.meth, ootype.Meth) + Class._add_methods(static_meths) + Class._add_static_fields(dict((t[0], resolve(t[1])) for t in + classdesc.StaticFields + classdesc.StaticProperties)) + + TYPE._add_methods(meths) + TYPE._add_fields(dict((t[0], resolve(t[1])) for t in + classdesc.Fields + classdesc.Properties if not TYPE._has_field(t[0]))) -from mech.fusion.avm2.library import get_playerglobal + return Class -playerglobal_lib = get_playerglobal(Library=PyPyLibrary) -playerglobal_lib.install_global(__name__.rpartition(".")[0]+".playerglobal") +convert_package = functools.partial(make_package, Interface=ClassDescConverter) +get_playerglobal = functools.partial(make_package, playerglobal, ClassDescConverter) Modified: pypy/branch/avm/pypy/translator/avm2/metavm.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/metavm.py (original) +++ pypy/branch/avm/pypy/translator/avm2/metavm.py Thu Jul 22 02:36:39 2010 @@ -1,19 +1,37 @@ + from pypy.rpython.ootypesystem import ootype -from pypy.translator.oosupport.metavm import MicroInstruction, PushAllArgs, \ - StoreResult, GetField, SetField, DownCast, _Call as _OOCall, \ - get_primitive_name -from pypy.translator.avm2.runtime import _static_meth, NativeInstance -from pypy.translator.avm2 import types_ as types -from mech.fusion.avm2 import constants - -STRING_HELPER_CLASS = '[pypylib]pypy.runtime.String' - -def functype_to_cts(cts, FUNC): - ret_type = cts.lltype_to_cts(FUNC.RESULT) - arg_types = [cts.lltype_to_cts(arg).typename() - for arg in FUNC.ARGS - if arg is not ootype.Void] - return ret_type, arg_types +from pypy.objspace.flow.model import Constant +from pypy.translator.oosupport.metavm import MicroInstruction, _Call as _OOCall +from pypy.translator.avm2.runtime import _static_meth +from pypy.translator.avm2.rlib import RLib +from pypy.translator.avm2.types_ import types + +from mech.fusion.avm2.interfaces import IMultiname + +class AddSub(MicroInstruction): + def __init__(self, int, add): + if int: + if add: + self.p1, self.n1 = 'increment_i', 'decrement_i' + else: + self.p1, self.n1 = 'decrement_i', 'increment_i' + else: + if add: + self.p1, self.n1 = 'increment', 'decrement' + else: + self.p1, self.n1 = 'decrement', 'increment' + + def render(self, generator, op): + if isinstance(op.args[1], Constant): + if op.args[1].value == 1: + generator.load(op.args[0]) + generator.emit(self.p1) + elif op.args[1].value == -1: + generator.load(op.args[0]) + generator.emit(self.n1) + for arg in op.args: + generator.load(arg) + generator.emit('add') class _Call(_OOCall): def render(self, generator, op): @@ -24,58 +42,47 @@ generator.call_graph(op.args[0].value.graph, op.args[1:]) def _render_static_function(self, generator, funcdesc, args): - import pdb; pdb.set_trace() - for func_arg in args[1:]: # push parameters - self._load_arg_or_null(generator, func_arg) cts = generator.cts + generator.load(*args[1:]) generator.emit() - def _load_arg_or_null(self, generator, *args): - for arg in args: - if arg.concretetype is ootype.Void: - if arg.value is None: - generator.ilasm.push_null() # special-case: use None as a null value - else: - assert False, "Don't know how to load this arg" - else: - generator.load(arg) - - class _CallMethod(_Call): - DISPATCH = { - ootype.Array : { - "ll_setitem_fast": lambda gen: gen.array_setitem(), - "ll_getitem_fast": lambda gen: gen.array_getitem(), - "ll_length": lambda gen: gen.get_field("length", types.types.int), - }, - ootype.AbstractString : { - "ll_append": lambda gen: gen.emit('add'), - "ll_stritem_nonneg": lambda gen: gen.call_method("charAt", 1, types.types.string), - "ll_strlen": lambda gen: gen.get_field("length", types.types.int), - }, + donothing = object() + ONELINER = { + "list_ll_length": lambda gen,a: gen.get_field("length", types.int), + + "str_ll_append": lambda gen,a: gen.emit('add'), + "str_ll_strlen": lambda gen,a: gen.get_field("length", types.int), + "str_ll_stritem_nonneg": lambda gen,a: gen.call_method("charAt", 1, types.string), + "str_ll_strconcat": lambda gen,a: gen.emit('add'), + "str_ll_streq": lambda gen,a: gen.emit('equals'), + "str_ll_strcmp": lambda gen,a: gen.call_method("localeCompare", 1, types.int), + + "stringbuilder_ll_allocate": donothing, + "stringbuilder_ll_build": donothing, } def render(self, generator, op): method = op.args[0] self._render_method(generator, method.value, op.args[1:]) def _render_method(self, generator, method_name, args): - this = args[0] - native = isinstance(this.concretetype, NativeInstance) - if native: - self._load_arg_or_null(generator, *args) - else: + DISPATCH, meth = self.ONELINER, args[0].concretetype.oopspec_name+'_'+method_name + if meth in DISPATCH: generator.load(*args) - - for TYPE, D in self.DISPATCH.iteritems(): - if isinstance(this.concretetype, TYPE): - D[method_name](generator) - break + if DISPATCH[meth] is not self.donothing: + DISPATCH[meth](generator, args) + elif getattr(generator, meth, None): + getattr(generator, meth)(args) + elif meth in RLib: + RLib[meth](generator, *args) else: - generator.call_method(method_name, len(args)-1) + # storeresult expects something on the stack + generator.push_null() class _IndirectCall(_CallMethod): def render(self, generator, op): # discard the last argument because it's used only for analysis + assert False self._render_method(generator, 'Invoke', op.args[:-1]) class ConstCall(MicroInstruction): @@ -84,8 +91,7 @@ self.__args = args def render(self, generator, op): - generator.load(*self.__args) - generator.call_method(self.__method, len(self.__args)) + generator.call_method_constargs(self.__method, None, *self.__args) class ConstCallArgs(MicroInstruction): def __init__(self, method, numargs): @@ -103,32 +109,52 @@ class _OOString(MicroInstruction): def render(self, generator, op): - if op.args[1] == -1: - generator.emit('findpropstrict', types.str_qname) - generator.load(op.args[0]) - generator.emit('callproperty', types.str_qname, 1) + obj, base = op.args + if base == -1 or (isinstance(base, Constant) and base.value == -1): + if isinstance(obj.concretetype, ootype.Instance): + generator.load("<") + generator.call_method_constargs('getName', obj) + generator.emit('add') + generator.load(" object>") + generator.emit('add') + else: + generator.call_method_constargs('toString', obj) else: - generator.load(*op.args) - generator.emit('callproperty', constants.QName("toString"), 1) + generator.call_method_constargs('toString', obj, base) class _OOParseInt(MicroInstruction): def render(self, generator, op): - generator.load(*op.args) - generator.emit('getglobalscope') - generator.emit('callproperty', constants.QName("parseInt"), 2) + generator.call_function_constargs('parseInt', *op.args) class _OOParseFloat(MicroInstruction): def render(self, generator, op): - generator.load(*op.args) - generator.emit('getglobalscope') - generator.emit('callproperty', constants.QName("parseFloat"), 1) + generator.call_function_constargs('parseFloat', *op.args) + +class _SetField(MicroInstruction): + def render(self, generator, op): + this, field, value = op.args + if value.concretetype is ootype.Void: + return + generator.load(this) + generator.load(value) + generator.set_field(field.value) + +class _GetField(MicroInstruction): + def render(self, generator, op): + # OOType produces void values on occassion that can safely be ignored + if op.result.concretetype is ootype.Void: + return + this, field = op.args + generator.load(this) + generator.get_field(field.value) + class PushClass(MicroInstruction): def __init__(self, classname): - self.__class = constants.QName(classname) + self.__class = IMultiname(classname) def render(self, generator, op): - generator.emit('getlex', constants.QName(self.__class)) + generator.load(self.__class) # class _NewCustomDict(MicroInstruction): # def render(self, generator, op): @@ -220,33 +246,8 @@ class _TypeOf(MicroInstruction): def render(self, generator, op): - c_type, = op.args - assert c_type.concretetype is ootype.Void - if isinstance(c_type.value, ootype.StaticMethod): - FUNC = c_type.value - fullname = generator.cts.lltype_to_cts(FUNC) - else: - cliClass = c_type.value - fullname = cliClass._INSTANCE._name generator.gettype() -class _EventHandler(MicroInstruction): - def render(self, generator, op): - cts = generator.cts - v_obj, c_methname = op.args - assert c_methname.concretetype is ootype.Void - TYPE = v_obj.concretetype - classname = TYPE._name - methname = 'o' + c_methname.value # XXX: do proper mangling - _, meth = TYPE._lookup(methname) - METH = ootype.typeOf(meth) - ret_type, arg_types = functype_to_cts(cts, METH) - arg_list = ', '.join(arg_types) - generator.load(v_obj) - desc = '%s class %s::%s(%s)' % (ret_type, classname, methname, arg_list) - generator.ilasm.opcode('ldftn instance', desc) - generator.ilasm.opcode('newobj', 'instance void class [mscorlib]System.EventHandler::.ctor(object, native int)') - class _GetStaticField(MicroInstruction): def render(self, generator, op): cts_class = op.args[0].value @@ -262,17 +263,6 @@ generator.ilasm.swap() generator.ilasm.set_field(fldname) -class _FieldInfoForConst(MicroInstruction): - def render(self, generator, op): - from pypy.translator.cli.constant import CONST_CLASS - llvalue = op.args[0].value - constgen = generator.db.constant_generator - const = constgen.record_const(llvalue) - generator.ilasm.opcode('ldtoken', CONST_CLASS) - generator.ilasm.call('class [mscorlib]System.Type class [mscorlib]System.Type::GetTypeFromHandle(valuetype [mscorlib]System.RuntimeTypeHandle)') - generator.ilasm.opcode('ldstr', '"%s"' % const.name) - generator.ilasm.call_method('class [mscorlib]System.Reflection.FieldInfo class [mscorlib]System.Type::GetField(string)', virtual=True) - # OOTYPE_TO_MNEMONIC = { # ootype.Bool: 'i1', @@ -300,11 +290,11 @@ GetArrayElem = _GetArrayElem() SetArrayElem = _SetArrayElem() TypeOf = _TypeOf() -EventHandler = _EventHandler() GetStaticField = _GetStaticField() SetStaticField = _SetStaticField() -FieldInfoForConst = _FieldInfoForConst() OOString = _OOString() OOParseInt = _OOParseInt() OOParseFloat = _OOParseFloat() # CastPrimitive = _CastPrimitive() +GetField = _GetField() +SetField = _SetField() Modified: pypy/branch/avm/pypy/translator/avm2/opcodes.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/opcodes.py (original) +++ pypy/branch/avm/pypy/translator/avm2/opcodes.py Thu Jul 22 02:36:39 2010 @@ -1,16 +1,18 @@ from pypy.translator.avm2.metavm import Call, CallMethod, \ - IndirectCall, GetField, SetField, DownCast, \ - NewArray, GetArrayElem, SetArrayElem, OOParseInt, OOParseFloat, \ - TypeOf, EventHandler, GetStaticField, SetStaticField, \ - FieldInfoForConst, OOString, ConstCall, ConstCallArgs, PushClass -from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ - New, RuntimeNew, CastTo, PushPrimitive, OONewArray + IndirectCall, GetField, SetField, AddSub, \ + NewArray, OOParseInt, OOParseFloat, \ + TypeOf, GetStaticField, SetStaticField, \ + OOString, ConstCall, ConstCallArgs, PushClass +from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, \ + StoreResult, InstructionList, New, RuntimeNew, CastTo, PushPrimitive, \ + OONewArray, DownCast from pypy.translator.cli.cts import WEAKREF from pypy.rpython.ootypesystem import ootype # some useful instruction patterns Not = ['not'] DoNothing = [PushAllArgs] +DontStoreResult = object() Ignore = [] def _not(op): @@ -45,8 +47,10 @@ # 'cli_setstaticfield': [SetStaticField], # 'cli_fieldinfo_for_const': [FieldInfoForConst], 'oois': 'strictequals', - 'oononnull': [PushAllArgs, 'pushnull', 'equals', 'not'], - 'classof': [PushAllArgs, 'callvirt instance class [mscorlib]System.Type object::GetType()'], +# 'oononnull': [PushAllArgs, 'pushnull', 'equals', 'not'], + 'oononnull': [PushAllArgs, 'convert_b'], + 'ooisnull': [PushAllArgs, 'pushnull', 'equals'], + 'classof': [PushAllArgs, TypeOf], 'instanceof': [CastTo], 'subclassof': [CastTo], 'ooidentityhash': [PushAllArgs, 'callvirt instance int32 object::GetHashCode()'], @@ -134,12 +138,16 @@ 'char_gt': 'greaterthan', 'char_ge': 'greaterequals', - 'unichar_eq': 'ceq', - 'unichar_ne': _not('ceq'), + 'unichar_eq': 'equals', + 'unichar_ne': _not('equals'), 'int_add': 'add_i', + 'int_add_ovf': 'add_i', + 'int_add_nonneg_ovf': 'add_i', 'int_sub': 'subtract_i', + 'int_sub_ovf': 'subtract_i', 'int_mul': 'multiply_i', + 'int_mul_ovf': 'multiply_i', 'int_floordiv': [PushAllArgs, 'divide', 'convert_i'], # 'int_floordiv_zer': _check_zer('div'), 'int_mod': 'modulo', @@ -155,7 +163,7 @@ 'int_rshift': 'rshift', 'int_xor': 'bitxor', # 'int_add_ovf': _check_ovf('add.ovf'), -# 'int_add_nonneg_ovf': _check_ovf('add.ovf'), + # 'int_sub_ovf': _check_ovf('sub.ovf'), # 'int_mul_ovf': _check_ovf('mul.ovf'), 'int_floordiv_ovf': 'divide', # these can't overflow! @@ -253,7 +261,9 @@ if type(value) is str: value = InstructionList([PushAllArgs, value, StoreResult]) elif value is not None: - if value is not Ignore and StoreResult not in value: + if DontStoreResult in value: + value.remove(DontStoreResult) + elif value is not Ignore and StoreResult not in value: value.append(StoreResult) value = InstructionList(value) Modified: pypy/branch/avm/pypy/translator/avm2/record.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/record.py (original) +++ pypy/branch/avm/pypy/translator/avm2/record.py Thu Jul 22 02:36:39 2010 @@ -1,10 +1,11 @@ + from pypy.rpython.ootypesystem import ootype -from pypy.translator.cli.node import Node +from pypy.translator.avm2.node import ClassNodeBase -from mech.fusion.avm2 import constants as c, traits -from pypy.translator.avm2 import types_ as types +from mech.fusion.avm2.constants import QName +from mech.fusion.avm2.interfaces import IMultiname -class Record(Node): +class Record(ClassNodeBase): def __init__(self, db, record, name): self.db = db self.cts = db.genoo.TypeSystem(db) @@ -15,55 +16,55 @@ return hash(self.record) def __eq__(self, other): - return self.record == other.record + return type(self) == type(other) and self.record == other.record def __ne__(self, other): return not self == other + def __repr__(self): + return '' % self.name + + def get_type(self): + return QName(self.name) + def get_name(self): return self.name def get_base_class(self): - return c.QName("Object") + return QName("Object") - def render(self, ilasm): - self.ilasm = ilasm - ilasm.begin_class(c.QName(self.name)) - for f_name, (FIELD_TYPE, f_default) in self.record._fields.iteritems(): - f_name = self.cts.escape_name(f_name) - cts_type = self.cts.lltype_to_cts(FIELD_TYPE) - if cts_type != types.types.void: - ilasm.context.add_instance_trait(traits.AbcSlotTrait(c.QName(f_name), cts_type.multiname())) - self._toString() - #self._getHashCode() - ilasm.exit_context() - - def _toString(self): - # only for testing purposes, and only if the Record represents a tuple - from pypy.translator.cli.test.runtest import format_object + def get_fields(self): + return self.record._fields.iteritems() + + def _fieldToString(self, ilasm, fieldnum): + f_name = 'item%d' % (fieldnum,) + FIELD_TYPE, f_default = self.record._fields[f_name] + if FIELD_TYPE is ootype.Void: + return True + ilasm.push_this() + ilasm.emit('getproperty', IMultiname(f_name)) + ilasm.emit('convert_s') + def render_toString(self, ilasm): for f_name in self.record._fields: if not f_name.startswith('item'): return # it's not a tuple - self.ilasm.begin_method('toString', [], c.QName("String")) - self.ilasm.push_const("(") - for i in xrange(len(self.record._fields)): - f_name = 'item%d' % i - FIELD_TYPE, f_default = self.record._fields[f_name] - if FIELD_TYPE is ootype.Void: + ilasm.begin_method('toString', [], QName("String")) + ilasm.load("(") + fieldlen = len(self.record._fields)-1 + for i in xrange(fieldlen): + if self._fieldToString(ilasm, i): continue - self.ilasm.push_this() - #self.ilasm.get_field((f_type, self.name, f_name)) - self.ilasm.emit('getproperty', c.QName(f_name)) - self.ilasm.emit('callproperty', c.Multiname("toString", c.PROP_NAMESPACE_SET), 0) - self.ilasm.emit('add') - self.ilasm.push_const(", ") - self.ilasm.emit('add') - self.ilasm.push_const(")") - self.ilasm.emit('add') - self.ilasm.emit('returnvalue') - self.ilasm.exit_context() + ilasm.emit('add') + ilasm.load(", ") + ilasm.emit('add') + self._fieldToString(ilasm, fieldlen) + ilasm.emit('add') + ilasm.load(")") + ilasm.emit('add') + ilasm.emit('returnvalue') + ilasm.exit_context() # def _equals(self): # # field by field comparison Modified: pypy/branch/avm/pypy/translator/avm2/runtime.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/runtime.py (original) +++ pypy/branch/avm/pypy/translator/avm2/runtime.py Thu Jul 22 02:36:39 2010 @@ -3,10 +3,9 @@ from pypy.tool.pairtype import pair, pairtype from pypy.annotation.model import SomeObject, SomeInstance, SomeOOInstance, SomeInteger, s_None,\ s_ImpossibleValue, lltype_to_annotation, annotation_to_lltype, SomeChar, SomeString -# from pypy.annotation.unaryop import immutablevalue -# from pypy.annotation.binaryop import _make_none_union + from pypy.annotation import model as annmodel -# from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong + from pypy.rpython.error import TyperError from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.rmodel import Repr @@ -17,17 +16,16 @@ ## Annotation model -class SomeNativeClass(SomeObject): +class SomeAVM2Class(SomeObject): def getattr(self, s_attr): assert self.is_constant() assert s_attr.is_constant() nativeclass = self.const attrname = s_attr.const if attrname in nativeclass._static_fields: - TYPE = nativeclass._static_fields[attrname] - return OverloadingResolver.lltype_to_annotation(TYPE) + return nativeclass._static_fields[attrname] elif attrname in nativeclass._static_methods: - return SomeNativeStaticMethod(nativeclass, attrname) + return SomeAVM2StaticMethod(nativeclass, attrname) else: return s_ImpossibleValue @@ -45,13 +43,13 @@ return SomeOOInstance(self.const._INSTANCE) def rtyper_makerepr(self, rtyper): - return NativeClassRepr(self.const) + return AVM2ClassRepr(self.const) def rtyper_makekey(self): return self.__class__, self.const -class SomeNativeStaticMethod(SomeObject): +class SomeAVM2StaticMethod(SomeObject): def __init__(self, native_class, meth_name): self.native_class = native_class self.meth_name = meth_name @@ -60,7 +58,7 @@ return self.native_class._ann_static_method(self.meth_name, args_s) def rtyper_makerepr(self, rtyper): - return NativeStaticMethodRepr(self.native_class, self.meth_name) + return AVM2StaticMethodRepr(self.native_class, self.meth_name) def rtyper_makekey(self): return self.__class__, self.native_class, self.meth_name @@ -96,7 +94,7 @@ ## Rtyper model -class NativeClassRepr(Repr): +class AVM2ClassRepr(Repr): lowleveltype = ootype.Void def __init__(self, native_class): @@ -122,14 +120,13 @@ return hop.genop("setstaticfield", [c_class, c_name, v_value], resulttype=hop.r_result.lowleveltype) def rtype_simple_call(self, hop): - # TODO: resolve constructor overloading INSTANCE = hop.args_r[0].native_class._INSTANCE cINST = hop.inputconst(ootype.Void, INSTANCE) vlist = hop.inputargs(*hop.args_r)[1:] # discard the first argument hop.exception_is_here() return hop.genop("new", [cINST]+vlist, resulttype=hop.r_result.lowleveltype) -class NativeStaticMethodRepr(Repr): +class AVM2StaticMethodRepr(Repr): lowleveltype = ootype.Void def __init__(self, native_class, meth_name): @@ -153,15 +150,11 @@ class __extend__(pairtype(OOInstanceRepr, IntegerRepr)): def rtype_getitem((r_inst, r_int), hop): - if not r_inst.lowleveltype._isArray: - raise TyperError("getitem() on a non-array instance") v_array, v_index = hop.inputargs(r_inst, ootype.Signed) hop.exception_is_here() return hop.genop('getelem', [v_array, v_index], hop.r_result.lowleveltype) def rtype_setitem((r_inst, r_int), hop): - if not r_inst.lowleveltype._isArray: - raise TyperError("setitem() on a non-array instance") vlist = hop.inputargs(*hop.args_r) hop.exception_is_here() return hop.genop('setelem', vlist, hop.r_result.lowleveltype) @@ -176,50 +169,17 @@ hop.exception_cannot_occur() return hop.genop('arraylength', vlist, hop.r_result.lowleveltype) - def rtype_simple_call(self, hop): - TYPE = self.lowleveltype - _, meth = TYPE._lookup('Invoke') - assert isinstance(meth, ootype._overloaded_meth) - ARGS = tuple([repr.lowleveltype for repr in hop.args_r[1:]]) - desc = meth._get_desc('Invoke', ARGS) - cname = hop.inputconst(ootype.Void, desc) - vlist = hop.inputargs(self, *hop.args_r[1:]) - hop.exception_is_here() - return hop.genop("oosend", [cname]+vlist, - resulttype = hop.r_result.lowleveltype) + ## def rtype_simple_call(self, hop): + ## TYPE = self.lowleveltype + ## ARGS = tuple([repr.lowleveltype for repr in hop.args_r[1:]]) + ## vlist = hop.inputargs(self, *hop.args_r[1:]) + ## hop.exception_is_here() + ## return hop.genop("oosend", [cname]+vlist, + ## resulttype = hop.r_result.lowleveltype) ## OOType model -class OverloadingResolver(ootype.OverloadingResolver): - - def _can_convert_from_to(self, ARG1, ARG2): - if ARG1 is ootype.Void and isinstance(ARG2, NativeInstance): - return True # ARG1 could be None, that is always convertible to a NativeInstance - else: - return ootype.OverloadingResolver._can_convert_from_to(self, ARG1, ARG2) - - def annotation_to_lltype(cls, ann): - if isinstance(ann, SomeChar): - return ootype.Char - elif isinstance(ann, SomeString): - return ootype.String - else: - return annotation_to_lltype(ann) - annotation_to_lltype = classmethod(annotation_to_lltype) - - def lltype_to_annotation(cls, TYPE): - if isinstance(TYPE, NativeInstance): - return SomeOOInstance(TYPE) - elif TYPE is ootype.Char: - return SomeChar() - elif TYPE is ootype.String: - return SomeString(can_be_None=True) - else: - return lltype_to_annotation(TYPE) - lltype_to_annotation = classmethod(lltype_to_annotation) - - class _static_meth(object): def __init__(self, TYPE): @@ -233,22 +193,7 @@ #assert ARGS == self._TYPE.ARGS return self -class _overloaded_static_meth(object): - def __init__(self, *overloadings, **attrs): - resolver = attrs.pop('resolver', OverloadingResolver) - assert not attrs - self._resolver = resolver(overloadings) - - def _set_attrs(self, cls, name): - for meth in self._resolver.overloadings: - meth._set_attrs(cls, name) - - def _get_desc(self, ARGS): - meth = self._resolver.resolve(ARGS) - assert isinstance(meth, _static_meth) - return meth._get_desc(ARGS) - -class NativeInstance(ootype.Instance): +class AVM2Instance(ootype.Instance): def __init__(self, namespace, name, superclass, fields={}, methods={}, _is_root=False, _hints = {}): fullname = '%s.%s' % (namespace, name) @@ -258,11 +203,9 @@ ootype.Instance.__init__(self, fullname, superclass, fields, methods, _is_root, _hints) - - ## RPython interface definition -class NativeClass(object): +class AVM2Class(object): def __init__(self, INSTANCE, static_methods, static_fields): self._name = INSTANCE._name self._INSTANCE = INSTANCE @@ -294,10 +237,10 @@ class Entry(ExtRegistryEntry): - _type_ = NativeClass + _type_ = AVM2Class def compute_annotation(self): - return SomeNativeClass() + return SomeAVM2Class() def compute_result_annotation(self): return SomeOOInstance(self.instance._INSTANCE) @@ -325,7 +268,7 @@ def specialize_call(self, hop): c_type, v_length = hop.inputargs(*hop.args_r) hop.exception_cannot_occur() - return hop.genop('avm2_newvector', [c_type, v_length], hop.r_result.lowleveltype) + return hop.genop('newvector', [c_type, v_length], hop.r_result.lowleveltype) class Entry(ExtRegistryEntry): @@ -500,10 +443,6 @@ # def compute_annotation(self): # return SomeOOInstance(CLR.System.Reflection.FieldInfo._INSTANCE) -from pypy.translator.avm2.query import NativeNamespace -playerglobal = NativeNamespace(None) -playerglobal._buildtree() - # known_delegates = { # ootype.StaticMethod([ootype.Signed], ootype.Signed): CLR.pypy.test.DelegateType_int__int_1, # ootype.StaticMethod([ootype.Signed] * 2, ootype.Signed): CLR.pypy.test.DelegateType_int__int_2, Modified: pypy/branch/avm/pypy/translator/avm2/test/browsertest.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/test/browsertest.py (original) +++ pypy/branch/avm/pypy/translator/avm2/test/browsertest.py Thu Jul 22 02:36:39 2010 @@ -23,11 +23,11 @@ """ -class InstanceWrapper: +class InstanceWrapper(Exception): def __init__(self, class_name): self.class_name = class_name -class ExceptionWrapper: +class ExceptionWrapper(Exception): def __init__(self, class_name): self.class_name = class_name Modified: pypy/branch/avm/pypy/translator/avm2/test/entrypoint.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/test/entrypoint.py (original) +++ pypy/branch/avm/pypy/translator/avm2/test/entrypoint.py Thu Jul 22 02:36:39 2010 @@ -1,103 +1,107 @@ + import py +import subprocess + from pypy.conftest import option +from pypy.rpython.ootypesystem import ootype from pypy.translator.avm2.test.browsertest import browsertest -from pypy.translator.avm2.avm2gen import PyPyAvm2ilasm +from pypy.translator.avm2.codegen import PyPyCodeGenerator +from pypy.translator.avm2.entrypoint import BaseEntryPoint +from mech.fusion.swf import tags from mech.fusion.swf.swfdata import SwfData -from mech.fusion.swf.tags import FileAttributes, SetBackgroundColor, \ - DefineEditText, SymbolClass, PlaceObject2, DoABC, ShowFrame, End from mech.fusion.swf.records import Rect, RGBA from mech.fusion.avm2.abc_ import AbcFile -from mech.fusion.avm2.constants import QName, packagedQName +from mech.fusion.avm2.constants import QName, packagedQName, MultinameL from mech.fusion.avm2.traits import AbcSlotTrait -class BaseTestEntryPoint(object): - def __init__(self, name, gen, wrap_exceptions): - self.excwrap = wrap_exceptions +class BaseTestEntryPoint(BaseEntryPoint): + def __init__(self, name, graph, excwrap): self.name = name - self.swf = SwfData() - self.swf.add_tag(FileAttributes()) - self.swf.add_tag(SetBackgroundColor(0x333333)) - self.swf.add_tag(DefineEditText(Rect(0, 0, 600, 400), "", (''' -Running test %s. -If no text saying "Go:" appears below, the test probably had an error. -====================================================================== -'''.strip() % (name,)), color=RGBA(0xFFFFFF))) - self.swf.add_tag(PlaceObject2(1, 2, name="edittext")) - self.abc = DoABC("PyPy Main") - self.actions = PyPyAvm2ilasm(gen.db, self.abc, option.mf_optim) - - self.swf.add_tag(self.abc) - self.swf.add_tag(SymbolClass({0:"PyPyTest_EntryPoint"})) - self.swf.add_tag(ShowFrame()) - self.swf.add_tag(End()) - - def update_text(self): - pass - - def print_text(self, text): - self.actions.push_const(text) - self.actions.store_var('text') - self.update_text() - - def print_var(self, prefix, varname): - self.actions.push_const(prefix) - self.actions.push_var(varname) - self.actions.emit('add') - self.actions.store_var('text') - self.update_text() - - def print_stack(self, prefix, dup=False): - if dup: - self.actions.dup() - self.actions.store_var('text') - self.update_text() + self.graph = graph + self.excwrap = excwrap + self.maincls_name = packagedQName("PyPyTest", name) + + def get_name(self): + return self.name + + def dependencies(self): + self.db.pending_function(self.graph) + + def render(self, ilasm): + self.actions = ilasm + self.prologue() + self.start_test() + self.call_test() + self.finish_test() - def start_test_maincls(self): + def prologue(self): pass - def start_test(self): - self.start_test_maincls() + def call_test(self): + qname = self.actions.cts.graph_to_qname(self.graph) if self.excwrap: self.actions.begin_try() + self.actions.emit('findpropstrict', qname) + self.gather_arguments() + self.actions.emit('callproperty', qname, len(self.graph.getargs())) def finish_test(self): - # WHHEEEEEEE! + # WHHEEEEEEEE! if self.excwrap: self.actions.end_try() - self.actions.emit('convert_s') self.actions.branch_unconditionally("PyPy::ExceptionWrapLabel") self.actions.begin_catch(QName("Error")) self.actions.push_exception() self.actions.emit('convert_s') self.actions.end_catch() self.actions.set_label("PyPy::ExceptionWrapLabel") - self.actions.store_var('result') - self.print_var("Got: ", 'result') self.publish_test() + self.actions.exit_until_type("script") + + def publish_test(self): + pass + + def gather_arguments(self): + pass + + def finish_compiling(self): self.epilogue() self.actions.finish() + self.save_test() - def do_test(self): + def epilogue(self): pass - def epilogue(self): + def save_test(self): pass - def publish_test(self): + def run_test(self, args): pass class SWFTestEntryPoint(BaseTestEntryPoint): - def do_test(self): - self.finish_test() - f = open("%s.flash.swf" % (self.name,), "wb") - f.write(self.swf.serialize()) - f.close() - f = open("%s.flash.abc" % (self.name,), "wb") - f.write(AbcFile.serialize(self.abc)) - f.close() - + def prologue(self): + self.swf = SwfData() + self.swf.add_tag(tags.FileAttributes()) + self.swf.add_tag(tags.SetBackgroundColor(0x333333)) + self.swf.add_tag(tags.DefineEditText(Rect(0, 0, 600, 400), "", (''' +Running test %s. If no text saying "Got: " +appears below, the test probably had an error. +================================================== +'''.strip() % (self.name,)), color=RGBA(0xFFFFFF))) + self.swf.add_tag(tags.PlaceObject2(1, 1, name="edittext")) + self.swf.add_tag(tags.DoABC("PyPy Main", self.actions.abc)) + self.swf.add_tag(tags.SymbolClass({0:"PyPyTest::" + self.name})) + self.swf.add_tag(tags.ShowFrame()) + self.swf.add_tag(tags.End()) + + def save_test(self): + barename = py.path.local(self.name + ".flash") + barename.new(ext=".swf").write(self.swf.serialize(), mode="wb") + barename.new(ext=".abc").write(self.actions.abc.serialize(), mode="wb") + + def run_test(self, args): return browsertest(self.name, self.swf) def get_edittext(self): @@ -109,23 +113,27 @@ def update_text(self): self.get_edittext() - self.actions.push_const("\n") + self.actions.load("\n") self.actions.push_var('text') self.actions.emit('add') self.actions.emit('callpropvoid', QName("appendText"), 1) - def start_test_maincls(self): - self.maincls = self.actions.begin_class(QName("PyPyTest_EntryPoint"), packagedQName("flash.display", "Sprite")) + def start_test(self): + self.maincls = self.actions.begin_class(self.maincls_name, + packagedQName("flash.display", "Sprite")) self.maincls.make_iinit() - self.maincls.add_instance_trait(AbcSlotTrait(QName("edittext"), packagedQName("flash.text", "TextField"))) + self.maincls.add_instance_trait(AbcSlotTrait(QName("edittext"), + packagedQName("flash.text", "TextField"))) def publish_test(self): + self.actions.store_var('text') + self.update_text() self.actions.emit('findpropstrict', packagedQName("flash.net", "URLRequest")) - self.actions.push_const('./test.result') + self.actions.load('./test.result') self.actions.emit('constructprop', packagedQName("flash.net", "URLRequest"), 1) self.actions.store_var('request') self.actions.push_var('request') - self.actions.push_const('POST') + self.actions.load('POST') self.actions.set_field('method') self.actions.push_var('request') self.actions.emit('findpropstrict', packagedQName("flash.net", "URLVariables")) @@ -140,32 +148,50 @@ self.actions.emit('callpropvoid', packagedQName("flash.net", "sendToURL"), 1) self.actions.emit('findpropstrict', packagedQName("flash.net", "navigateToURL")) self.actions.emit('findpropstrict', packagedQName("flash.net", "URLRequest")) - self.actions.push_const('javascript: self.close()') + self.actions.load('javascript: self.close()') self.actions.emit('constructprop', packagedQName("flash.net", "URLRequest"), 1) - self.actions.push_const('_self') + self.actions.load('_self') self.actions.emit('callpropvoid', packagedQName("flash.net", "navigateToURL"), 2) class TamarinTestEntryPoint(BaseTestEntryPoint): - def do_test(self): - self.finish_test() - f = open("%s.tamarin.abc" % (self.name,), "wb") - f.write(AbcFile.serialize(self.abc)) - f.close() - asdf + def save_test(self): + abc = py.path.local(self.name+".tamarin.abc") + abc.write(AbcFile.serialize(self.actions.abc), "wb") + + def run_test(self, args): + procargs = [option.tamexec, self.name + ".tamarin.abc", "--"] + map(str, args) + testproc = subprocess.Popen(procargs, stdout=subprocess.PIPE) + return testproc.stdout.read() - def update_text(self): + def publish_test(self): self.actions.emit('findpropstrict', QName("print")) - self.actions.push_const("\n") - self.actions.push_var('text') - self.actions.emit('add') + self.actions.swap() self.actions.emit('callpropvoid', QName("print"), 1) + def gather_arguments(self): + self.actions.load(dict(False=False, True=True)) + self.actions.store_var("booltable") + self.actions.load(packagedQName("avmplus", "System")) + self.actions.get_field("argv") + for index, arg in enumerate(self.graph.getargs()): + if arg.concretetype is not ootype.Void: + self.actions.dup() + if arg.concretetype is ootype.Bool: + self.actions.get_field(str(index)) + self.actions.push_var("booltable") + self.actions.swap() + self.actions.emit('getproperty', MultinameL()) + else: + self.actions.get_field(str(index), arg.concretetype) + self.actions.swap() + self.actions.pop() + def epilogue(self): self.actions.exit_until_type("script") self.actions.context.make_init() - self.actions.push_var('this') - self.actions.emit('constructprop', QName("PyPyTest_EntryPoint"), 0) + self.actions.push_this() + self.actions.emit('constructprop', self.maincls_name, 0) - def start_test_maincls(self): - self.maincls = self.actions.begin_class(QName("PyPyTest_EntryPoint")) + def start_test(self): + self.maincls = self.actions.begin_class(self.maincls_name) self.maincls.make_iinit() Modified: pypy/branch/avm/pypy/translator/avm2/test/runtest.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/test/runtest.py (original) +++ pypy/branch/avm/pypy/translator/avm2/test/runtest.py Thu Jul 22 02:36:39 2010 @@ -15,47 +15,53 @@ from pypy.translator.avm2.test.browsertest import ExceptionWrapper, InstanceWrapper from pypy.translator.avm2.genavm import GenAVM2 -ENTRY_POINTS = dict(swf=SWFTestEntryPoint, tamarin=TamarinTestEntryPoint) +class UnparsableResult(Exception): + pass def parse_result(string): string = string.strip() + if string.startswith("Got:"): + return parse_result_raw(string[4:].strip()) + elif string.startswith("ExceptionWrapper"): + return eval(string.splitlines()[0]) + raise UnparsableResult(string) + +def parse_result_raw(string): + if string == "": + return "" if string == "true": return True - elif string == "false": + if string == "false": return False - elif string == "undefined" or string == "null": + if string == "undefined" or string == "null": return None - elif all(c in "123456789-" for c in string): + if string[0] + string[-1] in ('()', '[]'): + res = [parse_result_raw(s) for s in string[1:-1].split(",")] + return tuple(res) if string[0] == '(' else res + try: return int(string) - elif "," in string: - if string.startswith("(") and string.endswith(")"): - return tuple(parse_result(s) for s in string[1:-1].split(",")) - return [parse_result(s) for s in string.split(",")] - elif string.startswith("ExceptionWrapper"): - return eval(string) - else: - try: - return float(string) - except ValueError: - pass + except ValueError: + pass + try: + return float(string) + except ValueError: + pass return string -def compile_function(func, name, annotation=[], backendopt=True, +def compile_function(func, name, annotation, backendopt=True, exctrans=False, annotatorpolicy=None, wrapexc=False): olddefs = patch_os() - gen = _build_gen(func, annotation, backendopt, exctrans, annotatorpolicy) - entry_point = ENTRY_POINTS[option.tamtarget](name, gen, wrapexc) - gen.ilasm = entry_point.actions + gen = _build_gen(func, name, annotation, backendopt, + exctrans, annotatorpolicy, wrapexc) gen.generate_source() + gen.entrypoint.finish_compiling() unpatch_os(olddefs) # restore original values - return entry_point, gen + return gen + +def _build_gen(func, name, annotation, backendopt=True, + exctrans=False, annotatorpolicy=None, wrapexc=False): + func = getattr(func, "im_func", func) -def _build_gen(func, annotation, backendopt=True, exctrans=False, - annotatorpolicy=None): - try: - func = func.im_func - except AttributeError: - pass t = TranslationContext() ann = t.buildannotator(policy=annotatorpolicy) ann.build_types(func, annotation) @@ -70,15 +76,21 @@ tmpdir = py.path.local('.') - return GenAVM2(tmpdir, t, None, exctrans) + if option.browsertest: + entry = SWFTestEntryPoint + else: + entry = TamarinTestEntryPoint + ep = entry(name, t.graphs[0], wrapexc) + + return GenAVM2(tmpdir, t, ep, exctrans) class AVM2Test(BaseRtypingTest, OORtypeMixin): def __init__(self): self._func = None self._ann = None - self._harness = None + self._entry_point = None - def _compile(self, fn, args, ann=None, backendopt=True, exctrans=False, wrapexc=False): + def _compile(self, fn, ann=None, backendopt=True, exctrans=False, wrapexc=False): frame = sys._getframe() while frame: name = frame.f_code.co_name @@ -88,14 +100,13 @@ else: name = "test_unknown" - if ann is None: - ann = [lltype_to_annotation(typeOf(x)) for x in args] - - self._entry_point = compile_function(fn, "%s.%s" % (type(self).__name__, name), + if fn == self._func and self._ann == ann: + return self._gen + self._gen = compile_function(fn, "%s.%s" % (type(self).__name__, name), ann, backendopt=backendopt, exctrans=exctrans, wrapexc=wrapexc) self._func = fn self._ann = ann - return self._entry_point + return self._gen def _skip_win(self, reason): if platform.system() == 'Windows': @@ -116,11 +127,11 @@ def interpret(self, fn, args, annotation=None, backendopt=None, exctrans=False, wrapexc=False): backendopt = self._get_backendopt(backendopt) - entry_point, gen = self._compile(fn, args, annotation, - backendopt, exctrans, wrapexc) - entry_point.start_test() - entry_point.actions.call_graph(gen.translator.graphs[0], args) - result = parse_result(entry_point.do_test()) + if annotation is None: + annotation = [lltype_to_annotation(typeOf(x)) for x in args] + gen = self._compile(fn, annotation, + backendopt, exctrans, wrapexc) + result = parse_result(gen.entrypoint.run_test(args)) if isinstance(result, ExceptionWrapper): raise result return result Modified: pypy/branch/avm/pypy/translator/avm2/test/test_int.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/test/test_int.py (original) +++ pypy/branch/avm/pypy/translator/avm2/test/test_int.py Thu Jul 22 02:36:39 2010 @@ -1,10 +1,8 @@ -import autopath -import py -from pypy.translator.avm1.test.runtest import AVM1Test + +from pypy.translator.avm2.test.runtest import AVM2Test from pypy.rpython.test.test_rint import BaseTestRint -from pypy.rlib.rarithmetic import r_longlong -class TestAVM1Int(AVM1Test, BaseTestRint): +class TestAVM2Int(AVM2Test, BaseTestRint): def test_char_constant(self): def dummyfn(i): return chr(i) @@ -17,34 +15,4 @@ pass # it doesn't make sense here div_mod_iteration_count = 20 - - def test_div_mod(self): - import random - - for inttype in (int, r_longlong): - - # def d(x, y): - # return x/y - # for i in range(self.div_mod_iteration_count): - # x = inttype(random.randint(-100000, 100000)) - # y = inttype(random.randint(-100000, 100000)) - # if not y: continue - # res = self.interpret(d, [x, y]) - # print "x:", x, "y:", y, "result in Flash:", res, "result in Python:", d(x, y) - # assert res == d(x, y) - - def m(x, y): - return x%y - - for i in range(self.div_mod_iteration_count): - x = inttype(random.randint(-100000, 100000)) - y = inttype(random.randint(-100000, 100000)) - if not y: continue - res = self.interpret(m, [x, y]) - print "x:", x, "y:", y, "result in Flash:", res, "result in Python:", m(x, y) - assert res == m(x, y) - - -if __name__=="__main__": - TestAVM1Int().test_div_mod() Modified: pypy/branch/avm/pypy/translator/avm2/test/test_string.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/test/test_string.py (original) +++ pypy/branch/avm/pypy/translator/avm2/test/test_string.py Thu Jul 22 02:36:39 2010 @@ -7,7 +7,7 @@ EMPTY_STRING_HASH = 0 def test_unichar_const(self): - py.test.skip("CLI interpret doesn't support unicode for input arguments") + py.test.skip("Tamarin interpret doesn't support unicode for input arguments") test_unichar_eq = test_unichar_const test_unichar_ord = test_unichar_const test_unichar_hash = test_unichar_const @@ -15,11 +15,11 @@ test_char_unichar_eq_2 = test_unichar_const def test_upper(self): - py.test.skip("CLI doens't support backquotes inside string literals") + py.test.skip("Tamarin doesn't support backquotes inside string literals") test_lower = test_upper def test_hlstr(self): - py.test.skip("CLI tests can't have string as input arguments") + py.test.skip("Tamarin tests can't have string as input arguments") test_inplace_add = test_hlstr Modified: pypy/branch/avm/pypy/translator/avm2/types_.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/types_.py (original) +++ pypy/branch/avm/pypy/translator/avm2/types_.py Thu Jul 22 02:36:39 2010 @@ -1,71 +1,42 @@ """ -bTranslate between PyPy ootypesystem and .NET Common Type System +Translate between PyPy ootypesystem and the Tamarin Type System. """ -import exceptions - -from py.builtin import set from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype -from pypy.translator.cli import oopspec -from pypy.translator.cli.option import getoption -from mech.fusion.avm2 import constants +from py.builtin import set -from pypy.tool.ansi_print import ansi_log +from mech.fusion.avm2.constants import packagedQName, QName, TypeName, null +from mech.fusion.avm2.interfaces import IMultiname -vec_qname = constants.packagedQName("__AS3__.vec", "Vector") -str_qname = constants.QName("String") -arr_qname = constants.QName("Array") +from zope.component import adapter, provideAdapter +from zope.interface import implementer class Avm2Type(object): - def typename(self): - raise NotImplementedError + def __init__(self, multiname, default): + self.multiname = IMultiname(multiname) + self.default = default + + @classmethod + def from_string(cls, name, default=None): + ns, _, name = name.rpartition('::') + return cls(packagedQName(ns, name), default) def load_default(self, asm): - raise NotImplementedError + asm.load(self.default) def __str__(self): - return str(self.multiname()) - - def __hash__(self): - return hash(self.multiname()) + return str(self.multiname) - def __eq__(self, other): - return isinstance(other, type(self)) and self.multiname() == other.multiname() - - def __ne__(self, other): - return not self == other - -class Avm2PrimitiveType(Avm2Type): - def __init__(self, name, default): - self.name, self.default = name, default - - def load_default(self, gen): - gen.load(self.default) + at adapter(Avm2Type) + at implementer(IMultiname) +def _adapter(self): + return self.multiname - def typename(self): - return self.name +provideAdapter(_adapter) - def multiname(self): - return constants.QName(self.typename()) - -class Avm2NamespacedType(Avm2Type): - nstype = constants.TYPE_NAMESPACE_PackageNamespace - def __init__(self, name, namespace=''): - if '::' in name and namespace == '': - self.ns, self.name = name.rsplit('::', 1) - else: - self.name = name - self.ns = namespace - - def typename(self): - return "%s::%s" % (self.ns, self.name) - - def multiname(self): - return constants.QName(self.name, constants.Namespace(self.nstype, self.ns)) - -class Avm2ArrayType(Avm2Type): +class Avm2VectorType(object): def __init__(self, itemtype): self.ITEM = itemtype @@ -73,25 +44,27 @@ gen.oonewarray(self, 0) def __str__(self): - return "%s.<%s>" % (vec_qname, constants.QName(self.ITEM)) + return "Vector.<%s>" % (IMultiname(self.ITEM)) + + at adapter(Avm2VectorType) + at implementer(IMultiname) +def _adapter(self): + return TypeName(packagedQName("__AS3__.vec", "Vector"), IMultiname(self.ITEM)) - def multiname(self): - return constants.TypeName(vec_qname, constants.QName(self.ITEM)) +provideAdapter(_adapter) -T = Avm2PrimitiveType -# N = Avm2NamespacedType +T = Avm2Type class types: - void = T('void', constants.null) + void = T('*', null) int = T('int', -1) uint = T('uint', 0) bool = T('Boolean', False) float = T('Number', float('nan')) string = T('String', "") - type = T('Class', constants.QName('Class')) + type = T('Class', QName('Class')) object = T('Object', {}) - list = Avm2ArrayType + list = Avm2VectorType dict = T('Object', {}) -# sb = N('StringBuilder', 'pypy.lib') del T _lltype_to_cts = { @@ -106,9 +79,9 @@ ootype.UniChar: types.string, ootype.Class: types.type, ootype.String: types.string, -# ootype.StringBuilder: types.sb, + ootype.StringBuilder: types.string, ootype.Unicode: types.string, -# ootype.UnicodeBuilder: types.sb, + ootype.UnicodeBuilder: types.string, # maps generic types to their ordinal ootype.List.SELFTYPE_T: types.list, @@ -129,22 +102,22 @@ elif isinstance(t, ootype.Instance): NATIVE_INSTANCE = t._hints.get('NATIVE_INSTANCE', None) if NATIVE_INSTANCE: - return Avm2NamespacedType(NATIVE_INSTANCE._name) + return Avm2Type.from_string(NATIVE_INSTANCE._name) else: name = self.db.pending_class(t) - return Avm2NamespacedType(name) + return Avm2Type.from_string(name) elif isinstance(t, ootype.Record): name = self.db.pending_record(t) - return Avm2NamespacedType(name) + return Avm2Type.from_string(name) elif isinstance(t, ootype.StaticMethod): delegate = self.db.record_delegate(t) - return Avm2NamespacedType(delegate) + return Avm2Type.from_string(delegate) elif isinstance(t, (ootype.Array, ootype.List)): item_type = self.lltype_to_cts(t.ITEM) return types.list(item_type) elif isinstance(t, ootype.Dict): - key_type = self.lltype_to_cts(t._KEYTYPE) - value_type = self.lltype_to_cts(t._VALUETYPE) + #key_type = self.lltype_to_cts(t._KEYTYPE) + #value_type = self.lltype_to_cts(t._VALUETYPE) return types.dict ## elif isinstance(t, ootype.DictItemsIterator): ## key_type = self.lltype_to_cts(t._KEYTYPE) @@ -170,9 +143,14 @@ func_name = graph.name namespace = getattr(graph, '_namespace_', None) if namespace: - return constants.packagedQName(namespace, func_name) + return packagedQName(namespace, func_name) else: - return constants.QName(func_name) + return QName(func_name) + + def instance_to_qname(self, instance): + classname = self.db.class_name(instance) + ns, name = classname.rsplit('::', 1) + return packagedQName(ns, name) # def ctor_name(self, t): # return 'instance void %s::.ctor()' % self.lltype_to_cts(t) @@ -258,10 +236,10 @@ # else: # assert False -def dict_of_void_ll_copy_hack(TYPE, ret_type): - # XXX: ugly hack to make the ll_copy signature correct when - # CustomDict is special-cased to DictOfVoid. - if isinstance(TYPE, ootype.CustomDict) and TYPE._VALUETYPE is ootype.Void: - return ret_type.typename().replace('Dict`2', 'DictOfVoid`2') - else: - return ret_type +## def dict_of_void_ll_copy_hack(TYPE, ret_type): +## # XXX: ugly hack to make the ll_copy signature correct when +## # CustomDict is special-cased to DictOfVoid. +## if isinstance(TYPE, ootype.CustomDict) and TYPE._VALUETYPE is ootype.Void: +## return ret_type.typename().replace('Dict`2', 'DictOfVoid`2') +## else: +## return ret_type From jcreigh at codespeak.net Thu Jul 22 15:45:52 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 22 Jul 2010 15:45:52 +0200 (CEST) Subject: [pypy-svn] r76312 - pypy/branch/asmgcc-64/pypy/translator/platform Message-ID: <20100722134552.B7E38282BD4@codespeak.net> Author: jcreigh Date: Thu Jul 22 15:45:50 2010 New Revision: 76312 Modified: pypy/branch/asmgcc-64/pypy/translator/platform/posix.py Log: respect shared_only and standalone_only when generating a Makefile Modified: pypy/branch/asmgcc-64/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/posix.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/posix.py Thu Jul 22 15:45:50 2010 @@ -98,6 +98,11 @@ else: target_name = exe_name.basename + if shared: + cflags = self.cflags + self.shared_only + else: + cflags = self.cflags + self.standalone_only + m = GnuMakefile(path) m.exe_name = exe_name m.eci = eci @@ -126,7 +131,7 @@ ('LIBS', self._libs(eci.libraries)), ('LIBDIRS', self._libdirs(eci.library_dirs)), ('INCLUDEDIRS', self._includedirs(rel_includedirs)), - ('CFLAGS', self.cflags), + ('CFLAGS', cflags), ('CFLAGSEXTRA', list(eci.compile_extra)), ('LDFLAGS', linkflags), ('LDFLAGSEXTRA', list(eci.link_extra)), From jcreigh at codespeak.net Thu Jul 22 15:50:53 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 22 Jul 2010 15:50:53 +0200 (CEST) Subject: [pypy-svn] r76313 - pypy/branch/asmgcc-64/pypy/translator/platform Message-ID: <20100722135053.E7071282BD4@codespeak.net> Author: jcreigh Date: Thu Jul 22 15:50:52 2010 New Revision: 76313 Modified: pypy/branch/asmgcc-64/pypy/translator/platform/linux.py Log: link libffi dynamically on x86-64 Linux (for now) Modified: pypy/branch/asmgcc-64/pypy/translator/platform/linux.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/linux.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/linux.py Thu Jul 22 15:50:52 2010 @@ -3,7 +3,7 @@ from pypy.translator.platform import _run_subprocess from pypy.translator.platform.posix import BasePosix -class Linux(BasePosix): +class BaseLinux(BasePosix): name = "linux" link_flags = ['-pthread', '-lrt'] @@ -24,10 +24,12 @@ return self._pkg_config("libffi", "--libs-only-L", ['/usr/lib/libffi']) + +class Linux(BaseLinux): def library_dirs_for_libffi_a(self): # places where we need to look for libffi.a return self.library_dirs_for_libffi() + ['/usr/lib'] -class Linux64(Linux): - shared_only = ['-fPIC'] +class Linux64(BaseLinux): + pass From jcreigh at codespeak.net Thu Jul 22 17:06:47 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 22 Jul 2010 17:06:47 +0200 (CEST) Subject: [pypy-svn] r76314 - in pypy/branch/asmgcc-64/pypy/translator/c/gcc: . test Message-ID: <20100722150647.4ACA8282BF6@codespeak.net> Author: jcreigh Date: Thu Jul 22 17:06:45 2010 New Revision: 76314 Modified: pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py Log: move (de)compress_callshape, format_location, format_callshape to tracker class; update tests accordingly Modified: pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/test/test_trackgcroot.py Thu Jul 22 17:06:45 2010 @@ -1,15 +1,12 @@ import py import sys, re -from pypy.translator.c.gcc.trackgcroot import format_location -from pypy.translator.c.gcc.trackgcroot import format_callshape from pypy.translator.c.gcc.trackgcroot import LOC_NOWHERE, LOC_REG from pypy.translator.c.gcc.trackgcroot import LOC_EBP_PLUS, LOC_EBP_MINUS from pypy.translator.c.gcc.trackgcroot import LOC_ESP_PLUS from pypy.translator.c.gcc.trackgcroot import ElfAssemblerParser from pypy.translator.c.gcc.trackgcroot import DarwinAssemblerParser -from pypy.translator.c.gcc.trackgcroot import compress_callshape -from pypy.translator.c.gcc.trackgcroot import decompress_callshape from pypy.translator.c.gcc.trackgcroot import PARSERS +from pypy.translator.c.gcc.trackgcroot import ElfFunctionGcRootTracker32 from StringIO import StringIO import py.test @@ -17,40 +14,39 @@ def test_format_location(): - # FIXME - py.test.skip() - assert format_location(LOC_NOWHERE) == '?' - assert format_location(LOC_REG | (1<<2)) == '%ebx' - assert format_location(LOC_REG | (2<<2)) == '%esi' - assert format_location(LOC_REG | (3<<2)) == '%edi' - assert format_location(LOC_REG | (4<<2)) == '%ebp' - assert format_location(LOC_EBP_PLUS + 0) == '(%ebp)' - assert format_location(LOC_EBP_PLUS + 4) == '4(%ebp)' - assert format_location(LOC_EBP_MINUS + 4) == '-4(%ebp)' - assert format_location(LOC_ESP_PLUS + 0) == '(%esp)' - assert format_location(LOC_ESP_PLUS + 4) == '4(%esp)' + cls = ElfFunctionGcRootTracker32 + assert cls.format_location(LOC_NOWHERE) == '?' + assert cls.format_location(LOC_REG | (1<<2)) == '%ebx' + assert cls.format_location(LOC_REG | (2<<2)) == '%esi' + assert cls.format_location(LOC_REG | (3<<2)) == '%edi' + assert cls.format_location(LOC_REG | (4<<2)) == '%ebp' + assert cls.format_location(LOC_EBP_PLUS + 0) == '(%ebp)' + assert cls.format_location(LOC_EBP_PLUS + 4) == '4(%ebp)' + assert cls.format_location(LOC_EBP_MINUS + 4) == '-4(%ebp)' + assert cls.format_location(LOC_ESP_PLUS + 0) == '(%esp)' + assert cls.format_location(LOC_ESP_PLUS + 4) == '4(%esp)' def test_format_callshape(): - # FIXME - py.test.skip() + cls = ElfFunctionGcRootTracker32 expected = ('{4(%ebp) ' # position of the return address '| 8(%ebp), 12(%ebp), 16(%ebp), 20(%ebp) ' # 4 saved regs '| 24(%ebp), 28(%ebp)}') # GC roots - assert format_callshape((LOC_EBP_PLUS+4, - LOC_EBP_PLUS+8, - LOC_EBP_PLUS+12, - LOC_EBP_PLUS+16, - LOC_EBP_PLUS+20, - LOC_EBP_PLUS+24, - LOC_EBP_PLUS+28)) == expected + assert cls.format_callshape((LOC_EBP_PLUS+4, + LOC_EBP_PLUS+8, + LOC_EBP_PLUS+12, + LOC_EBP_PLUS+16, + LOC_EBP_PLUS+20, + LOC_EBP_PLUS+24, + LOC_EBP_PLUS+28)) == expected def test_compress_callshape(): + cls = ElfFunctionGcRootTracker32 shape = (1, 127, 0x1234, 0x5678, 0x234567, 0x765432, 0x61626364, 0x41424344) - bytes = list(compress_callshape(shape)) + bytes = list(cls.compress_callshape(shape)) print bytes assert len(bytes) == 1+1+2+3+4+4+5+5+1 - assert decompress_callshape(bytes) == list(shape) + assert cls.decompress_callshape(bytes) == list(shape) def test_find_functions_elf(): source = """\ @@ -143,7 +139,7 @@ tabledict = {} seen = {} for entry in table: - print '%s: %s' % (entry[0], format_callshape(PARSERS[format], entry[1])) + print '%s: %s' % (entry[0], tracker.format_callshape(entry[1])) tabledict[entry[0]] = entry[1] # find the ";; expected" lines prevline = "" @@ -156,7 +152,7 @@ label = prevmatch.group(1) assert label in tabledict got = tabledict[label] - assert format_callshape(PARSERS[format], got) == expected + assert tracker.format_callshape(got) == expected seen[label] = True if format == 'msvc': expectedlines.insert(i-2, 'PUBLIC\t%s\n' % (label,)) Modified: pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py Thu Jul 22 17:06:45 2010 @@ -358,6 +358,55 @@ self.lines.insert(call.lineno+1, '\t.globl\t%s\n' % (label,)) call.global_label = label + @classmethod + def compress_callshape(cls, shape): + # For a single shape, this turns the list of integers into a list of + # bytes and reverses the order of the entries. The length is + # encoded by inserting a 0 marker after the gc roots coming from + # shape[5:] and before the 5 values coming from shape[4] to + # shape[0]. In practice it seems that shapes contain many integers + # whose value is up to a few thousands, which the algorithm below + # compresses down to 2 bytes. Very small values compress down to a + # single byte. + + # Callee-save regs plus ret addr + min_size = len(cls.CALLEE_SAVE_REGISTERS) + 1 + + assert len(shape) >= min_size + shape = list(shape) + assert 0 not in shape[min_size:] + shape.insert(min_size, 0) + result = [] + for loc in shape: + assert loc >= 0 + flag = 0 + while loc >= 0x80: + result.append(int(loc & 0x7F) | flag) + flag = 0x80 + loc >>= 7 + result.append(int(loc) | flag) + result.reverse() + return result + + @classmethod + def decompress_callshape(cls, bytes): + # For tests. This logic is copied in asmgcroot.py. + result = [] + n = 0 + while n < len(bytes): + value = 0 + while True: + b = bytes[n] + n += 1 + value += b + if b < 0x80: + break + value = (value - 0x80) << 7 + result.append(value) + result.reverse() + assert result[5] == 0 + del result[5] + return result # ____________________________________________________________ CANNOT_COLLECT = { # some of the most used functions that cannot collect @@ -733,6 +782,64 @@ if target in ['__alldiv', '__allrem', '__allmul', '__alldvrm']: insns.append(InsnStackAdjust(16)) return insns + # __________ debugging output __________ + + @classmethod + def format_location(cls, loc): + # A 'location' is a single number describing where a value is stored + # across a call. It can be in one of the CALLEE_SAVE_REGISTERS, or + # in the stack frame at an address relative to either %esp or %ebp. + # The last two bits of the location number are used to tell the cases + # apart; see format_location(). + assert loc >= 0 + kind = loc & LOC_MASK + if kind == LOC_REG: + if loc == LOC_NOWHERE: + return '?' + reg = (loc >> 2) - 1 + return '%' + cls.CALLEE_SAVE_REGISTERS[reg].replace("%", "") + else: + offset = loc & ~ LOC_MASK + if kind == LOC_EBP_PLUS: + result = '(%' + cls.EBP.replace("%", "") + ')' + elif kind == LOC_EBP_MINUS: + result = '(%' + cls.EBP.replace("%", "") + ')' + offset = -offset + elif kind == LOC_ESP_PLUS: + result = '(%' + cls.ESP.replace("%", "") + ')' + else: + assert 0, kind + if offset != 0: + result = str(offset) + result + return result + + @classmethod + def format_callshape(cls, shape): + # A 'call shape' is a tuple of locations in the sense of + # format_location(). They describe where in a function frame + # interesting values are stored, when this function executes a 'call' + # instruction. + # + # shape[0] is the location that stores the fn's own return + # address (not the return address for the currently + # executing 'call') + # + # shape[1..N] is where the fn saved its own caller's value of a + # certain callee save register. (where N is the number + # of callee save registers.) + # + # shape[>N] are GC roots: where the fn has put its local GCPTR + # vars + # + num_callee_save_regs = len(cls.CALLEE_SAVE_REGISTERS) + assert isinstance(shape, tuple) + # + 1 for the return address + assert len(shape) >= (num_callee_save_regs + 1) + result = [cls.format_location(loc) for loc in shape] + return '{%s | %s | %s}' % (result[0], + ', '.join(result[1:(num_callee_save_regs+1)]), + ', '.join(result[(num_callee_save_regs+1):])) + class FunctionGcRootTracker32(FunctionGcRootTracker): WORD = 4 @@ -1179,7 +1286,7 @@ table = tracker.computegcmaptable(self.verbose) if self.verbose > 1: for label, state in table: - print >> sys.stderr, label, '\t', format_callshape(self, state) + print >> sys.stderr, label, '\t', tracker.format_callshape(state) table = compress_gcmaptable(table) if self.shuffle and random.random() < 0.5: self.gcmaptable[:0] = table @@ -1432,6 +1539,8 @@ else: word_decl = '.long' + tracker_cls = PARSERS[self.format].FunctionGcRootTracker + # The pypy_asm_stackwalk() function if self.format == 'msvc': @@ -1601,7 +1710,7 @@ n = shapes[state] except KeyError: n = shapes[state] = shapeofs - bytes = [str(b) for b in compress_callshape(state)] + bytes = [str(b) for b in tracker_cls.compress_callshape(state)] shapelines.append('\t%s,\t/* %s */\n' % ( ', '.join(bytes), shapeofs)) @@ -1633,7 +1742,7 @@ n = shapes[state] except KeyError: n = shapes[state] = shapeofs - bytes = [str(b) for b in compress_callshape(state)] + bytes = [str(b) for b in tracker_cls.compress_callshape(state)] shapelines.append('\t/*%d*/\t.byte\t%s\n' % ( shapeofs, ', '.join(bytes))) @@ -1643,7 +1752,7 @@ print >> output, '\t%s\t%s-%d' % ( word_decl, label, - PARSERS[self.format].FunctionGcRootTracker.OFFSET_LABELS) + tracker_cls.OFFSET_LABELS) print >> output, '\t%s\t%d' % (word_decl, n) print >> output, """\ @@ -1685,59 +1794,6 @@ pass -# __________ debugging output __________ - -def format_location(parser, loc): - # A 'location' is a single number describing where a value is stored - # across a call. It can be in one of the CALLEE_SAVE_REGISTERS, or - # in the stack frame at an address relative to either %esp or %ebp. - # The last two bits of the location number are used to tell the cases - # apart; see format_location(). - tracker_cls = parser.FunctionGcRootTracker - assert loc >= 0 - kind = loc & LOC_MASK - if kind == LOC_REG: - if loc == LOC_NOWHERE: - return '?' - reg = (loc >> 2) - 1 - return '%' + tracker_cls.CALLEE_SAVE_REGISTERS[reg].replace("%", "") - else: - offset = loc & ~ LOC_MASK - if kind == LOC_EBP_PLUS: - result = '(%' + tracker_cls.EBP.replace("%", "") + ')' - elif kind == LOC_EBP_MINUS: - result = '(%' + tracker_cls.EBP.replace("%", "") + ')' - offset = -offset - elif kind == LOC_ESP_PLUS: - result = '(%' + tracker_cls.ESP.replace("%", "") + ')' - else: - assert 0, kind - if offset != 0: - result = str(offset) + result - return result - -def format_callshape(parser, shape): - # A 'call shape' is a tuple of locations in the sense of format_location(). - # They describe where in a function frame interesting values are stored, - # when this function executes a 'call' instruction. - # - # shape[0] is the location that stores the fn's own return address - # (not the return address for the currently executing 'call') - # shape[1] is where the fn saved its own caller's %ebx value - # shape[2] is where the fn saved its own caller's %esi value - # shape[3] is where the fn saved its own caller's %edi value - # shape[4] is where the fn saved its own caller's %ebp value - # shape[>=5] are GC roots: where the fn has put its local GCPTR vars - # - num_callee_save_regs = len(parser.FunctionGcRootTracker.CALLEE_SAVE_REGISTERS) - assert isinstance(shape, tuple) - # + 1 for the return address - assert len(shape) >= (num_callee_save_regs + 1) - result = [format_location(parser, loc) for loc in shape] - return '{%s | %s | %s}' % (result[0], - ', '.join(result[1:(num_callee_save_regs+1)]), - ', '.join(result[(num_callee_save_regs+1):])) - # __________ table compression __________ def compress_gcmaptable(table): @@ -1764,54 +1820,6 @@ yield (label1, state, is_range) i = j -def compress_callshape(shape): - # For a single shape, this turns the list of integers into a list of - # bytes and reverses the order of the entries. The length is - # encoded by inserting a 0 marker after the gc roots coming from - # shape[5:] and before the 5 values coming from shape[4] to - # shape[0]. In practice it seems that shapes contain many integers - # whose value is up to a few thousands, which the algorithm below - # compresses down to 2 bytes. Very small values compress down to a - # single byte. - - # FIXME - # MIN_SIZE = 5 - MIN_SIZE = 7 - - assert len(shape) >= MIN_SIZE - shape = list(shape) - assert 0 not in shape[MIN_SIZE:] - shape.insert(MIN_SIZE, 0) - result = [] - for loc in shape: - assert loc >= 0 - flag = 0 - while loc >= 0x80: - result.append(int(loc & 0x7F) | flag) - flag = 0x80 - loc >>= 7 - result.append(int(loc) | flag) - result.reverse() - return result - -def decompress_callshape(bytes): - # For tests. This logic is copied in asmgcroot.py. - result = [] - n = 0 - while n < len(bytes): - value = 0 - while True: - b = bytes[n] - n += 1 - value += b - if b < 0x80: - break - value = (value - 0x80) << 7 - result.append(value) - result.reverse() - assert result[5] == 0 - del result[5] - return result def getidentifier(s): def mapchar(c): From getxsick at codespeak.net Thu Jul 22 21:30:09 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 22 Jul 2010 21:30:09 +0200 (CEST) Subject: [pypy-svn] r76317 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100722193009.B0307282B9D@codespeak.net> Author: getxsick Date: Thu Jul 22 21:30:06 2010 New Revision: 76317 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: add cache for calldescr Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Thu Jul 22 21:30:06 2010 @@ -81,7 +81,7 @@ else: raise ValueError(self.res_type) - calldescr = self.gen_calldescr() # XXX add cache + calldescr = self.gen_calldescr() self.looptoken = LoopToken() oplist = [ResOperation(rop.CALL, bargs, bres, descr=calldescr), ResOperation(rop.FINISH, [bres], None, @@ -92,7 +92,7 @@ cache[self.res_type] = { tuple(self.args_type) : self.looptoken } self.setup_stack() - def gen_calldescr(self): + def gen_calldescr(self, extrainfo=None): arg_classes = ''.join(self.args_type) gccache = self.cpu.gc_ll_descr @@ -108,9 +108,15 @@ raise NotImplementedError('Unknown type of descr: %s' % self.res_type) - calldescr = cls(arg_classes) - calldescr.create_call_stub(gccache.rtyper, self.res) - return calldescr + key = (cls, arg_classes, extrainfo) + cache = gccache._cache_call + try: + return cache[key] + except KeyError: + calldescr = cls(arg_classes, extrainfo) + calldescr.create_call_stub(gccache.rtyper, self.res) + cache[key] = calldescr + return calldescr def call(self, push_result): res = self.cpu.execute_token(self.looptoken) @@ -153,4 +159,3 @@ _clsname = 'SignedCallDescr' def get_result_size(self, translate_support_code): return symbolic.get_size(lltype.Signed, translate_support_code) - From getxsick at codespeak.net Thu Jul 22 21:40:07 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 22 Jul 2010 21:40:07 +0200 (CEST) Subject: [pypy-svn] r76318 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100722194007.DF0C9282B9D@codespeak.net> Author: getxsick Date: Thu Jul 22 21:40:06 2010 New Revision: 76318 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: rollback r76317. there is no need to have cache if we use static classes. Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Thu Jul 22 21:40:06 2010 @@ -81,7 +81,7 @@ else: raise ValueError(self.res_type) - calldescr = self.gen_calldescr() + calldescr = self.gen_calldescr() # XXX add cache self.looptoken = LoopToken() oplist = [ResOperation(rop.CALL, bargs, bres, descr=calldescr), ResOperation(rop.FINISH, [bres], None, @@ -92,7 +92,7 @@ cache[self.res_type] = { tuple(self.args_type) : self.looptoken } self.setup_stack() - def gen_calldescr(self, extrainfo=None): + def gen_calldescr(self): arg_classes = ''.join(self.args_type) gccache = self.cpu.gc_ll_descr @@ -108,15 +108,9 @@ raise NotImplementedError('Unknown type of descr: %s' % self.res_type) - key = (cls, arg_classes, extrainfo) - cache = gccache._cache_call - try: - return cache[key] - except KeyError: - calldescr = cls(arg_classes, extrainfo) - calldescr.create_call_stub(gccache.rtyper, self.res) - cache[key] = calldescr - return calldescr + calldescr = cls(arg_classes) + calldescr.create_call_stub(gccache.rtyper, self.res) + return calldescr def call(self, push_result): res = self.cpu.execute_token(self.looptoken) @@ -159,3 +153,4 @@ _clsname = 'SignedCallDescr' def get_result_size(self, translate_support_code): return symbolic.get_size(lltype.Signed, translate_support_code) + From getxsick at codespeak.net Thu Jul 22 21:43:20 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 22 Jul 2010 21:43:20 +0200 (CEST) Subject: [pypy-svn] r76319 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100722194320.52C77282B9D@codespeak.net> Author: getxsick Date: Thu Jul 22 21:43:18 2010 New Revision: 76319 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: use cache per instance of _Get instead of global one Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Thu Jul 22 21:43:18 2010 @@ -7,8 +7,6 @@ from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.typesystem import deref -cache = {} # XXX global! - class CDLL(object): def __init__(self, name, load=True): if load: @@ -41,6 +39,7 @@ self.cpu = cpu lib = lib.handler bargs = [] + self._cache = {} try: self.funcaddr = rffi.cast(lltype.Signed, rdynload.dlsym(lib, func)) @@ -49,8 +48,10 @@ bargs.append(BoxInt()) # grab from the cache if possible + arg_classes = ''.join(self.args_type) + key = (self.res_type, arg_classes) try: - self.looptoken = cache[self.res_type][tuple(self.args_type)] + self.looptoken = self._cache[key] except KeyError: args = [] for arg in self.args_type: @@ -89,7 +90,7 @@ self.cpu.compile_loop(bargs, oplist, self.looptoken) # add to the cache - cache[self.res_type] = { tuple(self.args_type) : self.looptoken } + self._cache[key] = self.looptoken self.setup_stack() def gen_calldescr(self): @@ -153,4 +154,3 @@ _clsname = 'SignedCallDescr' def get_result_size(self, translate_support_code): return symbolic.get_size(lltype.Signed, translate_support_code) - From getxsick at codespeak.net Thu Jul 22 22:11:10 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 22 Jul 2010 22:11:10 +0200 (CEST) Subject: [pypy-svn] r76320 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100722201110.4C74C282B9D@codespeak.net> Author: getxsick Date: Thu Jul 22 22:11:08 2010 New Revision: 76320 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: do not create CPU instances dynamically. some progress with translation heh. Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Thu Jul 22 22:11:08 2010 @@ -7,6 +7,8 @@ from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.typesystem import deref +GLOBAL_CPU = CPU(None, None) + class CDLL(object): def __init__(self, name, load=True): if load: @@ -15,7 +17,7 @@ self.lib = None self.name = name - self.cpu = CPU(None, None) + self.cpu = GLOBAL_CPU def get(self, func, args_type, res_type='v'): return _Get(self.cpu, self.lib, func, args_type, res_type) From getxsick at codespeak.net Thu Jul 22 23:31:04 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 22 Jul 2010 23:31:04 +0200 (CEST) Subject: [pypy-svn] r76321 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100722213104.F285A282B9D@codespeak.net> Author: getxsick Date: Thu Jul 22 23:30:58 2010 New Revision: 76321 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: kill kill kill Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Thu Jul 22 23:30:58 2010 @@ -37,7 +37,6 @@ assert isinstance(args_type, list) self.args_type = args_type self.res_type = res_type - self.res = None self.cpu = cpu lib = lib.handler bargs = [] @@ -70,21 +69,17 @@ raise ValueError(arg) if self.res_type == 'i': - self.res = lltype.Signed bres = BoxInt() elif self.res_type == 'f': - self.res = lltype.Float bres = BoxFloat() elif self.res_type == 'p': - self.res = lltype.Signed bres = BoxPtr() elif self.res_type == 'v': - self.res = lltype.Void bres = NULLBOX else: raise ValueError(self.res_type) - calldescr = self.gen_calldescr() # XXX add cache + calldescr = self.gen_calldescr() self.looptoken = LoopToken() oplist = [ResOperation(rop.CALL, bargs, bres, descr=calldescr), ResOperation(rop.FINISH, [bres], None, @@ -112,11 +107,10 @@ % self.res_type) calldescr = cls(arg_classes) - calldescr.create_call_stub(gccache.rtyper, self.res) return calldescr def call(self, push_result): - res = self.cpu.execute_token(self.looptoken) + self.cpu.execute_token(self.looptoken) if self.res_type == 'i': r = push_result(self.cpu.get_latest_value_int(0)) From getxsick at codespeak.net Fri Jul 23 00:01:36 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 23 Jul 2010 00:01:36 +0200 (CEST) Subject: [pypy-svn] r76322 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100722220136.BD389282B9C@codespeak.net> Author: getxsick Date: Fri Jul 23 00:01:34 2010 New Revision: 76322 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: Yet Another Massive Killing Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Fri Jul 23 00:01:34 2010 @@ -54,17 +54,13 @@ try: self.looptoken = self._cache[key] except KeyError: - args = [] for arg in self.args_type: if arg == 'i': bargs.append(BoxInt()) - args.append(lltype.Signed) elif arg == 'f': bargs.append(BoxFloat()) - args.append(lltype.Float) elif arg == 'p': bargs.append(BoxPtr()) - args.append(lltype.Signed) else: raise ValueError(arg) From getxsick at codespeak.net Fri Jul 23 00:13:28 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 23 Jul 2010 00:13:28 +0200 (CEST) Subject: [pypy-svn] r76323 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100722221328.0FD41282B9C@codespeak.net> Author: getxsick Date: Fri Jul 23 00:13:26 2010 New Revision: 76323 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: use more proper method name Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Fri Jul 23 00:13:26 2010 @@ -75,7 +75,7 @@ else: raise ValueError(self.res_type) - calldescr = self.gen_calldescr() + calldescr = self.get_calldescr() self.looptoken = LoopToken() oplist = [ResOperation(rop.CALL, bargs, bres, descr=calldescr), ResOperation(rop.FINISH, [bres], None, @@ -86,7 +86,7 @@ self._cache[key] = self.looptoken self.setup_stack() - def gen_calldescr(self): + def get_calldescr(self): arg_classes = ''.join(self.args_type) gccache = self.cpu.gc_ll_descr From hakanardo at codespeak.net Fri Jul 23 10:05:12 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Fri, 23 Jul 2010 10:05:12 +0200 (CEST) Subject: [pypy-svn] r76324 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100723080512.1CCC4282BF8@codespeak.net> Author: hakanardo Date: Fri Jul 23 10:05:05 2010 New Revision: 76324 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: tostring Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Fri Jul 23 10:05:05 2010 @@ -267,7 +267,7 @@ start, stop, step = space.decode_index(w_slice, self.len) if step < 0: w_lst = array_tolist__Array(space, self) - w_lst = space.getitem(w_lst, w_idx) + w_lst = space.getitem(w_lst, w_slice) w_a=mytype.w_class(self.space) w_a.fromsequence(w_lst) else: @@ -359,6 +359,17 @@ for i in range(len(s)): cbuf[oldlen * mytype.bytes + i] = s[i] + def array_tostring__Array(space, self): + cbuf = self.charbuf() + s = '' + i=0 + while i < self.len * mytype.bytes: + s += cbuf[i] + i+=1 + return self.space.wrap(s) + + + def array_fromfile__Array_ANY_ANY(space, self, w_f, w_n): if space.type(w_f).name != 'file': # FIXME: this cant be the right way? msg = "arg1 must be open file" @@ -377,17 +388,19 @@ raise OperationError(space.w_EOFError, space.wrap(msg)) array_fromstring__Array_ANY(space, self, w_item) - def array_fromunicode__Array_Unicode(space, self, w_ustr): - if mytype.typecode != 'u': + if mytype.typecode == 'u': + def array_fromunicode__Array_Unicode(space, self, w_ustr): + # XXX the following probable bug is not emulated: + # CPython accepts a non-unicode string or a buffer, and then + # behaves just like fromstring(), except that it strangely truncate + # string arguments at multiples of the unicode byte size. + # Let's only accept unicode arguments for now. + self.fromsequence(w_ustr) + else: + def array_fromunicode__Array_Unicode(space, self, w_ustr): msg = "fromunicode() may only be called on type 'u' arrays" raise OperationError(space.w_ValueError, space.wrap(msg)) - # XXX the following probable bug is not emulated: - # CPython accepts a non-unicode string or a buffer, and then - # behaves just like fromstring(), except that it strangely truncate - # string arguments at multiples of the unicode byte size. - # Let's only accept unicode arguments for now. - self.fromsequence(w_ustr) @@ -403,7 +416,7 @@ def repr__Array(space, self): if self.len == 0: - return "array('%s')" % self.typecode + return space.wrap("array('%s')" % self.typecode) elif self.typecode == "c": r = space.repr(space.call_method(self, 'tostring')) s = "array('%s', %s)" % (self.typecode, space.str_w(r)) @@ -415,7 +428,7 @@ else: r = space.repr(space.call_method(self, 'tolist')) s = "array('%s', %s)" % (self.typecode, space.str_w(r)) - return space.wrap(s) + return space.wrap(s) W_Array.__name__ = 'W_ArrayType_'+mytype.typecode From dan at codespeak.net Fri Jul 23 11:24:31 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Fri, 23 Jul 2010 11:24:31 +0200 (CEST) Subject: [pypy-svn] r76325 - in pypy/branch/micronumpy/pypy/module/micronumpy: . test Message-ID: <20100723092431.D9187282B9E@codespeak.net> Author: dan Date: Fri Jul 23 11:24:28 2010 New Revision: 76325 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Log: All is well with Fortran arrays now, as best I can tell, need to test with 3d arrays though. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Fri Jul 23 11:24:28 2010 @@ -29,7 +29,6 @@ _immutable_fields_ = ['array', 'stride'] def __init__(self, array): self.array = array - self.stride = array.stride(0) self.i = 0 def descr_iter(self, space): @@ -37,16 +36,17 @@ descr_iter.unwrap_spec = ['self', ObjSpace] def descr_next(self, space): - if self.i < self.array.opposite_stride(1): #will change for row/column + if self.i < self.array.shape[0]: #will change for row/column if len(self.array.shape) > 1: ar = MicroArray(self.array.shape[1:], # ok for column major? self.array.dtype, - self.array, - offset = self.array.offset + self.i * self.stride) + parent=self.array, + strides=self.array.strides[1:], + offset=self.array.get_offset(self.array.flatten_index([self.i]))) self.i += 1 return space.wrap(ar) elif len(self.array.shape) == 1: - next = self.array.getitem(space, self.i * self.stride) + next = self.array.getitem(space, self.array.flatten_index([self.i])) self.i += 1 return next else: @@ -74,18 +74,27 @@ ) class MicroArray(BaseNumArray): - def __init__(self, shape, dtype, parent=None, offset = 0): + _immutable_fields_ = ['parent', 'offset'] + def __init__(self, shape, dtype, order='C', strides=None, parent=None, offset=0): self.shape = shape self.dtype = dtype self.parent = parent self.offset = offset - self.order = 'C' #XXX: more forgiving to give it valid default + + self.order = order assert self.dtype is not None dtype = dtype.dtype #XXX: ugly size = size_from_shape(shape) + if strides is not None: + self.strides = strides + else: + self.strides = [0] * len(self.shape) + for i in range(len(self.shape)): + self.strides[i] = self.stride(i) # XXX calling self.stride repeatedly is a bit wasteful + if size > 0 and parent is None: self.data = dtype.alloc(size) elif parent is not None: @@ -93,6 +102,9 @@ else: self.data = null_data + def get_offset(self, index): + return self.offset + index + def descr_len(self, space): return space.wrap(self.shape[0]) descr_len.unwrap_spec = ['self', ObjSpace] @@ -100,14 +112,15 @@ def getitem(self, space, index): try: dtype = self.dtype.dtype #XXX: kinda ugly - return dtype.w_getitem(space, self.data, self.offset + index) + return dtype.w_getitem(space, self.data, + self.get_offset(index)) except IndexError, e: raise OperationError(space.w_IndexError, space.wrap("index out of bounds")) def setitem(self, space, index, w_value): dtype = self.dtype.dtype #XXX: kinda ugly - dtype.w_setitem(space, self.data, self.offset + index, w_value) #maybe hang onto w_dtype separately? + dtype.w_setitem(space, self.data, self.get_offset(index), w_value) #maybe hang onto w_dtype separately? def flatten_applevel_index(self, space, w_index): try: @@ -119,7 +132,8 @@ index = space.fixedview(w_index) index = [index_w(space, w_x) for w_x in index] - # XXX: normalize + + # Normalize indices for i in range(len(index)): if index[i] < 0: index[i] = self.shape[i] + index[i] @@ -128,8 +142,7 @@ def flatten_index(self, index): offset = 0 for i in range(len(index)): - stride = self.stride(i) - offset += index[i] * stride + offset += index[i] * self.strides[i] return offset def stride(self, i): @@ -155,8 +168,6 @@ index = self.flatten_applevel_index(space, w_index) try: - #w_iter = space.iter(w_index) # XXX: I guess len() should throw TypeError - # FIXME: what about slices? index_dimension = space.int_w(space.len(w_index)) except OperationError, e: if e.match(space, space.w_TypeError): @@ -168,7 +179,7 @@ elif index_dimension < len(self.shape): assert index_dimension > 0 array = MicroArray(self.shape[index_dimension:], self.dtype, - parent=self, offset=self.offset + index) + parent=self, offset=self.get_offset(index)) return space.wrap(array) else: raise OperationError(space.w_IndexError, @@ -337,6 +348,10 @@ fill_array = app_fill_array.interphook('fill_array') def array(space, w_xs, w_dtype=NoneNotWrapped, copy=True, order='C', subok=False, w_ndim=NoneNotWrapped): + if not order in 'CF': + raise OperationError(space.w_TypeError, + space.wrap("order not understood")) + if w_dtype is None: dtype = micronumpy.dtype.infer_from_iterable(space, w_xs) else: @@ -347,8 +362,7 @@ shape = infer_shape(space, w_xs) - ar = MicroArray(shape, wrappable_dtype) - ar.order = order + ar = MicroArray(shape, wrappable_dtype, order=order) w_ar = space.wrap(ar) fill_array(space, Modified: pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Fri Jul 23 11:24:28 2010 @@ -437,7 +437,6 @@ memlen = len(column_major) ar = array(space, w_data, w_dtype=w_int_descr, order='C') #C for C not column - for i in range(memlen): array_element = space.unwrap(ar.getitem(space, i)) # ugly, but encapsulates everything assert array_element == row_major[i] @@ -446,4 +445,3 @@ for i in range(memlen): array_element = space.unwrap(ar.getitem(space, i)) # ugly, but encapsulates everything assert array_element == column_major[i] - From hruske at codespeak.net Fri Jul 23 13:32:32 2010 From: hruske at codespeak.net (hruske at codespeak.net) Date: Fri, 23 Jul 2010 13:32:32 +0200 (CEST) Subject: [pypy-svn] r76326 - in pypy/trunk/lib_pypy: . pypy_test Message-ID: <20100723113232.5BEC3282B9E@codespeak.net> Author: hruske Date: Fri Jul 23 13:32:30 2010 New Revision: 76326 Modified: pypy/trunk/lib_pypy/datetime.py pypy/trunk/lib_pypy/pypy_test/test_datetime.py Log: Fix and test for issue 548 - timestamps were rounded differently in PyPy and CPython. Modified: pypy/trunk/lib_pypy/datetime.py ============================================================================== --- pypy/trunk/lib_pypy/datetime.py (original) +++ pypy/trunk/lib_pypy/datetime.py Fri Jul 23 13:32:30 2010 @@ -1412,7 +1412,7 @@ def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - if 1 - (t % 1.0) < 0.000001: + if 1 - (t % 1.0) < 0.0000005: t = float(int(t)) + 1 if t < 0: t -= 1 Modified: pypy/trunk/lib_pypy/pypy_test/test_datetime.py ============================================================================== --- pypy/trunk/lib_pypy/pypy_test/test_datetime.py (original) +++ pypy/trunk/lib_pypy/pypy_test/test_datetime.py Fri Jul 23 13:32:30 2010 @@ -15,4 +15,18 @@ expected = datetime.datetime(*(time.strptime(string, format)[0:6])) got = datetime.datetime.strptime(string, format) assert expected == got + +def test_datetime_rounding(): + b = 0.0000001 + a = 0.9999994 + + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 + assert datetime.datetime.utcfromtimestamp(a).second == 1 + From arigo at codespeak.net Fri Jul 23 15:23:19 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 23 Jul 2010 15:23:19 +0200 (CEST) Subject: [pypy-svn] r76327 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20100723132319.ECE96282B9E@codespeak.net> Author: arigo Date: Fri Jul 23 15:23:09 2010 New Revision: 76327 Modified: pypy/trunk/pypy/rpython/lltypesystem/rstr.py Log: issue485 resolved Fixed by adding an _annenforceargs_. Modified: pypy/trunk/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rstr.py Fri Jul 23 15:23:09 2010 @@ -674,6 +674,7 @@ res_index += item_len i += 1 return result + ll_join_strs._annenforceargs_ = [int, None] def ll_join_chars(length, chars): # no need to optimize this, will be replaced by string builder From hruske at codespeak.net Fri Jul 23 17:32:04 2010 From: hruske at codespeak.net (hruske at codespeak.net) Date: Fri, 23 Jul 2010 17:32:04 +0200 (CEST) Subject: [pypy-svn] r76332 - pypy/trunk/pypy/module/_rawffi/test Message-ID: <20100723153204.65B21282B9E@codespeak.net> Author: hruske Date: Fri Jul 23 17:32:02 2010 New Revision: 76332 Modified: pypy/trunk/pypy/module/_rawffi/test/test__rawffi.py Log: Add a test with negative pointers for rawffi.Array.fromaddress. Modified: pypy/trunk/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/trunk/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/trunk/pypy/module/_rawffi/test/test__rawffi.py Fri Jul 23 17:32:02 2010 @@ -677,7 +677,12 @@ a = A(1) a[0] = -1234 a.free() - + + def test_long_with_fromaddress(self): + import _rawffi + addr = -1 + raises(ValueError, _rawffi.Array('u').fromaddress, addr, 100) + def test_passing_raw_pointers(self): import _rawffi lib = _rawffi.CDLL(self.lib_name) From wlav at codespeak.net Sat Jul 24 01:36:37 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Sat, 24 Jul 2010 01:36:37 +0200 (CEST) Subject: [pypy-svn] r76333 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src test Message-ID: <20100723233637.BA4AE282B9E@codespeak.net> Author: wlav Date: Sat Jul 24 01:36:35 2010 New Revision: 76333 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Log: Allow passing of C-floats. Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Sat Jul 24 01:36:35 2010 @@ -41,6 +41,7 @@ "cppyy_deallocate", [C_TYPEHANDLE, C_OBJECT], lltype.Void, compilation_info=eci) + c_call_v = rffi.llexternal( "cppyy_call_v", [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], lltype.Void, @@ -57,10 +58,15 @@ "cppyy_call_l", [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.LONG, compilation_info=eci) +c_call_f = rffi.llexternal( + "cppyy_call_f", + [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.DOUBLE, + compilation_info=eci) c_call_d = rffi.llexternal( "cppyy_call_d", [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.DOUBLE, compilation_info=eci) + c_destruct = rffi.llexternal( "cppyy_destruct", [C_TYPEHANDLE, C_OBJECT], lltype.Void, Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Sat Jul 24 01:36:35 2010 @@ -1,5 +1,6 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rlib.rarithmetic import r_singlefloat from pypy.module.cppyy import helper, capi @@ -47,6 +48,13 @@ x[0] = arg return rffi.cast(rffi.VOIDP, x) +class FloatConverter(TypeConverter): + def convert_argument(self, space, w_obj): + arg = space.float_w(w_obj) + x = lltype.malloc(rffi.FLOATP.TO, 1, flavor='raw') + x[0] = r_singlefloat(arg) + return rffi.cast(rffi.VOIDP, x) + class DoubleConverter(TypeConverter): def convert_argument(self, space, w_obj): arg = space.float_w(w_obj) @@ -112,5 +120,6 @@ _converters["char"] = CharConverter() _converters["unsigned char"] = CharConverter() _converters["int"] = IntConverter() +_converters["float"] = FloatConverter() _converters["double"] = DoubleConverter() _converters["const char*"] = CStringConverter() Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Sat Jul 24 01:36:35 2010 @@ -30,6 +30,11 @@ result = capi.c_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) return space.wrap(result) +class FloatExecutor(FunctionExecutor): + def execute(self, space, func, cppthis, num_args, args): + result = capi.c_call_f(func.cpptype.handle, func.method_index, cppthis, num_args, args) + return space.wrap(result) + class DoubleExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): result = capi.c_call_d(func.cpptype.handle, func.method_index, cppthis, num_args, args) @@ -78,5 +83,6 @@ _executors["unsigned char"] = CharExecutor() _executors["int"] = LongExecutor() _executors["long int"] = LongExecutor() +_executors["float"] = FloatExecutor() _executors["double"] = DoubleExecutor() _executors["char*"] = CStringExecutor() Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Sat Jul 24 01:36:35 2010 @@ -18,7 +18,9 @@ int cppyy_call_b(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); char cppyy_call_c(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); long cppyy_call_l(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); + double cppyy_call_f(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); double cppyy_call_d(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); + void cppyy_destruct(cppyy_typehandle_t handle, cppyy_object_t self); cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_typehandle_t handle, int method_index); Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Sat Jul 24 01:36:35 2010 @@ -62,6 +62,11 @@ return cppyy_call_T(handle, method_index, self, numargs, args); } +double cppyy_call_f(cppyy_typehandle_t handle, int method_index, + cppyy_object_t self, int numargs, void* args[]) { + return cppyy_call_T(handle, method_index, self, numargs, args); +} + double cppyy_call_d(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]) { return cppyy_call_T(handle, method_index, self, numargs, args); Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Sat Jul 24 01:36:35 2010 @@ -81,8 +81,12 @@ for i in range(len(names)): exec 'c.set_%s = %d' % (names[i],2*i) assert eval('c.m_%s' % names[i]) == i + """ # float types + c.set_float( 0.123 ); assert round(c.get_float() - 0.123, 5) == 0 + c.set_double( 0.456 ); assert round(c.get_double() - 0.456, 8) == 0 + """ c.m_float = 0.123; assert round(c.get_float() - 0.123, 5) == 0 c.set_float( 0.234 ); assert round(c.m_float - 0.234, 5) == 0 c.m_double = 0.456; assert round(c.get_double() - 0.456, 8) == 0 From antocuni at codespeak.net Sun Jul 25 18:41:49 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 25 Jul 2010 18:41:49 +0200 (CEST) Subject: [pypy-svn] r76340 - in pypy/extradoc/talk/ep2010: pypy_demo pypy_demo/mandelbrot talk Message-ID: <20100725164149.09DAA5080B@codespeak.net> Author: antocuni Date: Sun Jul 25 18:41:47 2010 New Revision: 76340 Modified: pypy/extradoc/talk/ep2010/pypy_demo/index.html pypy/extradoc/talk/ep2010/pypy_demo/index2.html pypy/extradoc/talk/ep2010/pypy_demo/mandelbrot/views.py pypy/extradoc/talk/ep2010/talk/talk.pdf.info Log: small tweaks Modified: pypy/extradoc/talk/ep2010/pypy_demo/index.html ============================================================================== --- pypy/extradoc/talk/ep2010/pypy_demo/index.html (original) +++ pypy/extradoc/talk/ep2010/pypy_demo/index.html Sun Jul 25 18:41:47 2010 @@ -2,8 +2,8 @@

          Mandelbrot PyPy demo

          Show more Modified: pypy/extradoc/talk/ep2010/pypy_demo/index2.html ============================================================================== --- pypy/extradoc/talk/ep2010/pypy_demo/index2.html (original) +++ pypy/extradoc/talk/ep2010/pypy_demo/index2.html Sun Jul 25 18:41:47 2010 @@ -2,8 +2,8 @@

          Mandelbrot PyPy demo

          Modified: pypy/extradoc/talk/ep2010/pypy_demo/mandelbrot/views.py ============================================================================== --- pypy/extradoc/talk/ep2010/pypy_demo/mandelbrot/views.py (original) +++ pypy/extradoc/talk/ep2010/pypy_demo/mandelbrot/views.py Sun Jul 25 18:41:47 2010 @@ -2,14 +2,17 @@ from django.http import HttpResponse +WIDTH=640 +HEIGHT=480 + # only for benchmarking purposes def empty(request): return HttpResponse('') # render a mandelbrot image def render(request): - w = int(request.GET.get('w', 320)) - h = int(request.GET.get('h', 240)) + w = int(request.GET.get('w', WIDTH)) + h = int(request.GET.get('h', HEIGHT)) from py_mandel import mandelbrot img = mandelbrot(w, h) @@ -19,8 +22,8 @@ # render a mandelbrot image through the execnet pypy child, which is set up # below def pypy_render(request): - w = int(request.GET.get('w', 320)) - h = int(request.GET.get('h', 240)) + w = int(request.GET.get('w', WIDTH)) + h = int(request.GET.get('h', HEIGHT)) channel = pypy.remote_exec(""" from py_mandel import mandelbrot @@ -48,5 +51,3 @@ os.chdir("mandelbrot") sys.path.insert(0, '') """) - - Modified: pypy/extradoc/talk/ep2010/talk/talk.pdf.info ============================================================================== --- pypy/extradoc/talk/ep2010/talk/talk.pdf.info (original) +++ pypy/extradoc/talk/ep2010/talk/talk.pdf.info Sun Jul 25 18:41:47 2010 @@ -1,6 +1,6 @@ AvailableTransitions=[Crossfade] TransitionDuration = 100 -EstimatedDuration = 60*60 # in seconds +EstimatedDuration = 90*60 # in seconds MinutesOnly = True PageProps = { From antocuni at codespeak.net Sun Jul 25 18:42:44 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 25 Jul 2010 18:42:44 +0200 (CEST) Subject: [pypy-svn] r76341 - pypy/extradoc/talk/ep2010/talk Message-ID: <20100725164244.8F0595080B@codespeak.net> Author: antocuni Date: Sun Jul 25 18:42:43 2010 New Revision: 76341 Added: pypy/extradoc/talk/ep2010/talk/talk.pdf (contents, props changed) Log: add the pdf Added: pypy/extradoc/talk/ep2010/talk/talk.pdf ============================================================================== Binary file. No diff available. From jcreigh at codespeak.net Mon Jul 26 17:07:54 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 26 Jul 2010 17:07:54 +0200 (CEST) Subject: [pypy-svn] r76350 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/test Message-ID: <20100726150754.308B4282B9E@codespeak.net> Author: jcreigh Date: Mon Jul 26 17:07:47 2010 New Revision: 76350 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py Log: fix bug in float tests that sometimes caused a false negative depending on arbitrary set ordering Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/test/runner_test.py Mon Jul 26 17:07:47 2010 @@ -1072,9 +1072,10 @@ operations[1].fail_args = [] looptoken = LoopToken() # Use "set" to unique-ify inputargs - self.cpu.compile_loop(list(set(testcase)), operations, + unique_testcase_list = list(set(testcase)) + self.cpu.compile_loop(unique_testcase_list, operations, looptoken) - for i, box in enumerate(testcase): + for i, box in enumerate(unique_testcase_list): self.cpu.set_future_value_float(i, box.value) fail = self.cpu.execute_token(looptoken) if fail.identifier != 5 - (expected_id^expected): From fijal at codespeak.net Mon Jul 26 20:22:22 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Jul 2010 20:22:22 +0200 (CEST) Subject: [pypy-svn] r76351 - pypy/trunk/pypy/rlib Message-ID: <20100726182222.5F2F4282B9E@codespeak.net> Author: fijal Date: Mon Jul 26 20:22:19 2010 New Revision: 76351 Modified: pypy/trunk/pypy/rlib/objectmodel.py Log: Simplify @specialize. There is more code now, but most of it is simple and contains docstrings (unlike magic __getattr__ before). magic-- Modified: pypy/trunk/pypy/rlib/objectmodel.py ============================================================================== --- pypy/trunk/pypy/rlib/objectmodel.py (original) +++ pypy/trunk/pypy/rlib/objectmodel.py Mon Jul 26 20:22:19 2010 @@ -19,29 +19,66 @@ # def f(... # -class _AttachSpecialization(object): +class _Specialize(object): + def memo(self): + """ Specialize functions based on argument values. All arguments has + to be constant at the compile time. The whole function call is replaced + by a call result then. + """ + def decorated_func(func): + func._annspecialcase_ = 'memo()' + return func + return decorated_func - def __init__(self, tag): - self.tag = tag + def arg(self, *args): + """ Specialize function based on values of given positions of arguments. + They must be compile-time constants in order to work. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'args' + self._wrap(args) + return func - def __call__(self, *args): - if not args: - args = "" - else: - args = "("+','.join([repr(arg) for arg in args]) +")" - specialcase = "specialize:%s%s" % (self.tag, args) - - def specialize_decorator(func): - "NOT_RPYTHON" - func._annspecialcase_ = specialcase + return decorated_func + + def argtype(self, *args): + """ Specialize function based on types of arguments on given positions. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'argtypes' + self._wrap(args) return func - return specialize_decorator - -class _Specialize(object): + return decorated_func + + def ll(self): + """ This is version of argtypes that cares about low-level types + (so it'll get additional copies for two different types of pointers + for example). Same warnings about exponential behavior apply. + """ + def decorated_func(func): + func._annspecialcase_ = 'll()' + return func + + return decorated_func + + def ll_and_arg(self, arg): + """ XXX what does that do? + """ + def decorated_func(func): + func._annspecialcase_ = 'll_and_arg(%d)' % arg + return func + + return decorated_func - def __getattr__(self, name): - return _AttachSpecialization(name) + def _wrap(self, args): + return "("+','.join([repr(arg) for arg in args]) +")" specialize = _Specialize() From fijal at codespeak.net Mon Jul 26 20:23:59 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Jul 2010 20:23:59 +0200 (CEST) Subject: [pypy-svn] r76352 - pypy/branch/improved-asm-logging Message-ID: <20100726182359.68750282B9E@codespeak.net> Author: fijal Date: Mon Jul 26 20:23:58 2010 New Revision: 76352 Added: pypy/branch/improved-asm-logging/ (props changed) - copied from r76351, pypy/trunk/ Log: A branch to experiment with better logging what loops got executed. This might pose a problem with x86_64 merge, hence the branch From jcreigh at codespeak.net Mon Jul 26 20:26:18 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 26 Jul 2010 20:26:18 +0200 (CEST) Subject: [pypy-svn] r76353 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test Message-ID: <20100726182618.DFDBC282B9E@codespeak.net> Author: jcreigh Date: Mon Jul 26 20:26:17 2010 New Revision: 76353 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_zrpy_gc.py Log: skip hybrid GC tests on 64-bit Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_zrpy_gc.py Mon Jul 26 20:26:17 2010 @@ -17,6 +17,8 @@ from pypy.jit.backend.llsupport.gc import GcRefList, GcRootMap_asmgcc from pypy.jit.backend.llsupport.gc import GcLLDescr_framework from pypy.tool.udir import udir +from pypy.jit.backend.x86.arch import IS_X86_64 +import py.test class X(object): def __init__(self, x=0): @@ -126,6 +128,10 @@ class TestCompileHybrid(object): def setup_class(cls): + if IS_X86_64: + # No hybrid GC on 64-bit for the time being + py.test.skip() + funcs = [] name_to_func = {} for fullname in dir(cls): From fijal at codespeak.net Mon Jul 26 22:27:32 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Jul 2010 22:27:32 +0200 (CEST) Subject: [pypy-svn] r76355 - pypy/branch/improved-asm-logging/pypy/jit/backend/x86 Message-ID: <20100726202732.5E8A336C223@codespeak.net> Author: fijal Date: Mon Jul 26 22:27:30 2010 New Revision: 76355 Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py Log: Invent the loop counter, so we don't get too confused with regard to naming Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py Mon Jul 26 22:27:30 2010 @@ -118,6 +118,7 @@ self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 self.setup_failure_recovery() + self._loop_counter = 0 def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar @@ -281,7 +282,9 @@ for op in operations: if op.opnum == rop.DEBUG_MERGE_POINT: return op.args[0]._get_str() - return "" + # invent the counter, so we don't get too confused + self._loop_counter += 1 + return "" % self._loop_counter def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset From magcius at codespeak.net Tue Jul 27 00:14:12 2010 From: magcius at codespeak.net (magcius at codespeak.net) Date: Tue, 27 Jul 2010 00:14:12 +0200 (CEST) Subject: [pypy-svn] r76356 - in pypy/branch/avm/pypy/translator/avm2: . test Message-ID: <20100726221412.1AFD3282B90@codespeak.net> Author: magcius Date: Tue Jul 27 00:14:11 2010 New Revision: 76356 Added: pypy/branch/avm/pypy/translator/avm2/codegen.py (contents, props changed) pypy/branch/avm/pypy/translator/avm2/node.py pypy/branch/avm/pypy/translator/avm2/playerglobal.py pypy/branch/avm/pypy/translator/avm2/rlib.py (contents, props changed) Removed: pypy/branch/avm/pypy/translator/avm2/_playerglobal.py pypy/branch/avm/pypy/translator/avm2/avm2gen.py pypy/branch/avm/pypy/translator/avm2/query.py pypy/branch/avm/pypy/translator/avm2/test/bootstrap.py pypy/branch/avm/pypy/translator/avm2/test/simpletest.py pypy/branch/avm/pypy/translator/avm2/test/test_script.py Log: Clean up and get stuff working Added: pypy/branch/avm/pypy/translator/avm2/codegen.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/codegen.py Tue Jul 27 00:14:11 2010 @@ -0,0 +1,134 @@ +""" backend generator routines +""" + +import re + +from pypy.objspace.flow.model import Variable, Constant +from pypy.rpython.ootypesystem import ootype +from pypy.translator.avm2 import types_ as types +from pypy.translator.oosupport.metavm import Generator +from pypy.translator.oosupport.treebuilder import SubOperation +from pypy.translator.oosupport.constant import push_constant +from pypy.translator.oosupport.function import render_sub_op + +from mech.fusion.avm2 import instructions, codegen +from mech.fusion.avm2.constants import MultinameL +from mech.fusion.avm2.interfaces import LoadableAdapter, IMultiname + +from zope.interface import implementer +from zope.component import provideAdapter, adapter + + at adapter(Variable) +class VariableLoadable(LoadableAdapter): + def load(self, generator): + v = self.value + if v.concretetype is ootype.Void: + generator.push_null() + else: + generator.push_local(v) + +provideAdapter(VariableLoadable) + + at adapter(Constant) +class ConstantLoadable(LoadableAdapter): + def load(self, generator): + v = self.value + push_constant(generator.db, v.concretetype, v.value, generator) + +provideAdapter(ConstantLoadable) + + at adapter(SubOperation) +class SubOpLoadable(LoadableAdapter): + def load(self, generator): + render_sub_op(self.value, generator.db, generator) + +provideAdapter(SubOpLoadable) + +class PyPyCodeGenerator(codegen.CodeGenerator, Generator): + def __init__(self, db, abc, optimize=False): + super(PyPyCodeGenerator, self).__init__(abc, optimize=optimize) + self.db = db + self.cts = db.genoo.TypeSystem(db) + + def _get_type(self, TYPE): + t = self.cts.lltype_to_cts(TYPE) + if t: + return IMultiname(t) + return super(PyPyCodeGenerator, self)._get_type(TYPE) + + def oonewarray(self, TYPE, length=1): + super(PyPyCodeGenerator, self).new_vector(TYPE.ITEM, length) + + def store(self, v): + """ + Pop a value off the stack and store it in the variable. + """ + self.store_var(v.name) + + def push_local(self, v): + """ + Get the local occupied to "name" and push it to the stack. + """ + if self.HL(v.name): + self.push_var(v.name) + else: + self.push_null() + + def push_primitive_constant(self, TYPE, value): + if TYPE is ootype.Void: + self.push_null() + elif TYPE is ootype.String: + if value._str is None: + self.push_null() + self.downcast(types.types.string) + else: + self.load(value._str) + else: + self.load(value) + + def call_graph(self, graph, args=[]): + """ + Call a graph. + """ + self.db.pending_function(graph) + qname = self.cts.graph_to_qname(graph) + self.emit('findpropstrict', qname) + args = [a for a in args if a.concretetype is not ootype.Void] + self.load(*args) + self.emit('callproperty', qname, len(args)) + + def new(self, TYPE): + # XXX: assume no args for now + self.load(self._get_type(TYPE)) + self.I(instructions.construct(0)) + + def stringbuilder_ll_append(self, args): + self.load(*args) + self.I(instructions.add()) + self.store(args[0]) + self.push_null() + + stringbuilder_ll_append_char = stringbuilder_ll_append + + def list_ll_setitem_fast(self, args): + list, index, value = args + self.load(list) + if isinstance(index, Constant): + self.load(value) + self.set_field(index.value) + else: + self.load(index) + self.load(value) + self.set_field(MultinameL()) + # XXX: oosend expects a value to send to StoreResult + # We don't generate one, push a null. + self.push_null() + + def list_ll_getitem_fast(self, args): + list, index = args + self.load(list) + if isinstance(index, Constant): + self.get_field(index.value, list.concretetype.ITEM) + else: + self.load(index) + self.get_field(MultinameL(), list.concretetype.ITEM) Added: pypy/branch/avm/pypy/translator/avm2/node.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/node.py Tue Jul 27 00:14:11 2010 @@ -0,0 +1,57 @@ + +from pypy.translator.avm2 import types_ as types + +from mech.fusion.avm2.traits import AbcSlotTrait +from mech.fusion.avm2.constants import packagedQName +from mech.fusion.avm2.interfaces import IMultiname + +from zope.interface import implementer +from zope.component import adapter, provideAdapter + +class Node(object): + def get_name(self): + pass + + def dependencies(self): + pass + + def render(self, ilasm): + pass + +class ClassNodeBase(Node): + def get_fields(self): + pass + + def get_base_class(self): + pass + + def render(self, ilasm): + ilasm.begin_class(IMultiname(self), self.get_base_class()) + for f_name, (f_type, f_default) in self.get_fields(): + cts_type = self.cts.lltype_to_cts(f_type) + f_name = self.cts.escape_name(f_name) + if cts_type != types.types.void: + slot = AbcSlotTrait(IMultiname(f_name), IMultiname(cts_type)) + ilasm.context.add_instance_trait(slot) + + self.render_ctor(ilasm) + self.render_toString(ilasm) + self.render_methods(ilasm) + ilasm.exit_context() + + def render_ctor(self, ilasm): + pass + + def render_toString(self, ilasm): + pass + + def render_methods(self, ilasm): + pass + + + at implementer(IMultiname) + at adapter(ClassNodeBase) +def _adapter(self): + return IMultiname(self.get_type()) + +provideAdapter(_adapter) Added: pypy/branch/avm/pypy/translator/avm2/playerglobal.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/playerglobal.py Tue Jul 27 00:14:11 2010 @@ -0,0 +1,4 @@ +import sys +from pypy.translator.avm2.library import get_playerglobal +playerglobal = get_playerglobal() +sys.modules[__name__] = playerglobal Added: pypy/branch/avm/pypy/translator/avm2/rlib.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/rlib.py Tue Jul 27 00:14:11 2010 @@ -0,0 +1,139 @@ + +import inspect +import re + +from mech.fusion.avm2.codegen import Argument +from mech.fusion.avm2 import constants + +from pypy.rpython.ootypesystem import ootype +from pypy.objspace.flow import model as flowmodel +from pypy.translator.avm2.node import Node +from pypy.translator.avm2 import types_ as types + +from types import MethodType + +ESCAPE_FOR_REGEX = re.compile(r"([{}\(\)\^$&.\*\?\/\+\|\[\\\\]|\]|\-)") + +class FunctionNode(Node): + INSTANCE = None + def render(self, generator): + generator.begin_method(self.name, self.argspec, self.rettype) + self.fn(self.cls, generator, *[Argument(name) for t, name in self.argspec]) + if self.rettype != constants.QName("void"): + generator.return_value() + generator.end_method() + +def make_node(cls, fn, name, argtypes, rettype): + if getattr(fn, "node", None): + return fn.node + fn.node = node = FunctionNode() + node.cls = cls + node.fn = fn + node.name = name + node.argspec = zip(argtypes, inspect.getargspec(fn).args[2:]) + node.rettype = rettype + return node + +def export(argtypes, rettype): + def inner(fn): + def inner_(cls, asm, *args): + name = constants.packagedQName(cls.PACKAGE, fn.func_name) + asm.db.pending_node(make_node(cls, fn, name, argtypes, rettype)) + asm.emit('findpropstrict', name) + asm.load(*args) + asm.emit('callproperty', name, len(args)) + return classmethod(inner_) + return inner + +class RString(object): + WRAPPING = ootype.AbstractString + PACKAGE = "pypy.lib.str" + + @export(['String', 'String', 'int', 'int'], 'int') + def ll_find(cls, asm, string, substr, start, end): + asm.load(start, string) + asm.get_field("length") + asm.branch_if_greater_than("return -1") + asm.load(string, 0, end) + asm.call_method("slice", 2) + asm.load(substr, start) + asm.call_method("indexOf", 2) + asm.return_value() + asm.set_label("return -1") + asm.load(-1) + + ll_find_char = ll_find + + @export(['String', 'String', 'int', 'int'], 'int') + def ll_rfind(cls, asm, string, substr, start, end): + asm.load(start, string) + asm.get_field("length") + asm.branch_if_greater_than("return -1") + asm.load(string, start, end) + asm.call_method("slice", 2) + asm.load(substr) + asm.call_method("lastIndexOf", 1) + asm.dup() + asm.load(-1) + asm.branch_if_not_equal("return value") + asm.pop() + asm.set_label("return -1") + asm.load(-1) + asm.return_value() + asm.set_label("return value") + asm.load(start) + asm.emit('add_i') + asm.return_value() + + ll_rfind_char = ll_rfind + + @classmethod + def escape_for_regex(cls, asm, string): + if isinstance(string, flowmodel.Constant): + asm.load(ESCAPE_FOR_REGEX.sub(r"\\\1", string.value)) + else: + asm.emit('findpropstrict', constants.QName("RegExp")) + asm.load(string, ESCAPE_FOR_REGEX.pattern, "g") + asm.emit('constructprop', constants.QName("RegExp"), 2) + asm.load(r"\$1") + asm.call_method("replace", 2) + + @export(['String', 'String', 'int', 'int'], 'int') + def ll_count(cls, asm, string, substr, start, end): + asm.emit('findpropstrict', constants.QName("RegExp")) + cls.escape_for_regex(substr) + asm.load("g") + asm.emit('constructprop', constants.QName("RegExp"), 2) + asm.load(string, start, end) + asm.call_method("slice", 2) + asm.call_method("exec", 1) + asm.get_field("length") + + ll_count_char = ll_count + + @export(['String', 'String'], 'Boolean') + def ll_endswith(cls, asm, string, substr): + asm.load(substr, string, substr, substr) + asm.branch_if_false("rettrue") + asm.get_field("length") + asm.emit('negate_i') + asm.call_method("slice", 1) + asm.emit('equals') + asm.return_value() + asm.set_label("rettrue") + asm.load(True) + + @export(['String', 'String'], 'Boolean') + def ll_startswith(cls, asm, string, substr): + asm.load(substr, string, 0, substr) + asm.get_field("length") + asm.call_method("slice", 2) + asm.emit('equals') + +RLib = {} + +for cls in [RString]: + for name in dir(cls): + meth = getattr(cls, name) + if isinstance(meth, MethodType): + RLib[cls.WRAPPING.oopspec_name+'_'+name] = meth From magcius at codespeak.net Tue Jul 27 00:47:13 2010 From: magcius at codespeak.net (magcius at codespeak.net) Date: Tue, 27 Jul 2010 00:47:13 +0200 (CEST) Subject: [pypy-svn] r76357 - pypy/branch/avm/py Message-ID: <20100726224713.91A35282B90@codespeak.net> Author: magcius Date: Tue Jul 27 00:47:11 2010 New Revision: 76357 Added: pypy/branch/avm/py/ (props changed) - copied from r76356, pypy/trunk/py/ Log: From fijal at codespeak.net Tue Jul 27 13:39:18 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Jul 2010 13:39:18 +0200 (CEST) Subject: [pypy-svn] r76359 - pypy/trunk/lib-python Message-ID: <20100727113918.D25EA282BDE@codespeak.net> Author: fijal Date: Tue Jul 27 13:39:16 2010 New Revision: 76359 Modified: pypy/trunk/lib-python/conftest.py Log: don't skip ctypes tests Modified: pypy/trunk/lib-python/conftest.py ============================================================================== --- pypy/trunk/lib-python/conftest.py (original) +++ pypy/trunk/lib-python/conftest.py Tue Jul 27 13:39:16 2010 @@ -464,11 +464,7 @@ RegrTest('test_coding.py'), RegrTest('test_complex_args.py'), RegrTest('test_contextlib.py', usemodules="thread"), - # we skip test ctypes, since we adapted it massively in order - # to test what we want to support. There are real failures, - # but it's about missing features that we don't want to support - # now - RegrTest('test_ctypes.py', skip="we have a replacement"), + RegrTest('test_ctypes.py'), RegrTest('test_defaultdict.py'), RegrTest('test_email_renamed.py'), RegrTest('test_exception_variations.py'), From fijal at codespeak.net Tue Jul 27 13:42:54 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Jul 2010 13:42:54 +0200 (CEST) Subject: [pypy-svn] r76360 - pypy/trunk/lib-python Message-ID: <20100727114254.DC505282BDE@codespeak.net> Author: fijal Date: Tue Jul 27 13:42:53 2010 New Revision: 76360 Modified: pypy/trunk/lib-python/conftest.py Log: At least _rawffi is needed, possibly more, checking Modified: pypy/trunk/lib-python/conftest.py ============================================================================== --- pypy/trunk/lib-python/conftest.py (original) +++ pypy/trunk/lib-python/conftest.py Tue Jul 27 13:42:53 2010 @@ -464,7 +464,7 @@ RegrTest('test_coding.py'), RegrTest('test_complex_args.py'), RegrTest('test_contextlib.py', usemodules="thread"), - RegrTest('test_ctypes.py'), + RegrTest('test_ctypes.py', usemodules="_rawffi"), RegrTest('test_defaultdict.py'), RegrTest('test_email_renamed.py'), RegrTest('test_exception_variations.py'), From antocuni at codespeak.net Tue Jul 27 14:05:57 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 27 Jul 2010 14:05:57 +0200 (CEST) Subject: [pypy-svn] r76361 - in pypy/trunk/pypy/translator/goal: . test2 Message-ID: <20100727120557.DF80A282BDE@codespeak.net> Author: antocuni Date: Tue Jul 27 14:05:49 2010 New Revision: 76361 Modified: pypy/trunk/pypy/translator/goal/app_main.py pypy/trunk/pypy/translator/goal/test2/test_app_main.py Log: (Alex Morega, Andrei Laza, antocuni) don't import a site.py that it's in the current directory Modified: pypy/trunk/pypy/translator/goal/app_main.py ============================================================================== --- pypy/trunk/pypy/translator/goal/app_main.py (original) +++ pypy/trunk/pypy/translator/goal/app_main.py Tue Jul 27 14:05:49 2010 @@ -223,7 +223,6 @@ path = os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath - newpath.insert(0, '') # remove duplicates _seen = {} del sys.path[:] @@ -327,6 +326,10 @@ except: print >> sys.stderr, "'import site' failed" + # update sys.path *after* loading site.py, in case there is a + # "site.py" file in the script's directory. + sys.path.insert(0, '') + if warnoptions: sys.warnoptions.append(warnoptions) from warnings import _processoptions Modified: pypy/trunk/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/trunk/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/trunk/pypy/translator/goal/test2/test_app_main.py Tue Jul 27 14:05:49 2010 @@ -5,6 +5,7 @@ import sys, os, re import autopath from pypy.tool.udir import udir +from contextlib import contextmanager banner = sys.version.splitlines()[0] @@ -326,8 +327,9 @@ class TestNonInteractive: def run(self, cmdline, senddata='', expect_prompt=False, - expect_banner=False): - cmdline = '%s "%s" %s' % (sys.executable, app_main, cmdline) + expect_banner=False, python_flags=''): + cmdline = '%s %s "%s" %s' % (sys.executable, python_flags, + app_main, cmdline) print 'POPEN:', cmdline child_in, child_out_err = os.popen4(cmdline) child_in.write(senddata) @@ -449,6 +451,43 @@ assert data == '\x00(STDOUT)\n\x00' # from stdout child_out_err.close() + def test_proper_sys_path(self, tmpdir): + + @contextmanager + def chdir_and_unset_pythonpath(new_cwd): + old_cwd = new_cwd.chdir() + old_pythonpath = os.getenv('PYTHONPATH') + os.unsetenv('PYTHONPATH') + try: + yield + finally: + old_cwd.chdir() + os.putenv('PYTHONPATH', old_pythonpath) + + tmpdir.join('site.py').write('print "SHOULD NOT RUN"') + runme_py = tmpdir.join('runme.py') + runme_py.write('print "some text"') + + cmdline = str(runme_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline, python_flags='-S') + + assert data == "some text\n" + + runme2_py = tmpdir.mkdir('otherpath').join('runme2.py') + runme2_py.write('print "some new text"\n' + 'import sys\n' + 'print sys.path\n') + + cmdline2 = str(runme2_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline2, python_flags='-S') + + assert data.startswith("some new text\n") + assert repr(str(tmpdir.join('otherpath'))) in data + class AppTestAppMain: From jcreigh at codespeak.net Tue Jul 27 14:21:00 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 27 Jul 2010 14:21:00 +0200 (CEST) Subject: [pypy-svn] r76362 - in pypy/branch/x86-64-jit-backend: . lib-python lib_pypy lib_pypy/pypy_test pypy/jit/metainterp pypy/module/_rawffi/test pypy/rlib pypy/rpython/lltypesystem pypy/translator/goal pypy/translator/goal/test2 Message-ID: <20100727122100.37388282BDE@codespeak.net> Author: jcreigh Date: Tue Jul 27 14:20:58 2010 New Revision: 76362 Modified: pypy/branch/x86-64-jit-backend/ (props changed) pypy/branch/x86-64-jit-backend/lib-python/conftest.py pypy/branch/x86-64-jit-backend/lib_pypy/datetime.py pypy/branch/x86-64-jit-backend/lib_pypy/pypy_test/test_datetime.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/executor.py pypy/branch/x86-64-jit-backend/pypy/module/_rawffi/test/test__rawffi.py pypy/branch/x86-64-jit-backend/pypy/rlib/objectmodel.py pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rstr.py pypy/branch/x86-64-jit-backend/pypy/translator/goal/app_main.py pypy/branch/x86-64-jit-backend/pypy/translator/goal/test2/test_app_main.py Log: merged changes from trunk through r76361 Modified: pypy/branch/x86-64-jit-backend/lib-python/conftest.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/conftest.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/conftest.py Tue Jul 27 14:20:58 2010 @@ -464,11 +464,7 @@ RegrTest('test_coding.py'), RegrTest('test_complex_args.py'), RegrTest('test_contextlib.py', usemodules="thread"), - # we skip test ctypes, since we adapted it massively in order - # to test what we want to support. There are real failures, - # but it's about missing features that we don't want to support - # now - RegrTest('test_ctypes.py', skip="we have a replacement"), + RegrTest('test_ctypes.py', usemodules="_rawffi"), RegrTest('test_defaultdict.py'), RegrTest('test_email_renamed.py'), RegrTest('test_exception_variations.py'), Modified: pypy/branch/x86-64-jit-backend/lib_pypy/datetime.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib_pypy/datetime.py (original) +++ pypy/branch/x86-64-jit-backend/lib_pypy/datetime.py Tue Jul 27 14:20:58 2010 @@ -1412,7 +1412,7 @@ def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - if 1 - (t % 1.0) < 0.000001: + if 1 - (t % 1.0) < 0.0000005: t = float(int(t)) + 1 if t < 0: t -= 1 Modified: pypy/branch/x86-64-jit-backend/lib_pypy/pypy_test/test_datetime.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib_pypy/pypy_test/test_datetime.py (original) +++ pypy/branch/x86-64-jit-backend/lib_pypy/pypy_test/test_datetime.py Tue Jul 27 14:20:58 2010 @@ -15,4 +15,18 @@ expected = datetime.datetime(*(time.strptime(string, format)[0:6])) got = datetime.datetime.strptime(string, format) assert expected == got + +def test_datetime_rounding(): + b = 0.0000001 + a = 0.9999994 + + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 + assert datetime.datetime.utcfromtimestamp(a).second == 1 + Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/executor.py Tue Jul 27 14:20:58 2010 @@ -91,7 +91,7 @@ return BoxInt(cpu.bh_getarrayitem_gc_i(arraydescr, array, index)) def do_getarrayitem_raw(cpu, _, arraybox, indexbox, arraydescr): - array = arraybox.getref_base() + array = arraybox.getint() index = indexbox.getint() assert not arraydescr.is_array_of_pointers() if arraydescr.is_array_of_floats(): Modified: pypy/branch/x86-64-jit-backend/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_rawffi/test/test__rawffi.py Tue Jul 27 14:20:58 2010 @@ -677,7 +677,12 @@ a = A(1) a[0] = -1234 a.free() - + + def test_long_with_fromaddress(self): + import _rawffi + addr = -1 + raises(ValueError, _rawffi.Array('u').fromaddress, addr, 100) + def test_passing_raw_pointers(self): import _rawffi lib = _rawffi.CDLL(self.lib_name) Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/objectmodel.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/objectmodel.py Tue Jul 27 14:20:58 2010 @@ -19,29 +19,66 @@ # def f(... # -class _AttachSpecialization(object): +class _Specialize(object): + def memo(self): + """ Specialize functions based on argument values. All arguments has + to be constant at the compile time. The whole function call is replaced + by a call result then. + """ + def decorated_func(func): + func._annspecialcase_ = 'memo()' + return func + return decorated_func - def __init__(self, tag): - self.tag = tag + def arg(self, *args): + """ Specialize function based on values of given positions of arguments. + They must be compile-time constants in order to work. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'args' + self._wrap(args) + return func - def __call__(self, *args): - if not args: - args = "" - else: - args = "("+','.join([repr(arg) for arg in args]) +")" - specialcase = "specialize:%s%s" % (self.tag, args) - - def specialize_decorator(func): - "NOT_RPYTHON" - func._annspecialcase_ = specialcase + return decorated_func + + def argtype(self, *args): + """ Specialize function based on types of arguments on given positions. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'argtypes' + self._wrap(args) return func - return specialize_decorator - -class _Specialize(object): + return decorated_func + + def ll(self): + """ This is version of argtypes that cares about low-level types + (so it'll get additional copies for two different types of pointers + for example). Same warnings about exponential behavior apply. + """ + def decorated_func(func): + func._annspecialcase_ = 'll()' + return func + + return decorated_func + + def ll_and_arg(self, arg): + """ XXX what does that do? + """ + def decorated_func(func): + func._annspecialcase_ = 'll_and_arg(%d)' % arg + return func + + return decorated_func - def __getattr__(self, name): - return _AttachSpecialization(name) + def _wrap(self, args): + return "("+','.join([repr(arg) for arg in args]) +")" specialize = _Specialize() Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rstr.py Tue Jul 27 14:20:58 2010 @@ -674,6 +674,7 @@ res_index += item_len i += 1 return result + ll_join_strs._annenforceargs_ = [int, None] def ll_join_chars(length, chars): # no need to optimize this, will be replaced by string builder Modified: pypy/branch/x86-64-jit-backend/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/goal/app_main.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/goal/app_main.py Tue Jul 27 14:20:58 2010 @@ -223,7 +223,6 @@ path = os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath - newpath.insert(0, '') # remove duplicates _seen = {} del sys.path[:] @@ -327,6 +326,10 @@ except: print >> sys.stderr, "'import site' failed" + # update sys.path *after* loading site.py, in case there is a + # "site.py" file in the script's directory. + sys.path.insert(0, '') + if warnoptions: sys.warnoptions.append(warnoptions) from warnings import _processoptions Modified: pypy/branch/x86-64-jit-backend/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/goal/test2/test_app_main.py Tue Jul 27 14:20:58 2010 @@ -5,6 +5,7 @@ import sys, os, re import autopath from pypy.tool.udir import udir +from contextlib import contextmanager banner = sys.version.splitlines()[0] @@ -326,8 +327,9 @@ class TestNonInteractive: def run(self, cmdline, senddata='', expect_prompt=False, - expect_banner=False): - cmdline = '%s "%s" %s' % (sys.executable, app_main, cmdline) + expect_banner=False, python_flags=''): + cmdline = '%s %s "%s" %s' % (sys.executable, python_flags, + app_main, cmdline) print 'POPEN:', cmdline child_in, child_out_err = os.popen4(cmdline) child_in.write(senddata) @@ -449,6 +451,43 @@ assert data == '\x00(STDOUT)\n\x00' # from stdout child_out_err.close() + def test_proper_sys_path(self, tmpdir): + + @contextmanager + def chdir_and_unset_pythonpath(new_cwd): + old_cwd = new_cwd.chdir() + old_pythonpath = os.getenv('PYTHONPATH') + os.unsetenv('PYTHONPATH') + try: + yield + finally: + old_cwd.chdir() + os.putenv('PYTHONPATH', old_pythonpath) + + tmpdir.join('site.py').write('print "SHOULD NOT RUN"') + runme_py = tmpdir.join('runme.py') + runme_py.write('print "some text"') + + cmdline = str(runme_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline, python_flags='-S') + + assert data == "some text\n" + + runme2_py = tmpdir.mkdir('otherpath').join('runme2.py') + runme2_py.write('print "some new text"\n' + 'import sys\n' + 'print sys.path\n') + + cmdline2 = str(runme2_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline2, python_flags='-S') + + assert data.startswith("some new text\n") + assert repr(str(tmpdir.join('otherpath'))) in data + class AppTestAppMain: From fijal at codespeak.net Tue Jul 27 14:38:37 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Jul 2010 14:38:37 +0200 (CEST) Subject: [pypy-svn] r76363 - pypy/trunk/pypy/rlib Message-ID: <20100727123837.A6C5A282C04@codespeak.net> Author: fijal Date: Tue Jul 27 14:38:36 2010 New Revision: 76363 Modified: pypy/trunk/pypy/rlib/objectmodel.py Log: fix typos Modified: pypy/trunk/pypy/rlib/objectmodel.py ============================================================================== --- pypy/trunk/pypy/rlib/objectmodel.py (original) +++ pypy/trunk/pypy/rlib/objectmodel.py Tue Jul 27 14:38:36 2010 @@ -39,7 +39,7 @@ exponential behavior!). """ def decorated_func(func): - func._annspecialcase_ = 'args' + self._wrap(args) + func._annspecialcase_ = 'arg' + self._wrap(args) return func return decorated_func @@ -52,7 +52,7 @@ exponential behavior!). """ def decorated_func(func): - func._annspecialcase_ = 'argtypes' + self._wrap(args) + func._annspecialcase_ = 'argtype' + self._wrap(args) return func return decorated_func From fijal at codespeak.net Tue Jul 27 14:42:45 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Jul 2010 14:42:45 +0200 (CEST) Subject: [pypy-svn] r76364 - pypy/trunk/pypy/rlib Message-ID: <20100727124245.568D4282BDE@codespeak.net> Author: fijal Date: Tue Jul 27 14:42:43 2010 New Revision: 76364 Modified: pypy/trunk/pypy/rlib/objectmodel.py Log: Oops, total nonsense. Modified: pypy/trunk/pypy/rlib/objectmodel.py ============================================================================== --- pypy/trunk/pypy/rlib/objectmodel.py (original) +++ pypy/trunk/pypy/rlib/objectmodel.py Tue Jul 27 14:42:43 2010 @@ -26,7 +26,7 @@ by a call result then. """ def decorated_func(func): - func._annspecialcase_ = 'memo()' + func._annspecialcase_ = 'specialize:memo()' return func return decorated_func @@ -39,7 +39,7 @@ exponential behavior!). """ def decorated_func(func): - func._annspecialcase_ = 'arg' + self._wrap(args) + func._annspecialcase_ = 'specialize:arg' + self._wrap(args) return func return decorated_func @@ -52,7 +52,7 @@ exponential behavior!). """ def decorated_func(func): - func._annspecialcase_ = 'argtype' + self._wrap(args) + func._annspecialcase_ = 'specialize:argtype' + self._wrap(args) return func return decorated_func @@ -63,7 +63,7 @@ for example). Same warnings about exponential behavior apply. """ def decorated_func(func): - func._annspecialcase_ = 'll()' + func._annspecialcase_ = 'specialize:ll()' return func return decorated_func @@ -72,7 +72,7 @@ """ XXX what does that do? """ def decorated_func(func): - func._annspecialcase_ = 'll_and_arg(%d)' % arg + func._annspecialcase_ = 'specialize:ll_and_arg(%d)' % arg return func return decorated_func From fijal at codespeak.net Tue Jul 27 15:29:00 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Jul 2010 15:29:00 +0200 (CEST) Subject: [pypy-svn] r76365 - in pypy/branch/improved-asm-logging/pypy/jit/backend/x86: . test Message-ID: <20100727132900.7920F282BDE@codespeak.net> Author: fijal Date: Tue Jul 27 15:28:56 2010 New Revision: 76365 Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py Log: An attempt to write down counter for how many times the loop has run Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py Tue Jul 27 15:28:56 2010 @@ -1,4 +1,4 @@ -import sys +import sys, os from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT,\ @@ -113,18 +113,25 @@ self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 + self.loop_run_counter = values_array(lltype.Signed, 10000) + self.loop_names = [] + # if we have 10000 loops, we have some other problems I guess self.loc_float_const_neg = None self.loc_float_const_abs = None self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 self.setup_failure_recovery() self._loop_counter = 0 + self._debug = False def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar llop.gc_assume_young_pointers(lltype.Void, llmemory.cast_ptr_to_adr(ptrs)) + def set_debug(self, v): + self._debug = v + def make_sure_mc_exists(self): if self.mc is None: # the address of the function called by 'new' @@ -158,6 +165,9 @@ self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + s = os.environ.get('PYPYJITLOG') + if s: + self.set_debug(True) def _build_float_constants(self): # 11 words: 8 words for the data, and up to 3 words for alignment @@ -279,12 +289,18 @@ self.mc.end_function() def _find_debug_merge_point(self, operations): + for op in operations: if op.opnum == rop.DEBUG_MERGE_POINT: - return op.args[0]._get_str() + funcname = op.args[0]._get_str() + break + else: + funcname = "" % self._loop_counter # invent the counter, so we don't get too confused + if self._debug: + self.loop_names.append(funcname) self._loop_counter += 1 - return "" % self._loop_counter + return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset @@ -295,6 +311,15 @@ def _assemble(self, regalloc, operations): self._regalloc = regalloc + if self._debug: + # before doing anything, let's increase a counter + # we need one register free (a bit of a hack, but whatever) + self.mc.PUSH(eax) + adr = self.loop_run_counter.get_addr_for_num(self._loop_counter - 1) + self.mc.MOV(eax, heap(adr)) + self.mc.ADD(eax, imm(1)) + self.mc.MOV(heap(adr), eax) + self.mc.POP(eax) regalloc.walk_operations(operations) self.mc.done() self.mc2.done() Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py Tue Jul 27 15:28:56 2010 @@ -9,6 +9,7 @@ from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.executor import execute from pypy.jit.backend.test.runner_test import LLtypeBackendTest +from pypy.jit.metainterp.test.oparser import parse import ctypes import sys @@ -390,3 +391,25 @@ self.cpu.set_future_value_int(0, base_v.value) self.cpu.execute_token(looptoken) assert self.cpu.get_latest_value_int(0) == 1024 + +class TestDebuggingAssembler(object): + def setup_method(self, meth): + self.cpu = CPU(rtyper=None, stats=FakeStats()) + + def test_debugger_on(self): + loop = """ + [i0] + debug_merge_point('xyz') + i1 = int_add(i0, 1) + i2 = int_ge(i1, 10) + guard_false(i2) [] + jump(i1) + """ + ops = parse(loop) + self.cpu.assembler.set_debug(True) + self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) + self.cpu.set_future_value_int(0, 0) + self.cpu.execute_token(ops.token) + # check debugging info + assert self.cpu.assembler.loop_names == ["xyz"] + assert self.cpu.assembler.loop_run_counter.getitem(0) == 10 From jcreigh at codespeak.net Tue Jul 27 15:34:34 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 27 Jul 2010 15:34:34 +0200 (CEST) Subject: [pypy-svn] r76366 - in pypy/branch/x86-64-jit-backend: . pypy/rlib Message-ID: <20100727133434.8C763282BDE@codespeak.net> Author: jcreigh Date: Tue Jul 27 15:34:32 2010 New Revision: 76366 Modified: pypy/branch/x86-64-jit-backend/ (props changed) pypy/branch/x86-64-jit-backend/pypy/rlib/objectmodel.py Log: merged changes from trunk through r76365 Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/objectmodel.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/objectmodel.py Tue Jul 27 15:34:32 2010 @@ -26,7 +26,7 @@ by a call result then. """ def decorated_func(func): - func._annspecialcase_ = 'memo()' + func._annspecialcase_ = 'specialize:memo()' return func return decorated_func @@ -39,7 +39,7 @@ exponential behavior!). """ def decorated_func(func): - func._annspecialcase_ = 'args' + self._wrap(args) + func._annspecialcase_ = 'specialize:arg' + self._wrap(args) return func return decorated_func @@ -52,7 +52,7 @@ exponential behavior!). """ def decorated_func(func): - func._annspecialcase_ = 'argtypes' + self._wrap(args) + func._annspecialcase_ = 'specialize:argtype' + self._wrap(args) return func return decorated_func @@ -63,7 +63,7 @@ for example). Same warnings about exponential behavior apply. """ def decorated_func(func): - func._annspecialcase_ = 'll()' + func._annspecialcase_ = 'specialize:ll()' return func return decorated_func @@ -72,7 +72,7 @@ """ XXX what does that do? """ def decorated_func(func): - func._annspecialcase_ = 'll_and_arg(%d)' % arg + func._annspecialcase_ = 'specialize:ll_and_arg(%d)' % arg return func return decorated_func From fijal at codespeak.net Tue Jul 27 15:39:07 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Jul 2010 15:39:07 +0200 (CEST) Subject: [pypy-svn] r76367 - in pypy/benchmarks: . own/twisted Message-ID: <20100727133907.DB8BF282BDE@codespeak.net> Author: fijal Date: Tue Jul 27 15:39:06 2010 New Revision: 76367 Modified: pypy/benchmarks/benchmarks.py pypy/benchmarks/own/twisted/benchlib.py Log: Increase default number of iterations for twisted, but decrease default number of seconds per iteration. This should make it more robust against running out of TCP connections. Run accepts Modified: pypy/benchmarks/benchmarks.py ============================================================================== --- pypy/benchmarks/benchmarks.py (original) +++ pypy/benchmarks/benchmarks.py Tue Jul 27 15:39:06 2010 @@ -46,11 +46,8 @@ for name in ['float', 'nbody_modified', 'meteor-contest', 'fannkuch', 'spectral-norm', 'chaos', 'telco']: _register_new_bm(name, name, globals(), **opts.get(name, {})) -for name in ['web', 'names', 'iteration', 'tcp', 'pb']:#, 'accepts']: - if name == 'web': - iteration_scaling = .07 - else: - iteration_scaling = .20 +for name in ['web', 'names', 'iteration', 'tcp', 'pb', 'accepts']: + iteration_scaling = 1.0 _register_new_bm_twisted(name, 'twisted_' + name, globals(), bm_env={'PYTHONPATH': ':'.join(TWISTED)}, iteration_scaling=iteration_scaling) Modified: pypy/benchmarks/own/twisted/benchlib.py ============================================================================== --- pypy/benchmarks/own/twisted/benchlib.py (original) +++ pypy/benchmarks/own/twisted/benchlib.py Tue Jul 27 15:39:06 2010 @@ -53,7 +53,7 @@ class BenchmarkOptions(Options): optParameters = [ ('iterations', 'n', 1, 'number of iterations', int), - ('duration', 'd', 5, 'duration of each iteration', float), + ('duration', 'd', 1, 'duration of each iteration', float), ('warmup', 'w', 3, 'number of warmup iterations', int), ] From fijal at codespeak.net Tue Jul 27 16:15:23 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Jul 2010 16:15:23 +0200 (CEST) Subject: [pypy-svn] r76368 - in pypy/branch/improved-asm-logging/pypy/jit/backend/x86: . test Message-ID: <20100727141523.E1F9F282BDE@codespeak.net> Author: fijal Date: Tue Jul 27 16:15:22 2010 New Revision: 76368 Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py pypy/branch/improved-asm-logging/pypy/jit/backend/x86/runner.py pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py Log: Dump statistics to a file if PYPYLOG is set Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py Tue Jul 27 16:15:22 2010 @@ -17,6 +17,7 @@ from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print from pypy.rlib import rgc +from pypy.rlib.streamio import open_file_as_stream # our calling convention - we pass first 6 args in registers # and the rest stays on the stack @@ -99,6 +100,7 @@ mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE _float_constants = None _regalloc = None + _output_loop_log = None def __init__(self, cpu, translate_support_code=False, failargs_limit=1000): @@ -165,9 +167,19 @@ self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() - s = os.environ.get('PYPYJITLOG') + s = os.environ.get('PYPYLOG') if s: self.set_debug(True) + self._output_loop_log = s + ".count" + + def finish_once(self): + if self._debug: + assert self._output_loop_log is not None + f = open_file_as_stream(self._output_loop_log, "w") + for i in range(self._loop_counter): + f.write(self.loop_names[i] + ":" + + str(self.loop_run_counter.getitem(i)) + "\n") + f.close() def _build_float_constants(self): # 11 words: 8 words for the data, and up to 3 words for alignment Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/improved-asm-logging/pypy/jit/backend/x86/runner.py Tue Jul 27 16:15:22 2010 @@ -44,6 +44,7 @@ self.profile_agent.startup() def finish_once(self): + self.assembler.finish_once() self.profile_agent.shutdown() def compile_loop(self, inputargs, operations, looptoken): Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py Tue Jul 27 16:15:22 2010 @@ -10,8 +10,10 @@ from pypy.jit.metainterp.executor import execute from pypy.jit.backend.test.runner_test import LLtypeBackendTest from pypy.jit.metainterp.test.oparser import parse +from pypy.tool.udir import udir import ctypes import sys +import os class FakeStats(object): pass @@ -394,8 +396,15 @@ class TestDebuggingAssembler(object): def setup_method(self, meth): + self.pypylog = os.environ.get('PYPYLOG', None) + self.logfile = str(udir.join('x86_runner.log')) + os.environ['PYPYLOG'] = self.logfile self.cpu = CPU(rtyper=None, stats=FakeStats()) + def teardown_method(self, meth): + if self.pypylog is not None: + os.environ['PYPYLOG'] = self.pypylog + def test_debugger_on(self): loop = """ [i0] @@ -413,3 +422,6 @@ # check debugging info assert self.cpu.assembler.loop_names == ["xyz"] assert self.cpu.assembler.loop_run_counter.getitem(0) == 10 + self.cpu.finish_once() + lines = py.path.local(self.logfile + ".count").readlines() + assert lines[0] == 'xyz:10\n' From fijal at codespeak.net Tue Jul 27 16:24:43 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Jul 2010 16:24:43 +0200 (CEST) Subject: [pypy-svn] r76369 - pypy/branch/improved-asm-logging/pypy/rlib Message-ID: <20100727142443.0B7D0282BDE@codespeak.net> Author: fijal Date: Tue Jul 27 16:24:41 2010 New Revision: 76369 Modified: pypy/branch/improved-asm-logging/pypy/rlib/objectmodel.py Log: Merge 76363:76368 from trunk Modified: pypy/branch/improved-asm-logging/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/rlib/objectmodel.py (original) +++ pypy/branch/improved-asm-logging/pypy/rlib/objectmodel.py Tue Jul 27 16:24:41 2010 @@ -26,7 +26,7 @@ by a call result then. """ def decorated_func(func): - func._annspecialcase_ = 'memo()' + func._annspecialcase_ = 'specialize:memo()' return func return decorated_func @@ -39,7 +39,7 @@ exponential behavior!). """ def decorated_func(func): - func._annspecialcase_ = 'args' + self._wrap(args) + func._annspecialcase_ = 'specialize:arg' + self._wrap(args) return func return decorated_func @@ -52,7 +52,7 @@ exponential behavior!). """ def decorated_func(func): - func._annspecialcase_ = 'argtypes' + self._wrap(args) + func._annspecialcase_ = 'specialize:argtype' + self._wrap(args) return func return decorated_func @@ -63,7 +63,7 @@ for example). Same warnings about exponential behavior apply. """ def decorated_func(func): - func._annspecialcase_ = 'll()' + func._annspecialcase_ = 'specialize:ll()' return func return decorated_func @@ -72,7 +72,7 @@ """ XXX what does that do? """ def decorated_func(func): - func._annspecialcase_ = 'll_and_arg(%d)' % arg + func._annspecialcase_ = 'specialize:ll_and_arg(%d)' % arg return func return decorated_func From fijal at codespeak.net Tue Jul 27 16:58:08 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Jul 2010 16:58:08 +0200 (CEST) Subject: [pypy-svn] r76370 - pypy/benchmarks Message-ID: <20100727145808.D6240282BDE@codespeak.net> Author: fijal Date: Tue Jul 27 16:58:07 2010 New Revision: 76370 Modified: pypy/benchmarks/benchmarks.py Log: Disable accepts Modified: pypy/benchmarks/benchmarks.py ============================================================================== --- pypy/benchmarks/benchmarks.py (original) +++ pypy/benchmarks/benchmarks.py Tue Jul 27 16:58:07 2010 @@ -46,7 +46,7 @@ for name in ['float', 'nbody_modified', 'meteor-contest', 'fannkuch', 'spectral-norm', 'chaos', 'telco']: _register_new_bm(name, name, globals(), **opts.get(name, {})) -for name in ['web', 'names', 'iteration', 'tcp', 'pb', 'accepts']: +for name in ['web', 'names', 'iteration', 'tcp', 'pb']:#, 'accepts']: iteration_scaling = 1.0 _register_new_bm_twisted(name, 'twisted_' + name, globals(), bm_env={'PYTHONPATH': ':'.join(TWISTED)}, From fijal at codespeak.net Tue Jul 27 16:59:42 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Jul 2010 16:59:42 +0200 (CEST) Subject: [pypy-svn] r76371 - pypy/branch/improved-asm-logging/pypy/jit/backend/x86 Message-ID: <20100727145942.71FBD282BDE@codespeak.net> Author: fijal Date: Tue Jul 27 16:59:40 2010 New Revision: 76371 Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py Log: fix translation (bummer) Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py Tue Jul 27 16:59:40 2010 @@ -174,8 +174,9 @@ def finish_once(self): if self._debug: - assert self._output_loop_log is not None - f = open_file_as_stream(self._output_loop_log, "w") + output_log = self._output_loop_log + assert output_log is not None + f = open_file_as_stream(output_log, "w") for i in range(self._loop_counter): f.write(self.loop_names[i] + ":" + str(self.loop_run_counter.getitem(i)) + "\n") From fijal at codespeak.net Tue Jul 27 19:00:01 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Jul 2010 19:00:01 +0200 (CEST) Subject: [pypy-svn] r76373 - pypy/branch/improved-asm-logging/pypy/jit/backend/x86 Message-ID: <20100727170001.451DC282BDE@codespeak.net> Author: fijal Date: Tue Jul 27 18:59:59 2010 New Revision: 76373 Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py Log: technically we should only have as many loops as we looked into when debugging Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py Tue Jul 27 18:59:59 2010 @@ -312,7 +312,7 @@ # invent the counter, so we don't get too confused if self._debug: self.loop_names.append(funcname) - self._loop_counter += 1 + self._loop_counter += 1 return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): From fijal at codespeak.net Tue Jul 27 19:03:09 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Jul 2010 19:03:09 +0200 (CEST) Subject: [pypy-svn] r76374 - pypy/branch/improved-asm-logging/pypy/jit/backend/x86 Message-ID: <20100727170309.92189282BDE@codespeak.net> Author: fijal Date: Tue Jul 27 19:03:08 2010 New Revision: 76374 Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py Log: Grumble. swap those calls Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py Tue Jul 27 19:03:08 2010 @@ -231,9 +231,9 @@ _x86_param_depth _x86_arglocs """ + self.make_sure_mc_exists() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() regalloc = RegAlloc(self, self.cpu.translate_support_code) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs @@ -268,9 +268,9 @@ def assemble_bridge(self, faildescr, inputargs, operations): + self.make_sure_mc_exists() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() arglocs = self.rebuild_faillocs_from_descr( faildescr._x86_failure_recovery_bytecode) if not we_are_translated(): From fijal at codespeak.net Tue Jul 27 19:20:35 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Jul 2010 19:20:35 +0200 (CEST) Subject: [pypy-svn] r76375 - in pypy/branch/improved-asm-logging/pypy/jit/backend/x86: . test Message-ID: <20100727172035.996E6282BDE@codespeak.net> Author: fijal Date: Tue Jul 27 19:20:33 2010 New Revision: 76375 Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py Log: Support different forms of PYPYLOG variable Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py Tue Jul 27 19:20:33 2010 @@ -169,6 +169,8 @@ self._build_malloc_fixedsize_slowpath() s = os.environ.get('PYPYLOG') if s: + if s.find(':') != -1: + s = s.split(':')[-1] self.set_debug(True) self._output_loop_log = s + ".count" Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py Tue Jul 27 19:20:33 2010 @@ -398,7 +398,7 @@ def setup_method(self, meth): self.pypylog = os.environ.get('PYPYLOG', None) self.logfile = str(udir.join('x86_runner.log')) - os.environ['PYPYLOG'] = self.logfile + os.environ['PYPYLOG'] = "mumble:" + self.logfile self.cpu = CPU(rtyper=None, stats=FakeStats()) def teardown_method(self, meth): From agaynor at codespeak.net Tue Jul 27 20:55:13 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 27 Jul 2010 20:55:13 +0200 (CEST) Subject: [pypy-svn] r76376 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20100727185513.F3727282BDE@codespeak.net> Author: agaynor Date: Tue Jul 27 20:55:11 2010 New Revision: 76376 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: In the JIT optimize int_add(x, 0) and int_add(0, x), fijal says this shows up in traces. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Jul 27 20:55:11 2010 @@ -1001,6 +1001,17 @@ self.make_equal_to(op.result, v1) else: return self.optimize_default(op) + + def optimize_INT_ADD(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + # If one side of the op is 0 the result is the other side. + if v1.is_constant() and v1.box.getint() == 0: + self.make_equal_to(op.result, v2) + elif v2.is_constant() and v2.box.getint() == 0: + self.make_equal_to(op.result, v1) + else: + self.optimize_default(op) optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Jul 27 20:55:11 2010 @@ -2063,6 +2063,28 @@ jump(i0) """ self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(i0, 0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(0, i0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) # ---------- From agaynor at codespeak.net Tue Jul 27 21:21:54 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 27 Jul 2010 21:21:54 +0200 (CEST) Subject: [pypy-svn] r76377 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20100727192154.5191D282BDE@codespeak.net> Author: agaynor Date: Tue Jul 27 21:21:51 2010 New Revision: 76377 Modified: pypy/trunk/pypy/jit/metainterp/test/test_loop.py pypy/trunk/pypy/jit/metainterp/test/test_loop_spec.py Log: Update a test to reflect the new, better, optimizations. Modified: pypy/trunk/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_loop.py Tue Jul 27 21:21:51 2010 @@ -9,6 +9,10 @@ class LoopTest(object): optimizer = OPTIMIZER_SIMPLE + automatic_promotion_result = { + 'int_add' : 6, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 3 + } def meta_interp(self, f, args, policy=None): return ll_meta_interp(f, args, optimizer=self.optimizer, @@ -477,9 +481,9 @@ res = self.meta_interp(main_interpreter_loop, [1]) assert res == main_interpreter_loop(1) self.check_loop_count(1) - # XXX maybe later optimize guard_value away - self.check_loops({'int_add' : 6, 'int_gt' : 1, - 'guard_false' : 1, 'jump' : 1, 'guard_value' : 3}) + # These loops do different numbers of ops based on which optimizer we + # are testing with. + self.check_loops(self.automatic_promotion_result) def test_can_enter_jit_outside_main_loop(self): myjitdriver = JitDriver(greens=[], reds=['i', 'j', 'a']) Modified: pypy/trunk/pypy/jit/metainterp/test/test_loop_spec.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_loop_spec.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_loop_spec.py Tue Jul 27 21:21:51 2010 @@ -5,6 +5,10 @@ class LoopSpecTest(test_loop.LoopTest): optimizer = OPTIMIZER_FULL + automatic_promotion_result = { + 'int_add' : 3, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 1 + } # ====> test_loop.py From fijal at codespeak.net Wed Jul 28 09:00:06 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 28 Jul 2010 09:00:06 +0200 (CEST) Subject: [pypy-svn] r76378 - pypy/extradoc/planning Message-ID: <20100728070006.96F35282B9C@codespeak.net> Author: fijal Date: Wed Jul 28 09:00:04 2010 New Revision: 76378 Modified: pypy/extradoc/planning/jit.txt Log: add a task Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Wed Jul 28 09:00:04 2010 @@ -28,6 +28,11 @@ current exception from the struct in memory, followed by a regular GUARD_CLASS. +- getfields which result is never used never get removed (probably cause - + they used to be as livevars in removed guards). also getfields which result + is only used as a livevar in a guard should be removed and encoded in + the guard recovert code. + TASKS ----- From fijal at codespeak.net Wed Jul 28 09:08:28 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 28 Jul 2010 09:08:28 +0200 (CEST) Subject: [pypy-svn] r76379 - pypy/trunk/pypy/rlib Message-ID: <20100728070828.49906282B9C@codespeak.net> Author: fijal Date: Wed Jul 28 09:08:26 2010 New Revision: 76379 Modified: pypy/trunk/pypy/rlib/objectmodel.py Log: Fix tests (translation doesn't care about ()) Modified: pypy/trunk/pypy/rlib/objectmodel.py ============================================================================== --- pypy/trunk/pypy/rlib/objectmodel.py (original) +++ pypy/trunk/pypy/rlib/objectmodel.py Wed Jul 28 09:08:26 2010 @@ -26,7 +26,7 @@ by a call result then. """ def decorated_func(func): - func._annspecialcase_ = 'specialize:memo()' + func._annspecialcase_ = 'specialize:memo' return func return decorated_func @@ -63,7 +63,7 @@ for example). Same warnings about exponential behavior apply. """ def decorated_func(func): - func._annspecialcase_ = 'specialize:ll()' + func._annspecialcase_ = 'specialize:ll' return func return decorated_func From fijal at codespeak.net Wed Jul 28 09:51:51 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 28 Jul 2010 09:51:51 +0200 (CEST) Subject: [pypy-svn] r76380 - pypy/extradoc/planning Message-ID: <20100728075151.AFE6B282B9C@codespeak.net> Author: fijal Date: Wed Jul 28 09:51:50 2010 New Revision: 76380 Modified: pypy/extradoc/planning/jit.txt Log: another item Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Wed Jul 28 09:51:50 2010 @@ -33,6 +33,10 @@ is only used as a livevar in a guard should be removed and encoded in the guard recovert code. +- think about strings more. since string are immutable, unnecessary copies + does not make any sense (sometimes people construct strings through + arrays, which is harder to track and then not use the result) + TASKS ----- From hruske at codespeak.net Wed Jul 28 10:22:57 2010 From: hruske at codespeak.net (hruske at codespeak.net) Date: Wed, 28 Jul 2010 10:22:57 +0200 (CEST) Subject: [pypy-svn] r76381 - in pypy/trunk: lib-python/modified-2.5.2/ctypes lib_pypy/_ctypes lib_pypy/pypy_test Message-ID: <20100728082257.31F12282B9C@codespeak.net> Author: hruske Date: Wed Jul 28 10:22:55 2010 New Revision: 76381 Modified: pypy/trunk/lib-python/modified-2.5.2/ctypes/__init__.py pypy/trunk/lib_pypy/_ctypes/__init__.py pypy/trunk/lib_pypy/_ctypes/array.py pypy/trunk/lib_pypy/_ctypes/builtin.py pypy/trunk/lib_pypy/_ctypes/primitive.py pypy/trunk/lib_pypy/pypy_test/test_ctypes_support.py Log: Fix issue 466 where ctypes False -> c_char_p argument conversion results in segfault. Made with at the Europython sprint with arigo's help. Modified: pypy/trunk/lib-python/modified-2.5.2/ctypes/__init__.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/ctypes/__init__.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/ctypes/__init__.py Wed Jul 28 10:22:55 2010 @@ -471,7 +471,7 @@ # functions -from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr +from _ctypes import _memmove_addr, _memset_addr, _cast_addr ## void *memmove(void *, const void *, size_t); memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) @@ -490,24 +490,34 @@ def cast(obj, typ): return _cast(obj, obj, typ) -_string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) +try: + from _ctypes import _string_at_addr +except ImportError: + from _ctypes import _string_at +else: + _string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) + def string_at(ptr, size=-1): """string_at(addr[, size]) -> string Return the string at addr.""" return _string_at(ptr, size) +def wstring_at(ptr, size=-1): + """wstring_at(addr[, size]) -> string + + Return the string at addr.""" + return _wstring_at(ptr, size) + try: from _ctypes import _wstring_at_addr except ImportError: - pass + try: + from _ctypes import _wstring_at + except ImportError: + del wstring_at else: _wstring_at = CFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) - def wstring_at(ptr, size=-1): - """wstring_at(addr[, size]) -> string - - Return the string at addr.""" - return _wstring_at(ptr, size) if _os.name in ("nt", "ce"): # COM stuff Modified: pypy/trunk/lib_pypy/_ctypes/__init__.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/__init__.py (original) +++ pypy/trunk/lib_pypy/_ctypes/__init__.py Wed Jul 28 10:22:55 2010 @@ -7,8 +7,8 @@ from _ctypes.dll import dlopen from _ctypes.structure import Structure from _ctypes.array import Array -from _ctypes.builtin import _memmove_addr, _string_at_addr, _memset_addr,\ - set_conversion_mode, _wstring_at_addr +from _ctypes.builtin import _memmove_addr, _string_at, _memset_addr,\ + set_conversion_mode, _wstring_at from _ctypes.union import Union import os as _os Modified: pypy/trunk/lib_pypy/_ctypes/array.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/array.py (original) +++ pypy/trunk/lib_pypy/_ctypes/array.py Wed Jul 28 10:22:55 2010 @@ -4,7 +4,6 @@ from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof from _ctypes.basics import keepalive_key, store_reference, ensure_objects from _ctypes.basics import CArgObject -from _ctypes.builtin import _string_at_addr, _wstring_at_addr def _create_unicode(buffer, maxlength): res = [] Modified: pypy/trunk/lib_pypy/_ctypes/builtin.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/builtin.py (original) +++ pypy/trunk/lib_pypy/_ctypes/builtin.py Wed Jul 28 10:22:55 2010 @@ -8,10 +8,10 @@ _memmove_addr = _rawffi.get_libc().getaddressindll('memmove') _memset_addr = _rawffi.get_libc().getaddressindll('memset') -def _string_at_addr(addr, lgt): +def _string_at(addr, lgt): # address here can be almost anything import ctypes - arg = ctypes.c_char_p._CData_value(addr) + arg = ctypes.c_void_p._CData_value(addr) return _rawffi.charp2rawstring(arg, lgt) def set_conversion_mode(encoding, errors): @@ -20,9 +20,9 @@ ConvMode.encoding = encoding return old_cm -def _wstring_at_addr(addr, lgt): +def _wstring_at(addr, lgt): import ctypes - arg = ctypes.c_wchar_p._CData_value(addr) + arg = ctypes.c_void_p._CData_value(addr) # XXX purely applevel if lgt == -1: lgt = sys.maxint Modified: pypy/trunk/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/primitive.py (original) +++ pypy/trunk/lib_pypy/_ctypes/primitive.py Wed Jul 28 10:22:55 2010 @@ -86,6 +86,8 @@ return value if isinstance(value, _Pointer): return cls.from_address(value._buffer.buffer) + if isinstance(value, (int, long)): + return cls(value) FROM_PARAM_BY_TYPE = { 'z': from_param_char_p, @@ -141,13 +143,13 @@ result.value = property(_getvalue, _setvalue) elif tp == 'Z': # c_wchar_p - from _ctypes import Array, _Pointer, _wstring_at_addr + from _ctypes import Array, _Pointer, _wstring_at def _getvalue(self): addr = self._buffer[0] if addr == 0: return None else: - return _wstring_at_addr(addr, -1) + return _wstring_at(addr, -1) def _setvalue(self, value): if isinstance(value, basestring): @@ -216,14 +218,14 @@ SysAllocStringLen = windll.oleaut32.SysAllocStringLen SysStringLen = windll.oleaut32.SysStringLen SysFreeString = windll.oleaut32.SysFreeString - from _ctypes import _wstring_at_addr + from _ctypes import _wstring_at def _getvalue(self): addr = self._buffer[0] if addr == 0: return None else: size = SysStringLen(addr) - return _wstring_at_addr(addr, size) + return _wstring_at(addr, size) def _setvalue(self, value): if isinstance(value, basestring): @@ -254,18 +256,21 @@ from_address = cdata_from_address def from_param(self, value): + if isinstance(value, self): + return value + from_param_f = FROM_PARAM_BY_TYPE.get(self._type_) if from_param_f: res = from_param_f(self, value) if res is not None: return res - - if isinstance(value, self): - return value - try: - return self(value) - except (TypeError, ValueError): - return super(SimpleType, self).from_param(value) + else: + try: + return self(value) + except (TypeError, ValueError): + pass + + return super(SimpleType, self).from_param(value) def _CData_output(self, resbuffer, base=None, index=-1): output = super(SimpleType, self)._CData_output(resbuffer, base, index) Modified: pypy/trunk/lib_pypy/pypy_test/test_ctypes_support.py ============================================================================== --- pypy/trunk/lib_pypy/pypy_test/test_ctypes_support.py (original) +++ pypy/trunk/lib_pypy/pypy_test/test_ctypes_support.py Wed Jul 28 10:22:55 2010 @@ -20,3 +20,14 @@ assert get_errno() != 0 set_errno(0) assert get_errno() == 0 + +def test_argument_conversion_and_checks(): + import ctypes + libc = ctypes.cdll.LoadLibrary("libc.so.6") + libc.strlen.argtypes = ctypes.c_char_p, + libc.strlen.restype = ctypes.c_size_t + assert libc.strlen("eggs") == 4 + + # Should raise ArgumentError, not segfault + py.test.raises(ctypes.ArgumentError, libc.strlen, False) + From fijal at codespeak.net Wed Jul 28 11:30:29 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 28 Jul 2010 11:30:29 +0200 (CEST) Subject: [pypy-svn] r76382 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20100728093029.D641C282B9C@codespeak.net> Author: fijal Date: Wed Jul 28 11:30:28 2010 New Revision: 76382 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/runner.py pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Log: Merge part of improved-asm-debugging branch. I'm not sure what to do with this data, but it's already fairly useful (branch will probably continue) Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Wed Jul 28 11:30:28 2010 @@ -1,4 +1,4 @@ -import sys +import sys, os from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT,\ @@ -17,6 +17,7 @@ from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print from pypy.rlib import rgc +from pypy.rlib.streamio import open_file_as_stream # our calling convention - we pass first 6 args in registers # and the rest stays on the stack @@ -99,6 +100,7 @@ mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE _float_constants = None _regalloc = None + _output_loop_log = None def __init__(self, cpu, translate_support_code=False, failargs_limit=1000): @@ -113,17 +115,25 @@ self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 + self.loop_run_counter = values_array(lltype.Signed, 10000) + self.loop_names = [] + # if we have 10000 loops, we have some other problems I guess self.loc_float_const_neg = None self.loc_float_const_abs = None self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 self.setup_failure_recovery() + self._loop_counter = 0 + self._debug = False def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar llop.gc_assume_young_pointers(lltype.Void, llmemory.cast_ptr_to_adr(ptrs)) + def set_debug(self, v): + self._debug = v + def make_sure_mc_exists(self): if self.mc is None: # the address of the function called by 'new' @@ -157,6 +167,22 @@ self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + s = os.environ.get('PYPYLOG') + if s: + if s.find(':') != -1: + s = s.split(':')[-1] + self.set_debug(True) + self._output_loop_log = s + ".count" + + def finish_once(self): + if self._debug: + output_log = self._output_loop_log + assert output_log is not None + f = open_file_as_stream(output_log, "w") + for i in range(self._loop_counter): + f.write(self.loop_names[i] + ":" + + str(self.loop_run_counter.getitem(i)) + "\n") + f.close() def _build_float_constants(self): # 11 words: 8 words for the data, and up to 3 words for alignment @@ -207,9 +233,9 @@ _x86_param_depth _x86_arglocs """ + self.make_sure_mc_exists() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() regalloc = RegAlloc(self, self.cpu.translate_support_code) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs @@ -244,9 +270,9 @@ def assemble_bridge(self, faildescr, inputargs, operations): + self.make_sure_mc_exists() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() arglocs = self.rebuild_faillocs_from_descr( faildescr._x86_failure_recovery_bytecode) if not we_are_translated(): @@ -278,10 +304,18 @@ self.mc.end_function() def _find_debug_merge_point(self, operations): + for op in operations: if op.opnum == rop.DEBUG_MERGE_POINT: - return op.args[0]._get_str() - return "" + funcname = op.args[0]._get_str() + break + else: + funcname = "" % self._loop_counter + # invent the counter, so we don't get too confused + if self._debug: + self.loop_names.append(funcname) + self._loop_counter += 1 + return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset @@ -292,6 +326,15 @@ def _assemble(self, regalloc, operations): self._regalloc = regalloc + if self._debug: + # before doing anything, let's increase a counter + # we need one register free (a bit of a hack, but whatever) + self.mc.PUSH(eax) + adr = self.loop_run_counter.get_addr_for_num(self._loop_counter - 1) + self.mc.MOV(eax, heap(adr)) + self.mc.ADD(eax, imm(1)) + self.mc.MOV(heap(adr), eax) + self.mc.POP(eax) regalloc.walk_operations(operations) self.mc.done() self.mc2.done() Modified: pypy/trunk/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/runner.py Wed Jul 28 11:30:28 2010 @@ -44,6 +44,7 @@ self.profile_agent.startup() def finish_once(self): + self.assembler.finish_once() self.profile_agent.shutdown() def compile_loop(self, inputargs, operations, looptoken): Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Wed Jul 28 11:30:28 2010 @@ -9,8 +9,11 @@ from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.executor import execute from pypy.jit.backend.test.runner_test import LLtypeBackendTest +from pypy.jit.metainterp.test.oparser import parse +from pypy.tool.udir import udir import ctypes import sys +import os class FakeStats(object): pass @@ -390,3 +393,35 @@ self.cpu.set_future_value_int(0, base_v.value) self.cpu.execute_token(looptoken) assert self.cpu.get_latest_value_int(0) == 1024 + +class TestDebuggingAssembler(object): + def setup_method(self, meth): + self.pypylog = os.environ.get('PYPYLOG', None) + self.logfile = str(udir.join('x86_runner.log')) + os.environ['PYPYLOG'] = "mumble:" + self.logfile + self.cpu = CPU(rtyper=None, stats=FakeStats()) + + def teardown_method(self, meth): + if self.pypylog is not None: + os.environ['PYPYLOG'] = self.pypylog + + def test_debugger_on(self): + loop = """ + [i0] + debug_merge_point('xyz') + i1 = int_add(i0, 1) + i2 = int_ge(i1, 10) + guard_false(i2) [] + jump(i1) + """ + ops = parse(loop) + self.cpu.assembler.set_debug(True) + self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) + self.cpu.set_future_value_int(0, 0) + self.cpu.execute_token(ops.token) + # check debugging info + assert self.cpu.assembler.loop_names == ["xyz"] + assert self.cpu.assembler.loop_run_counter.getitem(0) == 10 + self.cpu.finish_once() + lines = py.path.local(self.logfile + ".count").readlines() + assert lines[0] == 'xyz:10\n' From fijal at codespeak.net Wed Jul 28 11:51:40 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 28 Jul 2010 11:51:40 +0200 (CEST) Subject: [pypy-svn] r76383 - pypy/benchmarks/own/twisted Message-ID: <20100728095140.7612B282B9C@codespeak.net> Author: fijal Date: Wed Jul 28 11:51:38 2010 New Revision: 76383 Modified: pypy/benchmarks/own/twisted/benchlib.py Log: Make sure we read all the data from os.popen Modified: pypy/benchmarks/own/twisted/benchlib.py ============================================================================== --- pypy/benchmarks/own/twisted/benchlib.py (original) +++ pypy/benchmarks/own/twisted/benchlib.py Wed Jul 28 11:51:38 2010 @@ -109,7 +109,14 @@ import time, os for i in range(24): g = os.popen('netstat -atn') - data = g.read() + # make sure we read *all* data + all = [] + while True: + data = g.read() + if not data: + break + all.append(data) + data = "".join(all) g.close() if ('Active Internet connections' in data and data.count('TIME_WAIT') < 20): From dan at codespeak.net Thu Jul 29 01:10:58 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Thu, 29 Jul 2010 01:10:58 +0200 (CEST) Subject: [pypy-svn] r76384 - in pypy/branch/micronumpy/pypy/module/micronumpy: . test Message-ID: <20100728231058.918D1282C0B@codespeak.net> Author: dan Date: Thu Jul 29 01:10:56 2010 New Revision: 76384 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Log: Now passing dtype test, one less xfail. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py Thu Jul 29 01:10:56 2010 @@ -12,19 +12,15 @@ self.w_native_type = None def descr_eq(self, space, w_type): - if space.eq_w(self.w_native_type, w_type): - return space.w_True + if isinstance(w_type, TypeDescr): + other_type = w_type.dtype + else: + other_type = get(space, w_type) - try: - typestr = space.str_w(w_type) - if self.dtype.typecode == typestr: return space.w_True - elif self.name == typestr: return space.w_True - else: return space.w_False - except OperationError, e: - if e.match(space, space.w_TypeError): pass - else: raise + typecode = self.dtype.typecode + other_typecode = other_type.typecode - return space.w_False + return space.w_True if typecode == other_typecode else space.w_False descr_eq.unwrap_spec = ['self', ObjSpace, W_Root] def descr_repr(self, space): @@ -165,10 +161,16 @@ def from_wrapped_type(space, w_type): if w_type is space.w_int: return int_descr - else: + elif w_type is space.w_float: return float_descr #XXX: only handles two types! + else: + raise OperationError(space.w_TypeError, + space.wrap("unknown type %s" % w_type)) def get(space, w_dtype): + if isinstance(w_dtype, TypeDescr): + return w_dtype.dtype #FIXME: a little wasteful since we end up just getting the TypeDescr + try: s = space.str_w(w_dtype) @@ -185,13 +187,6 @@ except OperationError, e: if e.match(space, space.w_TypeError): pass # XXX: ValueError? - try: - i = space.int_w(w_dtype) - return _descriptors[i] - except OperationError, e: - if e.match(space, space.w_TypeError): pass - else: raise - return from_wrapped_type(space, w_dtype) # FIXME: watch for wrapped typedescrs! Modified: pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Thu Jul 29 01:10:56 2010 @@ -351,7 +351,6 @@ def setup_class(cls): cls.space = gettestobjspace(usemodules=('micronumpy',)) - @py.test.mark.xfail def test_eq(self): from micronumpy import zeros From fijal at codespeak.net Thu Jul 29 14:53:10 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Jul 2010 14:53:10 +0200 (CEST) Subject: [pypy-svn] r76386 - pypy/benchmarks Message-ID: <20100729125310.23A7B282BF8@codespeak.net> Author: fijal Date: Thu Jul 29 14:53:08 2010 New Revision: 76386 Modified: pypy/benchmarks/benchmarks.py Log: The last attempt to make that work Modified: pypy/benchmarks/benchmarks.py ============================================================================== --- pypy/benchmarks/benchmarks.py (original) +++ pypy/benchmarks/benchmarks.py Thu Jul 29 14:53:08 2010 @@ -47,7 +47,10 @@ 'spectral-norm', 'chaos', 'telco']: _register_new_bm(name, name, globals(), **opts.get(name, {})) for name in ['web', 'names', 'iteration', 'tcp', 'pb']:#, 'accepts']: - iteration_scaling = 1.0 + if name == 'web': + iteration_scaling = 0.1 + else: + iteration_scaling = 1.0 _register_new_bm_twisted(name, 'twisted_' + name, globals(), bm_env={'PYTHONPATH': ':'.join(TWISTED)}, iteration_scaling=iteration_scaling) From fijal at codespeak.net Thu Jul 29 15:02:28 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Jul 2010 15:02:28 +0200 (CEST) Subject: [pypy-svn] r76387 - in pypy/trunk/pypy: interpreter/astcompiler module/__builtin__ module/_ast module/_ast/test Message-ID: <20100729130228.79E71282BE0@codespeak.net> Author: fijal Date: Thu Jul 29 15:02:27 2010 New Revision: 76387 Modified: pypy/trunk/pypy/interpreter/astcompiler/consts.py pypy/trunk/pypy/module/__builtin__/compiling.py pypy/trunk/pypy/module/_ast/__init__.py pypy/trunk/pypy/module/_ast/test/test_ast.py Log: Rename the flag to be the same as CPython and not with a typo :-/ Modified: pypy/trunk/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/consts.py Thu Jul 29 15:02:27 2010 @@ -18,4 +18,4 @@ PyCF_SOURCE_IS_UTF8 = 0x0100 PyCF_DONT_IMPLY_DEDENT = 0x0200 -PyCF_AST_ONLY = 0x0400 +PyCF_ONLY_AST = 0x0400 Modified: pypy/trunk/pypy/module/__builtin__/compiling.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/compiling.py (original) +++ pypy/trunk/pypy/module/__builtin__/compiling.py Thu Jul 29 15:02:27 2010 @@ -38,7 +38,7 @@ str_ = space.str_w(w_source) ec = space.getexecutioncontext() - if flags & ~(ec.compiler.compiler_flags | consts.PyCF_AST_ONLY | + if flags & ~(ec.compiler.compiler_flags | consts.PyCF_ONLY_AST | consts.PyCF_DONT_IMPLY_DEDENT | consts.PyCF_SOURCE_IS_UTF8): raise OperationError(space.w_ValueError, space.wrap("compile() unrecognized flags")) @@ -53,7 +53,7 @@ "or 'eval' or 'single'")) if ast_node is None: - if flags & consts.PyCF_AST_ONLY: + if flags & consts.PyCF_ONLY_AST: mod = ec.compiler.compile_to_ast(str_, filename, mode, flags) return space.wrap(mod) else: Modified: pypy/trunk/pypy/module/_ast/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_ast/__init__.py (original) +++ pypy/trunk/pypy/module/_ast/__init__.py Thu Jul 29 15:02:27 2010 @@ -5,7 +5,7 @@ class Module(MixedModule): interpleveldefs = { - "PyCF_AST_ONLY" : "space.wrap(%s)" % consts.PyCF_AST_ONLY + "PyCF_ONLY_AST" : "space.wrap(%s)" % consts.PyCF_ONLY_AST } appleveldefs = {} Modified: pypy/trunk/pypy/module/_ast/test/test_ast.py ============================================================================== --- pypy/trunk/pypy/module/_ast/test/test_ast.py (original) +++ pypy/trunk/pypy/module/_ast/test/test_ast.py Thu Jul 29 15:02:27 2010 @@ -10,7 +10,7 @@ cls.w_get_ast = cls.space.appexec([], """(): def get_ast(source, mode="exec"): import _ast as ast - mod = compile(source, "", mode, ast.PyCF_AST_ONLY) + mod = compile(source, "", mode, ast.PyCF_ONLY_AST) assert isinstance(mod, ast.mod) return mod return get_ast""") From getxsick at codespeak.net Thu Jul 29 15:38:12 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 29 Jul 2010 15:38:12 +0200 (CEST) Subject: [pypy-svn] r76390 - in pypy/branch/fast-ctypes: . lib-python lib-python/modified-2.5.2/ctypes lib_pypy lib_pypy/_ctypes lib_pypy/pypy_test pypy/interpreter pypy/interpreter/astcompiler pypy/interpreter/astcompiler/tools pypy/interpreter/test pypy/jit/backend/llsupport/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/metainterp pypy/jit/metainterp/test pypy/module/__builtin__ pypy/module/_ast pypy/module/_ast/test pypy/module/_file pypy/module/_file/test pypy/module/_rawffi/test pypy/module/cpyext pypy/module/posix pypy/module/posix/test pypy/rlib pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/module pypy/rpython/module/test pypy/rpython/test pypy/translator/goal pypy/translator/goal/test2 Message-ID: <20100729133812.53180282C19@codespeak.net> Author: getxsick Date: Thu Jul 29 15:38:08 2010 New Revision: 76390 Added: pypy/branch/fast-ctypes/pypy/rlib/test/test_rposix.py - copied unchanged from r76389, pypy/trunk/pypy/rlib/test/test_rposix.py pypy/branch/fast-ctypes/pypy/rpython/module/ll_win32file.py - copied unchanged from r76389, pypy/trunk/pypy/rpython/module/ll_win32file.py Modified: pypy/branch/fast-ctypes/ (props changed) pypy/branch/fast-ctypes/lib-python/conftest.py pypy/branch/fast-ctypes/lib-python/modified-2.5.2/ctypes/__init__.py pypy/branch/fast-ctypes/lib_pypy/_ctypes/__init__.py pypy/branch/fast-ctypes/lib_pypy/_ctypes/array.py pypy/branch/fast-ctypes/lib_pypy/_ctypes/builtin.py pypy/branch/fast-ctypes/lib_pypy/_ctypes/primitive.py pypy/branch/fast-ctypes/lib_pypy/datetime.py pypy/branch/fast-ctypes/lib_pypy/pypy_test/test_ctypes_support.py pypy/branch/fast-ctypes/lib_pypy/pypy_test/test_datetime.py pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/ast.py pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/codegen.py pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/consts.py pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/symtable.py pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/tools/asdl_py.py pypy/branch/fast-ctypes/pypy/interpreter/baseobjspace.py pypy/branch/fast-ctypes/pypy/interpreter/error.py pypy/branch/fast-ctypes/pypy/interpreter/gateway.py pypy/branch/fast-ctypes/pypy/interpreter/test/test_compiler.py pypy/branch/fast-ctypes/pypy/interpreter/test/test_gateway.py pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/assembler.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/runner.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_runner.py pypy/branch/fast-ctypes/pypy/jit/metainterp/executor.py pypy/branch/fast-ctypes/pypy/jit/metainterp/optimizeopt.py pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_loop.py pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_loop_spec.py pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/fast-ctypes/pypy/module/__builtin__/compiling.py pypy/branch/fast-ctypes/pypy/module/_ast/__init__.py pypy/branch/fast-ctypes/pypy/module/_ast/test/test_ast.py pypy/branch/fast-ctypes/pypy/module/_file/interp_file.py pypy/branch/fast-ctypes/pypy/module/_file/test/test_file.py pypy/branch/fast-ctypes/pypy/module/_rawffi/test/test__rawffi.py pypy/branch/fast-ctypes/pypy/module/cpyext/methodobject.py pypy/branch/fast-ctypes/pypy/module/posix/interp_posix.py pypy/branch/fast-ctypes/pypy/module/posix/test/test_posix2.py pypy/branch/fast-ctypes/pypy/rlib/objectmodel.py pypy/branch/fast-ctypes/pypy/rlib/rposix.py pypy/branch/fast-ctypes/pypy/rlib/rwin32.py pypy/branch/fast-ctypes/pypy/rlib/streamio.py pypy/branch/fast-ctypes/pypy/rpython/extfunc.py pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/rffi.py pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/rstr.py pypy/branch/fast-ctypes/pypy/rpython/module/ll_os.py pypy/branch/fast-ctypes/pypy/rpython/module/ll_os_stat.py pypy/branch/fast-ctypes/pypy/rpython/module/test/test_ll_os_stat.py pypy/branch/fast-ctypes/pypy/rpython/test/test_extfunc.py pypy/branch/fast-ctypes/pypy/translator/goal/app_main.py pypy/branch/fast-ctypes/pypy/translator/goal/test2/test_app_main.py Log: merge from trunk Modified: pypy/branch/fast-ctypes/lib-python/conftest.py ============================================================================== --- pypy/branch/fast-ctypes/lib-python/conftest.py (original) +++ pypy/branch/fast-ctypes/lib-python/conftest.py Thu Jul 29 15:38:08 2010 @@ -464,11 +464,7 @@ RegrTest('test_coding.py'), RegrTest('test_complex_args.py'), RegrTest('test_contextlib.py', usemodules="thread"), - # we skip test ctypes, since we adapted it massively in order - # to test what we want to support. There are real failures, - # but it's about missing features that we don't want to support - # now - RegrTest('test_ctypes.py', skip="we have a replacement"), + RegrTest('test_ctypes.py', usemodules="_rawffi"), RegrTest('test_defaultdict.py'), RegrTest('test_email_renamed.py'), RegrTest('test_exception_variations.py'), Modified: pypy/branch/fast-ctypes/lib-python/modified-2.5.2/ctypes/__init__.py ============================================================================== --- pypy/branch/fast-ctypes/lib-python/modified-2.5.2/ctypes/__init__.py (original) +++ pypy/branch/fast-ctypes/lib-python/modified-2.5.2/ctypes/__init__.py Thu Jul 29 15:38:08 2010 @@ -471,7 +471,7 @@ # functions -from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr +from _ctypes import _memmove_addr, _memset_addr, _cast_addr ## void *memmove(void *, const void *, size_t); memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) @@ -490,24 +490,34 @@ def cast(obj, typ): return _cast(obj, obj, typ) -_string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) +try: + from _ctypes import _string_at_addr +except ImportError: + from _ctypes import _string_at +else: + _string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) + def string_at(ptr, size=-1): """string_at(addr[, size]) -> string Return the string at addr.""" return _string_at(ptr, size) +def wstring_at(ptr, size=-1): + """wstring_at(addr[, size]) -> string + + Return the string at addr.""" + return _wstring_at(ptr, size) + try: from _ctypes import _wstring_at_addr except ImportError: - pass + try: + from _ctypes import _wstring_at + except ImportError: + del wstring_at else: _wstring_at = CFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) - def wstring_at(ptr, size=-1): - """wstring_at(addr[, size]) -> string - - Return the string at addr.""" - return _wstring_at(ptr, size) if _os.name in ("nt", "ce"): # COM stuff Modified: pypy/branch/fast-ctypes/lib_pypy/_ctypes/__init__.py ============================================================================== --- pypy/branch/fast-ctypes/lib_pypy/_ctypes/__init__.py (original) +++ pypy/branch/fast-ctypes/lib_pypy/_ctypes/__init__.py Thu Jul 29 15:38:08 2010 @@ -7,8 +7,8 @@ from _ctypes.dll import dlopen from _ctypes.structure import Structure from _ctypes.array import Array -from _ctypes.builtin import _memmove_addr, _string_at_addr, _memset_addr,\ - set_conversion_mode, _wstring_at_addr +from _ctypes.builtin import _memmove_addr, _string_at, _memset_addr,\ + set_conversion_mode, _wstring_at from _ctypes.union import Union import os as _os Modified: pypy/branch/fast-ctypes/lib_pypy/_ctypes/array.py ============================================================================== --- pypy/branch/fast-ctypes/lib_pypy/_ctypes/array.py (original) +++ pypy/branch/fast-ctypes/lib_pypy/_ctypes/array.py Thu Jul 29 15:38:08 2010 @@ -4,7 +4,6 @@ from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof from _ctypes.basics import keepalive_key, store_reference, ensure_objects from _ctypes.basics import CArgObject -from _ctypes.builtin import _string_at_addr, _wstring_at_addr def _create_unicode(buffer, maxlength): res = [] Modified: pypy/branch/fast-ctypes/lib_pypy/_ctypes/builtin.py ============================================================================== --- pypy/branch/fast-ctypes/lib_pypy/_ctypes/builtin.py (original) +++ pypy/branch/fast-ctypes/lib_pypy/_ctypes/builtin.py Thu Jul 29 15:38:08 2010 @@ -8,10 +8,10 @@ _memmove_addr = _rawffi.get_libc().getaddressindll('memmove') _memset_addr = _rawffi.get_libc().getaddressindll('memset') -def _string_at_addr(addr, lgt): +def _string_at(addr, lgt): # address here can be almost anything import ctypes - arg = ctypes.c_char_p._CData_value(addr) + arg = ctypes.c_void_p._CData_value(addr) return _rawffi.charp2rawstring(arg, lgt) def set_conversion_mode(encoding, errors): @@ -20,9 +20,9 @@ ConvMode.encoding = encoding return old_cm -def _wstring_at_addr(addr, lgt): +def _wstring_at(addr, lgt): import ctypes - arg = ctypes.c_wchar_p._CData_value(addr) + arg = ctypes.c_void_p._CData_value(addr) # XXX purely applevel if lgt == -1: lgt = sys.maxint Modified: pypy/branch/fast-ctypes/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/branch/fast-ctypes/lib_pypy/_ctypes/primitive.py (original) +++ pypy/branch/fast-ctypes/lib_pypy/_ctypes/primitive.py Thu Jul 29 15:38:08 2010 @@ -86,6 +86,8 @@ return value if isinstance(value, _Pointer): return cls.from_address(value._buffer.buffer) + if isinstance(value, (int, long)): + return cls(value) FROM_PARAM_BY_TYPE = { 'z': from_param_char_p, @@ -141,13 +143,13 @@ result.value = property(_getvalue, _setvalue) elif tp == 'Z': # c_wchar_p - from _ctypes import Array, _Pointer, _wstring_at_addr + from _ctypes import Array, _Pointer, _wstring_at def _getvalue(self): addr = self._buffer[0] if addr == 0: return None else: - return _wstring_at_addr(addr, -1) + return _wstring_at(addr, -1) def _setvalue(self, value): if isinstance(value, basestring): @@ -216,14 +218,14 @@ SysAllocStringLen = windll.oleaut32.SysAllocStringLen SysStringLen = windll.oleaut32.SysStringLen SysFreeString = windll.oleaut32.SysFreeString - from _ctypes import _wstring_at_addr + from _ctypes import _wstring_at def _getvalue(self): addr = self._buffer[0] if addr == 0: return None else: size = SysStringLen(addr) - return _wstring_at_addr(addr, size) + return _wstring_at(addr, size) def _setvalue(self, value): if isinstance(value, basestring): @@ -254,18 +256,21 @@ from_address = cdata_from_address def from_param(self, value): + if isinstance(value, self): + return value + from_param_f = FROM_PARAM_BY_TYPE.get(self._type_) if from_param_f: res = from_param_f(self, value) if res is not None: return res - - if isinstance(value, self): - return value - try: - return self(value) - except (TypeError, ValueError): - return super(SimpleType, self).from_param(value) + else: + try: + return self(value) + except (TypeError, ValueError): + pass + + return super(SimpleType, self).from_param(value) def _CData_output(self, resbuffer, base=None, index=-1): output = super(SimpleType, self)._CData_output(resbuffer, base, index) Modified: pypy/branch/fast-ctypes/lib_pypy/datetime.py ============================================================================== --- pypy/branch/fast-ctypes/lib_pypy/datetime.py (original) +++ pypy/branch/fast-ctypes/lib_pypy/datetime.py Thu Jul 29 15:38:08 2010 @@ -1412,7 +1412,7 @@ def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - if 1 - (t % 1.0) < 0.000001: + if 1 - (t % 1.0) < 0.0000005: t = float(int(t)) + 1 if t < 0: t -= 1 Modified: pypy/branch/fast-ctypes/lib_pypy/pypy_test/test_ctypes_support.py ============================================================================== --- pypy/branch/fast-ctypes/lib_pypy/pypy_test/test_ctypes_support.py (original) +++ pypy/branch/fast-ctypes/lib_pypy/pypy_test/test_ctypes_support.py Thu Jul 29 15:38:08 2010 @@ -20,3 +20,14 @@ assert get_errno() != 0 set_errno(0) assert get_errno() == 0 + +def test_argument_conversion_and_checks(): + import ctypes + libc = ctypes.cdll.LoadLibrary("libc.so.6") + libc.strlen.argtypes = ctypes.c_char_p, + libc.strlen.restype = ctypes.c_size_t + assert libc.strlen("eggs") == 4 + + # Should raise ArgumentError, not segfault + py.test.raises(ctypes.ArgumentError, libc.strlen, False) + Modified: pypy/branch/fast-ctypes/lib_pypy/pypy_test/test_datetime.py ============================================================================== --- pypy/branch/fast-ctypes/lib_pypy/pypy_test/test_datetime.py (original) +++ pypy/branch/fast-ctypes/lib_pypy/pypy_test/test_datetime.py Thu Jul 29 15:38:08 2010 @@ -15,4 +15,18 @@ expected = datetime.datetime(*(time.strptime(string, format)[0:6])) got = datetime.datetime.strptime(string, format) assert expected == got + +def test_datetime_rounding(): + b = 0.0000001 + a = 0.9999994 + + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 + assert datetime.datetime.utcfromtimestamp(a).second == 1 + Modified: pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/ast.py Thu Jul 29 15:38:08 2010 @@ -230,6 +230,7 @@ visitor.visit_FunctionDef(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) if self.body: visitor._mutate_sequence(self.body) if self.decorators: @@ -784,6 +785,8 @@ def mutate_over(self, visitor): if self.body: visitor._mutate_sequence(self.body) + if self.handlers: + visitor._mutate_sequence(self.handlers) if self.orelse: visitor._mutate_sequence(self.orelse) return visitor.visit_TryExcept(self) @@ -927,6 +930,8 @@ visitor.visit_Import(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_Import(self) def sync_app_attrs(self, space): @@ -965,6 +970,8 @@ visitor.visit_ImportFrom(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_ImportFrom(self) def sync_app_attrs(self, space): @@ -1282,6 +1289,7 @@ visitor.visit_Lambda(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) self.body = self.body.mutate_over(visitor) return visitor.visit_Lambda(self) @@ -1398,6 +1406,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_ListComp(self) def sync_app_attrs(self, space): @@ -1437,6 +1447,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_GeneratorExp(self) def sync_app_attrs(self, space): @@ -1562,6 +1574,8 @@ self.func = self.func.mutate_over(visitor) if self.args: visitor._mutate_sequence(self.args) + if self.keywords: + visitor._mutate_sequence(self.keywords) if self.starargs: self.starargs = self.starargs.mutate_over(visitor) if self.kwargs: @@ -2293,6 +2307,13 @@ self.w_ifs = None self.initialization_state = 7 + def mutate_over(self, visitor): + self.target = self.target.mutate_over(visitor) + self.iter = self.iter.mutate_over(visitor) + if self.ifs: + visitor._mutate_sequence(self.ifs) + return visitor.visit_comprehension(self) + def walkabout(self, visitor): visitor.visit_comprehension(self) @@ -2327,6 +2348,15 @@ self.col_offset = col_offset self.initialization_state = 31 + def mutate_over(self, visitor): + if self.type: + self.type = self.type.mutate_over(visitor) + if self.name: + self.name = self.name.mutate_over(visitor) + if self.body: + visitor._mutate_sequence(self.body) + return visitor.visit_excepthandler(self) + def walkabout(self, visitor): visitor.visit_excepthandler(self) @@ -2366,6 +2396,13 @@ self.w_defaults = None self.initialization_state = 15 + def mutate_over(self, visitor): + if self.args: + visitor._mutate_sequence(self.args) + if self.defaults: + visitor._mutate_sequence(self.defaults) + return visitor.visit_arguments(self) + def walkabout(self, visitor): visitor.visit_arguments(self) @@ -2407,6 +2444,10 @@ self.value = value self.initialization_state = 3 + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + return visitor.visit_keyword(self) + def walkabout(self, visitor): visitor.visit_keyword(self) @@ -2426,6 +2467,9 @@ self.asname = asname self.initialization_state = 3 + def mutate_over(self, visitor): + return visitor.visit_alias(self) + def walkabout(self, visitor): visitor.visit_alias(self) Modified: pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/codegen.py Thu Jul 29 15:38:08 2010 @@ -258,9 +258,11 @@ # Load decorators first, but apply them after the function is created. if func.decorators: self.visit_sequence(func.decorators) - if func.args.defaults: - self.visit_sequence(func.args.defaults) - num_defaults = len(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + num_defaults = len(args.defaults) else: num_defaults = 0 code = self.sub_scope(FunctionCodeGenerator, func.name, func, @@ -274,9 +276,11 @@ def visit_Lambda(self, lam): self.update_position(lam.lineno) - if lam.args.defaults: - self.visit_sequence(lam.args.defaults) - default_count = len(lam.args.defaults) + args = lam.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + default_count = len(args.defaults) else: default_count = 0 code = self.sub_scope(LambdaCodeGenerator, "", lam, lam.lineno) @@ -1275,9 +1279,11 @@ else: self.add_const(self.space.w_None) start = 0 - if func.args.args: - self._handle_nested_args(func.args.args) - self.argcount = len(func.args.args) + args = func.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) for i in range(start, len(func.body)): func.body[i].walkabout(self) @@ -1286,9 +1292,11 @@ def _compile(self, lam): assert isinstance(lam, ast.Lambda) - if lam.args.args: - self._handle_nested_args(lam.args.args) - self.argcount = len(lam.args.args) + args = lam.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) # Prevent a string from being the first constant and thus a docstring. self.add_const(self.space.w_None) lam.body.walkabout(self) Modified: pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/consts.py Thu Jul 29 15:38:08 2010 @@ -18,4 +18,4 @@ PyCF_SOURCE_IS_UTF8 = 0x0100 PyCF_DONT_IMPLY_DEDENT = 0x0200 -PyCF_AST_ONLY = 0x0400 +PyCF_ONLY_AST = 0x0400 Modified: pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/symtable.py Thu Jul 29 15:38:08 2010 @@ -353,8 +353,10 @@ def visit_FunctionDef(self, func): self.note_symbol(func.name, SYM_ASSIGNED) # Function defaults and decorators happen in the outer scope. - if func.args.defaults: - self.visit_sequence(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) if func.decorators: self.visit_sequence(func.decorators) new_scope = FunctionScope(func.name, func.lineno, func.col_offset) @@ -420,8 +422,10 @@ self.note_symbol(name, SYM_GLOBAL) def visit_Lambda(self, lamb): - if lamb.args.defaults: - self.visit_sequence(lamb.args.defaults) + args = lamb.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) new_scope = FunctionScope("lambda", lamb.lineno, lamb.col_offset) self.push_scope(new_scope, lamb) lamb.args.walkabout(self) Modified: pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/tools/asdl_py.py Thu Jul 29 15:38:08 2010 @@ -100,6 +100,7 @@ self.emit("") self.make_constructor(product.fields, product) self.emit("") + self.make_mutate_over(product, name) self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (name,), 2) self.emit("") @@ -183,6 +184,26 @@ have_everything = self.data.required_masks[node] | \ self.data.optional_masks[node] self.emit("self.initialization_state = %i" % (have_everything,), 2) + + def make_mutate_over(self, cons, name): + self.emit("def mutate_over(self, visitor):", 1) + for field in cons.fields: + if (field.type.value not in asdl.builtin_types and + field.type.value not in self.data.simple_types): + if field.opt or field.seq: + level = 3 + self.emit("if self.%s:" % (field.name,), 2) + else: + level = 2 + if field.seq: + sub = (field.name,) + self.emit("visitor._mutate_sequence(self.%s)" % sub, level) + else: + sub = (field.name, field.name) + self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, + level) + self.emit("return visitor.visit_%s(self)" % (name,), 2) + self.emit("") def visitConstructor(self, cons, base, extra_attributes): self.emit("class %s(%s):" % (cons.name, base)) @@ -199,24 +220,7 @@ self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (cons.name,), 2) self.emit("") - self.emit("def mutate_over(self, visitor):", 1) - for field in cons.fields: - if field.type.value not in asdl.builtin_types and \ - field.type.value not in self.data.prod_simple: - if field.opt or field.seq: - level = 3 - self.emit("if self.%s:" % (field.name,), 2) - else: - level = 2 - if field.seq: - sub = (field.name,) - self.emit("visitor._mutate_sequence(self.%s)" % sub, level) - else: - sub = (field.name, field.name) - self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, - level) - self.emit("return visitor.visit_%s(self)" % (cons.name,), 2) - self.emit("") + self.make_mutate_over(cons, cons.name) self.make_var_syncer(cons.fields + self.data.cons_attributes[cons], cons, cons.name) Modified: pypy/branch/fast-ctypes/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/baseobjspace.py Thu Jul 29 15:38:08 2010 @@ -1102,17 +1102,6 @@ self.wrap('argument must be a unicode')) return self.unicode_w(w_obj) - def path_w(self, w_obj): - """ Like str_w, but if the object is unicode, encode it using - filesystemencoding - """ - filesystemencoding = self.sys.filesystemencoding - if (filesystemencoding and - self.is_true(self.isinstance(w_obj, self.w_unicode))): - w_obj = self.call_method(w_obj, "encode", - self.wrap(filesystemencoding)) - return self.str_w(w_obj) - def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. # This is here mostly just for gateway.int_unwrapping_space_method(). Modified: pypy/branch/fast-ctypes/pypy/interpreter/error.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/error.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/error.py Thu Jul 29 15:38:08 2010 @@ -344,7 +344,7 @@ else: _WINDOWS = True - def wrap_windowserror(space, e, filename=None): + def wrap_windowserror(space, e, w_filename=None): from pypy.rlib import rwin32 winerror = e.winerror @@ -353,19 +353,19 @@ except ValueError: msg = 'Windows Error %d' % winerror exc = space.w_WindowsError - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(winerror), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: w_error = space.call_function(exc, space.wrap(winerror), space.wrap(msg)) return OperationError(exc, w_error) -def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): +def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError'): assert isinstance(e, OSError) if _WINDOWS and isinstance(e, WindowsError): - return wrap_windowserror(space, e, filename) + return wrap_windowserror(space, e, w_filename) errno = e.errno try: @@ -373,10 +373,21 @@ except ValueError: msg = 'error %d' % errno exc = getattr(space, exception_name) - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(errno), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: - w_error = space.call_function(exc, space.wrap(errno), space.wrap(msg)) + w_error = space.call_function(exc, space.wrap(errno), + space.wrap(msg)) return OperationError(exc, w_error) +wrap_oserror2._annspecialcase_ = 'specialize:arg(3)' + +def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): + if filename is not None: + return wrap_oserror2(space, e, space.wrap(filename), + exception_name=exception_name) + else: + return wrap_oserror2(space, e, None, + exception_name=exception_name) wrap_oserror._annspecialcase_ = 'specialize:arg(3)' + Modified: pypy/branch/fast-ctypes/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/gateway.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/gateway.py Thu Jul 29 15:38:08 2010 @@ -137,9 +137,6 @@ def visit_c_nonnegint(self, el, app_sig): self.checked_space_method(el, app_sig) - def visit_path(self, el, app_sig): - self.checked_space_method(el, app_sig) - def visit__Wrappable(self, el, app_sig): name = el.__name__ argname = self.orig_arg() @@ -241,9 +238,6 @@ def visit_bufferstr(self, typ): self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),)) - def visit_path(self, typ): - self.run_args.append("space.path_w(%s)" % (self.scopenext(),)) - def visit_nonnegint(self, typ): self.run_args.append("space.nonnegint_w(%s)" % (self.scopenext(),)) @@ -371,9 +365,6 @@ def visit_bufferstr(self, typ): self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),)) - def visit_path(self, typ): - self.unwrap.append("space.path_w(%s)" % (self.nextarg(),)) - def visit_nonnegint(self, typ): self.unwrap.append("space.nonnegint_w(%s)" % (self.nextarg(),)) Modified: pypy/branch/fast-ctypes/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/test/test_compiler.py Thu Jul 29 15:38:08 2010 @@ -838,6 +838,23 @@ sys.stdout = save_stdout output = s.getvalue() assert "STOP_CODE" not in output + + def test_optimize_list_comp(self): + source = """def _f(a): + return [x for x in a if None] + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = sys.__stdout__ + output = s.getvalue() + assert "LOAD_GLOBAL" not in output class AppTestExceptions: def test_indentation_error(self): Modified: pypy/branch/fast-ctypes/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/test/test_gateway.py Thu Jul 29 15:38:08 2010 @@ -454,16 +454,6 @@ assert len(l) == 1 assert space.eq_w(l[0], w("foo")) - def test_interp2app_unwrap_spec_path(self, monkeypatch): - space = self.space - def g(space, p): - return p - - app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'path']) - w_app_g = space.wrap(app_g) - monkeypatch.setattr(space.sys, "filesystemencoding", "utf-8") - w_res = space.call_function(w_app_g, space.wrap(u"?")) - def test_interp2app_classmethod(self): space = self.space w = space.wrap Modified: pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/test/test_descr.py Thu Jul 29 15:38:08 2010 @@ -232,11 +232,11 @@ # descr2 = get_field_descr(c0, S, 'y') o, _ = symbolic.get_field_token(S, 'y', False) - assert descr2.repr_of_descr() == '' % o + assert descr2.repr_of_descr() == '' % o # descr2i = get_field_descr(c0, S, 'x') o, _ = symbolic.get_field_token(S, 'x', False) - assert descr2i.repr_of_descr() == '' % o + assert descr2i.repr_of_descr() == '' % o # descr3 = get_array_descr(c0, lltype.GcArray(lltype.Ptr(S))) assert descr3.repr_of_descr() == '' Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/assembler.py Thu Jul 29 15:38:08 2010 @@ -1,4 +1,4 @@ -import sys +import sys, os from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT,\ @@ -17,6 +17,7 @@ from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print from pypy.rlib import rgc +from pypy.rlib.streamio import open_file_as_stream # our calling convention - we pass first 6 args in registers # and the rest stays on the stack @@ -99,6 +100,7 @@ mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE _float_constants = None _regalloc = None + _output_loop_log = None def __init__(self, cpu, translate_support_code=False, failargs_limit=1000): @@ -113,17 +115,25 @@ self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 + self.loop_run_counter = values_array(lltype.Signed, 10000) + self.loop_names = [] + # if we have 10000 loops, we have some other problems I guess self.loc_float_const_neg = None self.loc_float_const_abs = None self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 self.setup_failure_recovery() + self._loop_counter = 0 + self._debug = False def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar llop.gc_assume_young_pointers(lltype.Void, llmemory.cast_ptr_to_adr(ptrs)) + def set_debug(self, v): + self._debug = v + def make_sure_mc_exists(self): if self.mc is None: # the address of the function called by 'new' @@ -157,6 +167,22 @@ self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + s = os.environ.get('PYPYLOG') + if s: + if s.find(':') != -1: + s = s.split(':')[-1] + self.set_debug(True) + self._output_loop_log = s + ".count" + + def finish_once(self): + if self._debug: + output_log = self._output_loop_log + assert output_log is not None + f = open_file_as_stream(output_log, "w") + for i in range(self._loop_counter): + f.write(self.loop_names[i] + ":" + + str(self.loop_run_counter.getitem(i)) + "\n") + f.close() def _build_float_constants(self): # 11 words: 8 words for the data, and up to 3 words for alignment @@ -207,9 +233,9 @@ _x86_param_depth _x86_arglocs """ + self.make_sure_mc_exists() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() regalloc = RegAlloc(self, self.cpu.translate_support_code) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs @@ -244,9 +270,9 @@ def assemble_bridge(self, faildescr, inputargs, operations): + self.make_sure_mc_exists() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() arglocs = self.rebuild_faillocs_from_descr( faildescr._x86_failure_recovery_bytecode) if not we_are_translated(): @@ -278,10 +304,18 @@ self.mc.end_function() def _find_debug_merge_point(self, operations): + for op in operations: if op.opnum == rop.DEBUG_MERGE_POINT: - return op.args[0]._get_str() - return "" + funcname = op.args[0]._get_str() + break + else: + funcname = "" % self._loop_counter + # invent the counter, so we don't get too confused + if self._debug: + self.loop_names.append(funcname) + self._loop_counter += 1 + return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset @@ -292,6 +326,15 @@ def _assemble(self, regalloc, operations): self._regalloc = regalloc + if self._debug: + # before doing anything, let's increase a counter + # we need one register free (a bit of a hack, but whatever) + self.mc.PUSH(eax) + adr = self.loop_run_counter.get_addr_for_num(self._loop_counter - 1) + self.mc.MOV(eax, heap(adr)) + self.mc.ADD(eax, imm(1)) + self.mc.MOV(heap(adr), eax) + self.mc.POP(eax) regalloc.walk_operations(operations) self.mc.done() self.mc2.done() Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/runner.py Thu Jul 29 15:38:08 2010 @@ -44,6 +44,7 @@ self.profile_agent.startup() def finish_once(self): + self.assembler.finish_once() self.profile_agent.shutdown() def compile_loop(self, inputargs, operations, looptoken): Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_runner.py Thu Jul 29 15:38:08 2010 @@ -9,8 +9,11 @@ from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.executor import execute from pypy.jit.backend.test.runner_test import LLtypeBackendTest +from pypy.jit.metainterp.test.oparser import parse +from pypy.tool.udir import udir import ctypes import sys +import os class FakeStats(object): pass @@ -390,3 +393,35 @@ self.cpu.set_future_value_int(0, base_v.value) self.cpu.execute_token(looptoken) assert self.cpu.get_latest_value_int(0) == 1024 + +class TestDebuggingAssembler(object): + def setup_method(self, meth): + self.pypylog = os.environ.get('PYPYLOG', None) + self.logfile = str(udir.join('x86_runner.log')) + os.environ['PYPYLOG'] = "mumble:" + self.logfile + self.cpu = CPU(rtyper=None, stats=FakeStats()) + + def teardown_method(self, meth): + if self.pypylog is not None: + os.environ['PYPYLOG'] = self.pypylog + + def test_debugger_on(self): + loop = """ + [i0] + debug_merge_point('xyz') + i1 = int_add(i0, 1) + i2 = int_ge(i1, 10) + guard_false(i2) [] + jump(i1) + """ + ops = parse(loop) + self.cpu.assembler.set_debug(True) + self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) + self.cpu.set_future_value_int(0, 0) + self.cpu.execute_token(ops.token) + # check debugging info + assert self.cpu.assembler.loop_names == ["xyz"] + assert self.cpu.assembler.loop_run_counter.getitem(0) == 10 + self.cpu.finish_once() + lines = py.path.local(self.logfile + ".count").readlines() + assert lines[0] == 'xyz:10\n' Modified: pypy/branch/fast-ctypes/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/metainterp/executor.py Thu Jul 29 15:38:08 2010 @@ -91,7 +91,7 @@ return BoxInt(cpu.bh_getarrayitem_gc_i(arraydescr, array, index)) def do_getarrayitem_raw(cpu, _, arraybox, indexbox, arraydescr): - array = arraybox.getref_base() + array = arraybox.getint() index = indexbox.getint() assert not arraydescr.is_array_of_pointers() if arraydescr.is_array_of_floats(): Modified: pypy/branch/fast-ctypes/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/metainterp/optimizeopt.py Thu Jul 29 15:38:08 2010 @@ -1001,6 +1001,17 @@ self.make_equal_to(op.result, v1) else: return self.optimize_default(op) + + def optimize_INT_ADD(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + # If one side of the op is 0 the result is the other side. + if v1.is_constant() and v1.box.getint() == 0: + self.make_equal_to(op.result, v2) + elif v2.is_constant() and v2.box.getint() == 0: + self.make_equal_to(op.result, v1) + else: + self.optimize_default(op) optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_loop.py Thu Jul 29 15:38:08 2010 @@ -9,6 +9,10 @@ class LoopTest(object): optimizer = OPTIMIZER_SIMPLE + automatic_promotion_result = { + 'int_add' : 6, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 3 + } def meta_interp(self, f, args, policy=None): return ll_meta_interp(f, args, optimizer=self.optimizer, @@ -477,9 +481,9 @@ res = self.meta_interp(main_interpreter_loop, [1]) assert res == main_interpreter_loop(1) self.check_loop_count(1) - # XXX maybe later optimize guard_value away - self.check_loops({'int_add' : 6, 'int_gt' : 1, - 'guard_false' : 1, 'jump' : 1, 'guard_value' : 3}) + # These loops do different numbers of ops based on which optimizer we + # are testing with. + self.check_loops(self.automatic_promotion_result) def test_can_enter_jit_outside_main_loop(self): myjitdriver = JitDriver(greens=[], reds=['i', 'j', 'a']) Modified: pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_loop_spec.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_loop_spec.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_loop_spec.py Thu Jul 29 15:38:08 2010 @@ -5,6 +5,10 @@ class LoopSpecTest(test_loop.LoopTest): optimizer = OPTIMIZER_FULL + automatic_promotion_result = { + 'int_add' : 3, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 1 + } # ====> test_loop.py Modified: pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 29 15:38:08 2010 @@ -2063,6 +2063,28 @@ jump(i0) """ self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(i0, 0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(0, i0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) # ---------- Modified: pypy/branch/fast-ctypes/pypy/module/__builtin__/compiling.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/__builtin__/compiling.py (original) +++ pypy/branch/fast-ctypes/pypy/module/__builtin__/compiling.py Thu Jul 29 15:38:08 2010 @@ -38,7 +38,7 @@ str_ = space.str_w(w_source) ec = space.getexecutioncontext() - if flags & ~(ec.compiler.compiler_flags | consts.PyCF_AST_ONLY | + if flags & ~(ec.compiler.compiler_flags | consts.PyCF_ONLY_AST | consts.PyCF_DONT_IMPLY_DEDENT | consts.PyCF_SOURCE_IS_UTF8): raise OperationError(space.w_ValueError, space.wrap("compile() unrecognized flags")) @@ -53,7 +53,7 @@ "or 'eval' or 'single'")) if ast_node is None: - if flags & consts.PyCF_AST_ONLY: + if flags & consts.PyCF_ONLY_AST: mod = ec.compiler.compile_to_ast(str_, filename, mode, flags) return space.wrap(mod) else: Modified: pypy/branch/fast-ctypes/pypy/module/_ast/__init__.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_ast/__init__.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_ast/__init__.py Thu Jul 29 15:38:08 2010 @@ -5,7 +5,7 @@ class Module(MixedModule): interpleveldefs = { - "PyCF_AST_ONLY" : "space.wrap(%s)" % consts.PyCF_AST_ONLY + "PyCF_ONLY_AST" : "space.wrap(%s)" % consts.PyCF_ONLY_AST } appleveldefs = {} Modified: pypy/branch/fast-ctypes/pypy/module/_ast/test/test_ast.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_ast/test/test_ast.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_ast/test/test_ast.py Thu Jul 29 15:38:08 2010 @@ -10,7 +10,7 @@ cls.w_get_ast = cls.space.appexec([], """(): def get_ast(source, mode="exec"): import _ast as ast - mod = compile(source, "", mode, ast.PyCF_AST_ONLY) + mod = compile(source, "", mode, ast.PyCF_ONLY_AST) assert isinstance(mod, ast.mod) return mod return get_ast""") Modified: pypy/branch/fast-ctypes/pypy/module/_file/interp_file.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_file/interp_file.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_file/interp_file.py Thu Jul 29 15:38:08 2010 @@ -4,6 +4,7 @@ from pypy.rlib.rarithmetic import r_longlong from pypy.module._file.interp_stream import W_AbstractStream from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror +from pypy.module.posix.interp_posix import dispatch_filename from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.typedef import TypeDef, GetSetProperty @@ -81,11 +82,11 @@ # file lock. They don't convert StreamErrors to OperationErrors, too. def direct___init__(self, w_name, mode='r', buffering=-1): - name = self.space.str_w(w_name) self.direct_close() self.w_name = w_name self.check_mode_ok(mode) - stream = streamio.open_file_as_stream(name, mode, buffering) + stream = dispatch_filename(streamio.open_file_as_stream)( + self.space, w_name, mode, buffering) fd = stream.try_to_find_file_descriptor() self.fdopenstream(stream, fd, mode) Modified: pypy/branch/fast-ctypes/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_file/test/test_file.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_file/test/test_file.py Thu Jul 29 15:38:08 2010 @@ -125,6 +125,15 @@ assert type(res) is str f.close() + def test_unicode_filename(self): + import sys + try: + u'\xe9'.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + skip("encoding not good enough") + f = self.file(self.temppath + u'\xe9', "w") + f.close() + def test_oserror_has_filename(self): try: f = self.file("file that is clearly not there") Modified: pypy/branch/fast-ctypes/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_rawffi/test/test__rawffi.py Thu Jul 29 15:38:08 2010 @@ -677,7 +677,12 @@ a = A(1) a[0] = -1234 a.free() - + + def test_long_with_fromaddress(self): + import _rawffi + addr = -1 + raises(ValueError, _rawffi.Array('u').fromaddress, addr, 100) + def test_passing_raw_pointers(self): import _rawffi lib = _rawffi.CDLL(self.lib_name) Modified: pypy/branch/fast-ctypes/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/fast-ctypes/pypy/module/cpyext/methodobject.py Thu Jul 29 15:38:08 2010 @@ -100,7 +100,11 @@ return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) def get_doc(space, self): - return space.wrap(rffi.charp2str(self.ml.c_ml_doc)) + doc = self.ml.c_ml_doc + if doc: + return space.wrap(rffi.charp2str(doc)) + else: + return space.w_None class W_PyCMethodObject(W_PyCFunctionObject): Modified: pypy/branch/fast-ctypes/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/fast-ctypes/pypy/module/posix/interp_posix.py Thu Jul 29 15:38:08 2010 @@ -1,8 +1,9 @@ from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong from pypy.rlib.unroll import unrolling_iterable -from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 from pypy.rpython.module.ll_os import RegisterOs from pypy.rpython.module import ll_os_stat from pypy.rpython.lltypesystem import rffi, lltype @@ -12,15 +13,78 @@ import os, sys _WIN = sys.platform == 'win32' -def open(space, fname, flag, mode=0777): +class FileEncoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_bytes = space.call_method(self.w_obj, 'encode', + getfilesystemencoding(space)) + return space.str_w(w_bytes) + + def as_unicode(self): + return self.space.unicode_w(self.w_obj) + +class FileDecoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + return self.space.str_w(self.w_obj) + + def as_unicode(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_unicode = space.call_method(self.w_obj, 'decode', + getfilesystemencoding(space)) + return space.unicode_w(w_unicode) + + at specialize.memo() +def dispatch_filename(func, tag=0): + def dispatch(space, w_fname, *args): + if space.isinstance_w(w_fname, space.w_unicode): + fname = FileEncoder(space, w_fname) + return func(fname, *args) + else: + fname = space.str_w(w_fname) + return func(fname, *args) + return dispatch + + at specialize.memo() +def dispatch_filename_2(func): + def dispatch(space, w_fname1, w_fname2, *args): + if space.isinstance_w(w_fname1, space.w_unicode): + fname1 = FileEncoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname1 = FileDecoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + return dispatch + +def open(space, w_fname, flag, mode=0777): """Open a file (for low level IO). Return a file descriptor (a small integer).""" - try: - fd = os.open(fname, flag, mode) + try: + fd = dispatch_filename(rposix.open)( + space, w_fname, flag, mode) except OSError, e: - raise wrap_oserror(space, e, fname) + raise wrap_oserror2(space, e, w_fname) return space.wrap(fd) -open.unwrap_spec = [ObjSpace, 'path', "c_int", "c_int"] +open.unwrap_spec = [ObjSpace, W_Root, "c_int", "c_int"] def lseek(space, fd, pos, how): """Set the current position of a file descriptor. Return the new position. @@ -159,7 +223,7 @@ return build_stat_result(space, st) fstat.unwrap_spec = [ObjSpace, "c_int"] -def stat(space, path): +def stat(space, w_path): """Perform a stat system call on the given path. Return an object with (at least) the following attributes: st_mode @@ -175,22 +239,22 @@ """ try: - st = os.stat(path) + st = dispatch_filename(rposix.stat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -stat.unwrap_spec = [ObjSpace, 'path'] +stat.unwrap_spec = [ObjSpace, W_Root] -def lstat(space, path): +def lstat(space, w_path): "Like stat(path), but do no follow symbolic links." try: - st = os.lstat(path) + st = dispatch_filename(rposix.lstat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -lstat.unwrap_spec = [ObjSpace, 'path'] +lstat.unwrap_spec = [ObjSpace, W_Root] class StatState(object): def __init__(self, space): @@ -231,7 +295,7 @@ raise wrap_oserror(space, e) dup2.unwrap_spec = [ObjSpace, "c_int", "c_int"] -def access(space, path, mode): +def access(space, w_path, mode): """ access(path, mode) -> 1 if granted, 0 otherwise @@ -242,12 +306,12 @@ existence, or the inclusive-OR of R_OK, W_OK, and X_OK. """ try: - ok = os.access(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) + ok = dispatch_filename(rposix.access)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) else: return space.wrap(ok) -access.unwrap_spec = [ObjSpace, str, "c_int"] +access.unwrap_spec = [ObjSpace, W_Root, "c_int"] def times(space): @@ -278,32 +342,38 @@ return space.wrap(rc) system.unwrap_spec = [ObjSpace, str] -def unlink(space, path): +def unlink(space, w_path): """Remove a file (same as remove(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -unlink.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +unlink.unwrap_spec = [ObjSpace, W_Root] -def remove(space, path): +def remove(space, w_path): """Remove a file (same as unlink(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -remove.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +remove.unwrap_spec = [ObjSpace, W_Root] -def _getfullpathname(space, path): +def _getfullpathname(space, w_path): """helper for ntpath.abspath """ - posix = __import__(os.name) # nt specific try: - fullpath = posix._getfullpathname(path) + if space.isinstance_w(w_path, space.w_unicode): + path = FileEncoder(space, w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) + else: + path = space.str_w(w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) except OSError, e: - raise wrap_oserror(space, e, path) - else: - return space.wrap(fullpath) -_getfullpathname.unwrap_spec = [ObjSpace, str] + raise wrap_oserror2(space, e, w_path) + else: + return w_fullpath +_getfullpathname.unwrap_spec = [ObjSpace, W_Root] def getcwd(space): """Return the current working directory.""" @@ -315,35 +385,46 @@ return space.wrap(cur) getcwd.unwrap_spec = [ObjSpace] -def getcwdu(space): - """Return the current working directory as a unicode string.""" - # XXX ascii encoding for now - return space.call_method(getcwd(space), 'decode') +if sys.platform == 'win32': + def getcwdu(space): + """Return the current working directory as a unicode string.""" + try: + cur = os.getcwdu() + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(cur) +else: + def getcwdu(space): + """Return the current working directory as a unicode string.""" + filesystemencoding = space.sys.filesystemencoding + return space.call_method(getcwd(space), 'decode', + space.wrap(filesystemencoding)) getcwdu.unwrap_spec = [ObjSpace] -def chdir(space, path): +def chdir(space, w_path): """Change the current working directory to the specified path.""" try: - os.chdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -chdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.chdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chdir.unwrap_spec = [ObjSpace, W_Root] -def mkdir(space, path, mode=0777): +def mkdir(space, w_path, mode=0777): """Create a directory.""" try: - os.mkdir(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -mkdir.unwrap_spec = [ObjSpace, str, "c_int"] + dispatch_filename(rposix.mkdir)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +mkdir.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rmdir(space, path): +def rmdir(space, w_path): """Remove a directory.""" try: - os.rmdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -rmdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.rmdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +rmdir.unwrap_spec = [ObjSpace, W_Root] def strerror(space, errno): """Translate an error code to a message string.""" @@ -410,7 +491,7 @@ unsetenv.unwrap_spec = [ObjSpace, str] -def listdir(space, dirname): +def listdir(space, w_dirname): """Return a list containing the names of the entries in the directory. \tpath: path of directory to list @@ -418,12 +499,18 @@ The list is in arbitrary order. It does not include the special entries '.' and '..' even if they are present in the directory.""" try: - result = os.listdir(dirname) + if space.isinstance_w(w_dirname, space.w_unicode): + dirname = FileEncoder(space, w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] + else: + dirname = space.str_w(w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] except OSError, e: - raise wrap_oserror(space, e, dirname) - result_w = [space.wrap(s) for s in result] + raise wrap_oserror2(space, e, w_dirname) return space.newlist(result_w) -listdir.unwrap_spec = [ObjSpace, str] +listdir.unwrap_spec = [ObjSpace, W_Root] def pipe(space): "Create a pipe. Returns (read_end, write_end)." @@ -434,21 +521,21 @@ return space.newtuple([space.wrap(fd1), space.wrap(fd2)]) pipe.unwrap_spec = [ObjSpace] -def chmod(space, path, mode): +def chmod(space, w_path, mode): "Change the access permissions of a file." - try: - os.chmod(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -chmod.unwrap_spec = [ObjSpace, str, "c_int"] + try: + dispatch_filename(rposix.chmod)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chmod.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rename(space, old, new): +def rename(space, w_old, w_new): "Rename a file or directory." - try: - os.rename(old, new) - except OSError, e: + try: + dispatch_filename_2(rposix.rename)(space, w_old, w_new) + except OSError, e: raise wrap_oserror(space, e) -rename.unwrap_spec = [ObjSpace, str, str] +rename.unwrap_spec = [ObjSpace, W_Root, W_Root] def umask(space, mask): "Set the current numeric umask and return the previous umask." @@ -576,7 +663,7 @@ raise wrap_oserror(space, e) execve.unwrap_spec = [ObjSpace, str, W_Root, W_Root] -def utime(space, path, w_tuple): +def utime(space, w_path, w_tuple): """ utime(path, (atime, mtime)) utime(path, None) @@ -585,10 +672,10 @@ """ if space.is_w(w_tuple, space.w_None): try: - os.utime(path, None) + dispatch_filename(rposix.utime, 1)(space, w_path, None) return except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) try: msg = "utime() arg 2 must be a tuple (atime, mtime) or None" args_w = space.fixedview(w_tuple) @@ -596,14 +683,14 @@ raise OperationError(space.w_TypeError, space.wrap(msg)) actime = space.float_w(args_w[0]) modtime = space.float_w(args_w[1]) - os.utime(path, (actime, modtime)) + dispatch_filename(rposix.utime, 2)(space, w_path, (actime, modtime)) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) except OperationError, e: if not e.match(space, space.w_TypeError): raise raise OperationError(space.w_TypeError, space.wrap(msg)) -utime.unwrap_spec = [ObjSpace, str, W_Root] +utime.unwrap_spec = [ObjSpace, W_Root, W_Root] def setsid(space): """setsid() -> pid Modified: pypy/branch/fast-ctypes/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/fast-ctypes/pypy/module/posix/test/test_posix2.py Thu Jul 29 15:38:08 2010 @@ -32,6 +32,9 @@ # even when running on top of CPython 2.4. os.stat_float_times(True) + # Initialize sys.filesystemencoding + space.call_method(space.getbuiltinmodule('sys'), 'getfilesystemencoding') + def need_sparse_files(): if sys.platform == 'darwin': py.test.skip("no sparse files on default Mac OS X file system") @@ -706,6 +709,28 @@ except OSError: pass +class AppTestUnicodeFilename: + def setup_class(cls): + ufilename = (unicode(udir.join('test_unicode_filename_')) + + u'\u65e5\u672c.txt') # "Japan" + try: + f = file(ufilename, 'w') + except UnicodeEncodeError: + py.test.skip("encoding not good enough") + f.write("test") + f.close() + cls.space = space + cls.w_filename = space.wrap(ufilename) + cls.w_posix = space.appexec([], GET_POSIX) + + def test_open(self): + fd = self.posix.open(self.filename, self.posix.O_RDONLY) + try: + content = self.posix.read(fd, 50) + finally: + self.posix.close(fd) + assert content == "test" + class TestPexpect(object): # XXX replace with AppExpectTest class as soon as possible Modified: pypy/branch/fast-ctypes/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/objectmodel.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/objectmodel.py Thu Jul 29 15:38:08 2010 @@ -19,29 +19,66 @@ # def f(... # -class _AttachSpecialization(object): +class _Specialize(object): + def memo(self): + """ Specialize functions based on argument values. All arguments has + to be constant at the compile time. The whole function call is replaced + by a call result then. + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:memo' + return func + return decorated_func - def __init__(self, tag): - self.tag = tag + def arg(self, *args): + """ Specialize function based on values of given positions of arguments. + They must be compile-time constants in order to work. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:arg' + self._wrap(args) + return func - def __call__(self, *args): - if not args: - args = "" - else: - args = "("+','.join([repr(arg) for arg in args]) +")" - specialcase = "specialize:%s%s" % (self.tag, args) - - def specialize_decorator(func): - "NOT_RPYTHON" - func._annspecialcase_ = specialcase + return decorated_func + + def argtype(self, *args): + """ Specialize function based on types of arguments on given positions. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:argtype' + self._wrap(args) return func - return specialize_decorator - -class _Specialize(object): + return decorated_func + + def ll(self): + """ This is version of argtypes that cares about low-level types + (so it'll get additional copies for two different types of pointers + for example). Same warnings about exponential behavior apply. + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:ll' + return func + + return decorated_func + + def ll_and_arg(self, arg): + """ XXX what does that do? + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:ll_and_arg(%d)' % arg + return func + + return decorated_func - def __getattr__(self, name): - return _AttachSpecialization(name) + def _wrap(self, args): + return "("+','.join([repr(arg) for arg in args]) +")" specialize = _Specialize() Modified: pypy/branch/fast-ctypes/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rposix.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rposix.py Thu Jul 29 15:38:08 2010 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rlib.rarithmetic import intmask +from pypy.rlib.objectmodel import specialize class CConstantErrno(CConstant): # these accessors are used when calling get_errno() or set_errno() @@ -42,3 +43,102 @@ os.close(fd) except OSError: pass + +#___________________________________________________________________ +# Wrappers around posix functions, that accept either strings, or +# instances with a "as_bytes()" method. +# - pypy.modules.posix.interp_posix passes an object containing a unicode path +# which can encode itself with sys.filesystemencoding. +# - but pypy.rpython.module.ll_os.py on Windows will replace these functions +# with other wrappers that directly handle unicode strings. + at specialize.argtype(0) +def open(path, flags, mode): + if isinstance(path, str): + return os.open(path, flags, mode) + else: + return os.open(path.as_bytes(), flags, mode) + + at specialize.argtype(0) +def stat(path): + if isinstance(path, str): + return os.stat(path) + else: + return os.stat(path.as_bytes()) + + at specialize.argtype(0) +def lstat(path): + if isinstance(path, str): + return os.lstat(path) + else: + return os.lstat(path.as_bytes()) + + at specialize.argtype(0) +def unlink(path): + if isinstance(path, str): + return os.unlink(path) + else: + return os.unlink(path.as_bytes()) + + at specialize.argtype(0, 1) +def rename(path1, path2): + if isinstance(path1, str): + return os.rename(path1, path2) + else: + return os.rename(path1.as_bytes(), path2.as_bytes()) + + at specialize.argtype(0) +def listdir(dirname): + if isinstance(dirname, str): + return os.listdir(dirname) + else: + return os.listdir(dirname.as_bytes()) + + at specialize.argtype(0) +def access(path, mode): + if isinstance(path, str): + return os.access(path, mode) + else: + return os.access(path.as_bytes(), mode) + + at specialize.argtype(0) +def chmod(path, mode): + if isinstance(path, str): + return os.chmod(path, mode) + else: + return os.chmod(path.as_bytes(), mode) + + at specialize.argtype(0, 1) +def utime(path, times): + if isinstance(path, str): + return os.utime(path, times) + else: + return os.utime(path.as_bytes(), times) + + at specialize.argtype(0) +def chdir(path): + if isinstance(path, str): + return os.chdir(path) + else: + return os.chdir(path.as_bytes()) + + at specialize.argtype(0) +def mkdir(path, mode=0777): + if isinstance(path, str): + return os.mkdir(path, mode) + else: + return os.mkdir(path.as_bytes(), mode) + + at specialize.argtype(0) +def rmdir(path): + if isinstance(path, str): + return os.rmdir(path) + else: + return os.rmdir(path.as_bytes()) + +if os.name == 'nt': + import nt + def _getfullpathname(path): + if isinstance(path, str): + return nt._getfullpathname(path) + else: + return nt._getfullpathname(path.as_bytes()) Modified: pypy/branch/fast-ctypes/pypy/rlib/rwin32.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rwin32.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rwin32.py Thu Jul 29 15:38:08 2010 @@ -31,6 +31,7 @@ DWORD = rffi_platform.SimpleType("DWORD", rffi.UINT) BOOL = rffi_platform.SimpleType("BOOL", rffi.LONG) BYTE = rffi_platform.SimpleType("BYTE", rffi.UCHAR) + WCHAR = rffi_platform.SimpleType("WCHAR", rffi.UCHAR) INT = rffi_platform.SimpleType("INT", rffi.INT) LONG = rffi_platform.SimpleType("LONG", rffi.LONG) PLONG = rffi_platform.SimpleType("PLONG", rffi.LONGP) @@ -38,6 +39,8 @@ LPCVOID = rffi_platform.SimpleType("LPCVOID", rffi.VOIDP) LPSTR = rffi_platform.SimpleType("LPSTR", rffi.CCHARP) LPCSTR = rffi_platform.SimpleType("LPCSTR", rffi.CCHARP) + LPWSTR = rffi_platform.SimpleType("LPWSTR", rffi.CWCHARP) + LPCWSTR = rffi_platform.SimpleType("LPCWSTR", rffi.CWCHARP) LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.INTP) SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T) ULONG_PTR = rffi_platform.SimpleType("ULONG_PTR", rffi.ULONG) @@ -87,6 +90,10 @@ GetLastError = winexternal('GetLastError', [], DWORD) SetLastError = winexternal('SetLastError', [DWORD], lltype.Void) + # In tests, the first call to GetLastError is always wrong, because error + # is hidden by operations in ll2ctypes. Call it now. + GetLastError() + LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], rffi.VOIDP) GetProcAddress = winexternal('GetProcAddress', [rffi.VOIDP, rffi.CCHARP], @@ -129,13 +136,29 @@ } return 0; }''') - exename = static_platform.compile( - [cfile], ExternalCompilationInfo(), - outputfilename = "dosmaperr", - standalone=True) - output = os.popen(str(exename)) - errors = dict(map(int, line.split()) - for line in output) + try: + exename = static_platform.compile( + [cfile], ExternalCompilationInfo(), + outputfilename = "dosmaperr", + standalone=True) + except WindowsError: + # Fallback for the mingw32 compiler + errors = { + 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, + 11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, + 22: 13, 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13, + 29: 13, 30: 13, 31: 13, 32: 13, 33: 13, 34: 13, 35: 13, + 36: 13, 53: 2, 65: 13, 67: 2, 80: 17, 82: 13, 83: 13, 89: 11, + 108: 13, 109: 32, 112: 28, 114: 9, 128: 10, 129: 10, 130: 9, + 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17, + 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8, + 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8, + 202: 8, 206: 2, 215: 11, 1816: 12, + } + else: + output = os.popen(str(exename)) + errors = dict(map(int, line.split()) + for line in output) return errors, errno.EINVAL # A bit like strerror... Modified: pypy/branch/fast-ctypes/pypy/rlib/streamio.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/streamio.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/streamio.py Thu Jul 29 15:38:08 2010 @@ -38,7 +38,9 @@ # import os, sys +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong, intmask +from pypy.rlib import rposix from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC O_BINARY = getattr(os, "O_BINARY", 0) @@ -71,6 +73,7 @@ return s.join(string.split(c)) + at specialize.argtype(0) def open_file_as_stream(path, mode="r", buffering=-1): os_flags, universal, reading, writing, basemode, binary = decode_mode(mode) stream = open_path_helper(path, os_flags, basemode == "a") @@ -89,9 +92,10 @@ return construct_stream_tower(stream, buffering, universal, reading, writing, binary) + at specialize.argtype(0) def open_path_helper(path, os_flags, append): # XXX for now always return DiskFile - fd = os.open(path, os_flags, 0666) + fd = rposix.open(path, os_flags, 0666) if append: try: os.lseek(fd, 0, 2) Modified: pypy/branch/fast-ctypes/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rpython/extfunc.py (original) +++ pypy/branch/fast-ctypes/pypy/rpython/extfunc.py Thu Jul 29 15:38:08 2010 @@ -52,7 +52,10 @@ return super(ExtRegistryEntry, self).__getattr__(attr) raise exc, exc_inst, tb -def registering(func): +def registering(func, condition=True): + if not condition: + return lambda method: None + def decorator(method): method._registering_func = func return method @@ -63,11 +66,9 @@ func = getattr(ns, name) except AttributeError: condition = False + func = None - if condition: - return registering(func) - else: - return lambda method: None + return registering(func, condition=condition) class LazyRegisteringMeta(type): def __new__(self, _name, _type, _vars): @@ -167,8 +168,6 @@ return signature_args def compute_result_annotation(self, *args_s): - if hasattr(self, 'ann_hook'): - self.ann_hook() self.normalize_args(*args_s) # check arguments return self.signature_result @@ -235,7 +234,6 @@ def register_external(function, args, result=None, export_name=None, llimpl=None, ooimpl=None, llfakeimpl=None, oofakeimpl=None, - annotation_hook=None, sandboxsafe=False): """ function: the RPython function that will be rendered as an external function (e.g.: math.floor) @@ -244,7 +242,6 @@ export_name: the name of the function as it will be seen by the backends llimpl, ooimpl: optional; if provided, these RPython functions are called instead of the target function llfakeimpl, oofakeimpl: optional; if provided, they are called by the llinterpreter - annotationhook: optional; a callable that is called during annotation, useful for genc hacks sandboxsafe: use True if the function performs no I/O (safe for --sandbox) """ @@ -271,8 +268,6 @@ lltypefakeimpl = staticmethod(llfakeimpl) if oofakeimpl: ootypefakeimpl = staticmethod(oofakeimpl) - if annotation_hook: - ann_hook = staticmethod(annotation_hook) if export_name: FunEntry.__name__ = export_name Modified: pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/rffi.py Thu Jul 29 15:38:08 2010 @@ -176,6 +176,15 @@ # XXX leaks if a str2charp() fails with MemoryError # and was not the first in this function freeme = arg + elif TARGET == CWCHARP: + if arg is None: + arg = lltype.nullptr(CWCHARP.TO) # None => (wchar_t*)NULL + freeme = arg + elif isinstance(arg, unicode): + arg = unicode2wcharp(arg) + # XXX leaks if a unicode2wcharp() fails with MemoryError + # and was not the first in this function + freeme = arg elif _isfunctype(TARGET) and not _isllptr(arg): # XXX pass additional arguments if invoke_around_handlers: Modified: pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/rstr.py Thu Jul 29 15:38:08 2010 @@ -674,6 +674,7 @@ res_index += item_len i += 1 return result + ll_join_strs._annenforceargs_ = [int, None] def ll_join_chars(length, chars): # no need to optimize this, will be replaced by string builder Modified: pypy/branch/fast-ctypes/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/fast-ctypes/pypy/rpython/module/ll_os.py Thu Jul 29 15:38:08 2010 @@ -6,12 +6,15 @@ # might be found in doc/rffi.txt import os, sys, errno +import py from pypy.rpython.module.support import ll_strcpy, OOSupport -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rlib.rarithmetic import r_longlong -from pypy.rpython.extfunc import BaseLazyRegistering +from pypy.rpython.extfunc import ( + BaseLazyRegistering, lazy_register, register_external) from pypy.rpython.extfunc import registering, registering_if, extdef -from pypy.annotation.model import SomeInteger, SomeString, SomeTuple, SomeFloat +from pypy.annotation.model import ( + SomeInteger, SomeString, SomeTuple, SomeFloat, SomeUnicodeString) from pypy.annotation.model import s_ImpossibleValue, s_None, s_Bool from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype @@ -26,7 +29,99 @@ from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.annlowlevel import llstr from pypy.rlib import rgc -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import keepalive_until_here, specialize + +def monkeypatch_rposix(posixfunc, unicodefunc, signature): + func_name = posixfunc.__name__ + + if hasattr(signature, '_default_signature_'): + signature = signature._default_signature_ + arglist = ['arg%d' % (i,) for i in range(len(signature))] + transformed_arglist = arglist[:] + for i, arg in enumerate(signature): + if arg is unicode: + transformed_arglist[i] = transformed_arglist[i] + '.as_unicode()' + + args = ', '.join(arglist) + transformed_args = ', '.join(transformed_arglist) + main_arg = 'arg%d' % (signature.index(unicode),) + + source = py.code.Source(""" + def %(func_name)s(%(args)s): + if isinstance(%(main_arg)s, str): + return posixfunc(%(args)s) + else: + return unicodefunc(%(transformed_args)s) + """ % locals()) + miniglobals = {'posixfunc' : posixfunc, + 'unicodefunc': unicodefunc, + '__name__': __name__, # for module name propagation + } + exec source.compile() in miniglobals + new_func = miniglobals[func_name] + specialized_args = [i for i in range(len(signature)) + if signature[i] in (unicode, None)] + new_func = specialize.argtype(*specialized_args)(new_func) + + # Monkeypatch the function in pypy.rlib.rposix + setattr(rposix, func_name, new_func) + +class StringTraits: + str = str + CHAR = rffi.CHAR + CCHARP = rffi.CCHARP + charp2str = staticmethod(rffi.charp2str) + str2charp = staticmethod(rffi.str2charp) + free_charp = staticmethod(rffi.free_charp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_' + name + +class UnicodeTraits: + str = unicode + CHAR = rffi.WCHAR_T + CCHARP = rffi.CWCHARP + charp2str = staticmethod(rffi.wcharp2unicode) + str2charp = staticmethod(rffi.unicode2wcharp) + free_charp = staticmethod(rffi.free_wcharp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + 'w' + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_w' + name + +def registering_str_unicode(posixfunc, condition=True): + if not condition: + return registering(None, condition=False) + + func_name = posixfunc.__name__ + + def register_posixfunc(self, method): + val = method(self, StringTraits()) + register_external(posixfunc, *val.def_args, **val.def_kwds) + + if sys.platform == 'win32': + val = method(self, UnicodeTraits()) + @func_renamer(func_name + "_unicode") + def unicodefunc(*args): + return posixfunc(*args) + register_external(unicodefunc, *val.def_args, **val.def_kwds) + signature = val.def_args[0] + monkeypatch_rposix(posixfunc, unicodefunc, signature) + + def decorator(method): + decorated = lambda self: register_posixfunc(self, method) + decorated._registering_func = posixfunc + return decorated + return decorator posix = __import__(os.name) @@ -282,8 +377,8 @@ return extdef([int, int], s_None, llimpl=dup2_llimpl, export_name="ll_os.ll_os_dup2") - @registering(os.utime) - def register_os_utime(self): + @registering_str_unicode(os.utime) + def register_os_utime(self, traits): UTIMBUFP = lltype.Ptr(self.UTIMBUF) os_utime = self.llexternal('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT) @@ -336,6 +431,9 @@ # tp is known to be None, and one version where it is known # to be a tuple of 2 floats. if not _WIN32: + assert traits.str is str + + @specialize.argtype(1) def os_utime_llimpl(path, tp): if tp is None: error = os_utime(path, lltype.nullptr(UTIMBUFP.TO)) @@ -346,85 +444,13 @@ if error == -1: raise OSError(rposix.get_errno(), "os_utime failed") else: - from pypy.rlib import rwin32 - from pypy.rpython.module.ll_os_stat import time_t_to_FILE_TIME - - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'], - ) - - FILE_WRITE_ATTRIBUTES = platform.ConstantInteger( - 'FILE_WRITE_ATTRIBUTES') - OPEN_EXISTING = platform.ConstantInteger( - 'OPEN_EXISTING') - FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger( - 'FILE_FLAG_BACKUP_SEMANTICS') - globals().update(platform.configure(CConfig)) - - CreateFile = rffi.llexternal( - 'CreateFileA', - [rwin32.LPCSTR, rwin32.DWORD, rwin32.DWORD, - rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD, - rwin32.HANDLE], - rwin32.HANDLE, - calling_conv='win') - - GetSystemTime = rffi.llexternal( - 'GetSystemTime', - [lltype.Ptr(rwin32.SYSTEMTIME)], - lltype.Void, - calling_conv='win') - - SystemTimeToFileTime = rffi.llexternal( - 'SystemTimeToFileTime', - [lltype.Ptr(rwin32.SYSTEMTIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv='win') - - SetFileTime = rffi.llexternal( - 'SetFileTime', - [rwin32.HANDLE, - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv = 'win') + from pypy.rpython.module.ll_win32file import make_utime_impl + os_utime_llimpl = make_utime_impl(traits) - def os_utime_llimpl(path, tp): - hFile = CreateFile(path, - FILE_WRITE_ATTRIBUTES, 0, - None, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, 0) - if hFile == rwin32.INVALID_HANDLE_VALUE: - raise rwin32.lastWindowsError() - ctime = lltype.nullptr(rwin32.FILETIME) - atime = lltype.malloc(rwin32.FILETIME, flavor='raw') - mtime = lltype.malloc(rwin32.FILETIME, flavor='raw') - try: - if tp is None: - now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw') - try: - GetSystemTime(now) - if (not SystemTimeToFileTime(now, atime) or - not SystemTimeToFileTime(now, mtime)): - raise rwin32.lastWindowsError() - finally: - lltype.free(now, flavor='raw') - else: - actime, modtime = tp - time_t_to_FILE_TIME(actime, atime) - time_t_to_FILE_TIME(modtime, mtime) - if not SetFileTime(hFile, ctime, atime, mtime): - raise rwin32.lastWindowsError() - finally: - rwin32.CloseHandle(hFile) - lltype.free(atime, flavor='raw') - lltype.free(mtime, flavor='raw') - os_utime_llimpl._annspecialcase_ = 'specialize:argtype(1)' - - s_string = SomeString() + if traits.str is str: + s_string = SomeString() + else: + s_string = SomeUnicodeString() s_tuple_of_2_floats = SomeTuple([SomeFloat(), SomeFloat()]) def os_utime_normalize_args(s_path, s_times): @@ -445,12 +471,12 @@ else: raise Exception("os.utime() arg 2 must be None or a tuple of " "2 floats, got %s" % (s_times,)) + os_utime_normalize_args._default_signature_ = [traits.str, None] return extdef(os_utime_normalize_args, s_None, "ll_os.ll_os_utime", llimpl=os_utime_llimpl) - @registering(os.times) def register_os_times(self): if sys.platform.startswith('win'): @@ -687,22 +713,21 @@ def register_os_setsid(self): return self.extdef_for_os_function_returning_int('setsid') - @registering(os.open) - def register_os_open(self): - os_open = self.llexternal(underscore_on_windows+'open', - [rffi.CCHARP, rffi.INT, rffi.MODE_T], + @registering_str_unicode(os.open) + def register_os_open(self, traits): + os_open = self.llexternal(traits.posix_function_name('open'), + [traits.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT) - def os_open_llimpl(path, flags, mode): result = rffi.cast(rffi.LONG, os_open(path, flags, mode)) if result == -1: raise OSError(rposix.get_errno(), "os_open failed") return result - def os_open_oofakeimpl(o_path, flags, mode): - return os.open(o_path._str, flags, mode) + def os_open_oofakeimpl(path, flags, mode): + return os.open(OOSupport.from_rstr(path), flags, mode) - return extdef([str, int, int], int, "ll_os.ll_os_open", + return extdef([traits.str, int, int], int, traits.ll_os_name('open'), llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) # ------------------------------- os.read ------------------------------- @@ -862,10 +887,10 @@ llimpl=fdatasync_llimpl, export_name="ll_os.ll_os_fdatasync") - @registering(os.access) - def register_os_access(self): - os_access = self.llexternal(underscore_on_windows + 'access', - [rffi.CCHARP, rffi.INT], + @registering_str_unicode(os.access) + def register_os_access(self, traits): + os_access = self.llexternal(traits.posix_function_name('access'), + [traits.CCHARP, rffi.INT], rffi.INT) if sys.platform.startswith('win'): @@ -882,44 +907,22 @@ def os_access_oofakeimpl(path, mode): return os.access(OOSupport.from_rstr(path), mode) - return extdef([str, int], s_Bool, llimpl=access_llimpl, - export_name="ll_os.ll_os_access", + return extdef([traits.str, int], s_Bool, llimpl=access_llimpl, + export_name=traits.ll_os_name("access"), oofakeimpl=os_access_oofakeimpl) - @registering_if(posix, '_getfullpathname') - def register_posix__getfullpathname(self): - from pypy.rlib import rwin32 + @registering_str_unicode(getattr(posix, '_getfullpathname', None), + condition=sys.platform=='win32') + def register_posix__getfullpathname(self, traits): # this nt function is not exposed via os, but needed # to get a correct implementation of os.abspath - # XXX why do we ignore WINAPI conventions everywhere? - LPSTRP = rffi.CArrayPtr(rwin32.LPSTR) - # XXX unicode? - GetFullPathName = self.llexternal( - 'GetFullPathNameA', - [rwin32.LPCSTR, - rwin32.DWORD, - rwin32.LPSTR, - rffi.CArrayPtr(rwin32.LPSTR)], - rwin32.DWORD) - - def _getfullpathname_llimpl(lpFileName): - nBufferLength = rwin32.MAX_PATH + 1 - lpBuffer = lltype.malloc(rwin32.LPSTR.TO, nBufferLength, flavor='raw') - try: - res = GetFullPathName( - lpFileName, rffi.cast(rwin32.DWORD, nBufferLength), - lpBuffer, lltype.nullptr(LPSTRP.TO)) - if res == 0: - raise rwin32.lastWindowsError("_getfullpathname failed") - result = rffi.charp2str(lpBuffer) - return result - finally: - lltype.free(lpBuffer, flavor='raw') + from pypy.rpython.module.ll_win32file import make_getfullpathname_impl + getfullpathname_llimpl = make_getfullpathname_impl(traits) - return extdef([str], # a single argument which is a str - str, # returns a string - "ll_os.posix__getfullpathname", - llimpl=_getfullpathname_llimpl) + return extdef([traits.str], # a single argument which is a str + traits.str, # returns a string + traits.ll_os_name('_getfullpathname'), + llimpl=getfullpathname_llimpl) @registering(os.getcwd) def register_os_getcwd(self): @@ -953,71 +956,42 @@ "ll_os.ll_os_getcwd", llimpl=os_getcwd_llimpl, oofakeimpl=os_getcwd_oofakeimpl) - @registering(os.listdir) - def register_os_listdir(self): - # we need a different approach on Windows and on Posix - if sys.platform.startswith('win'): - from pypy.rlib import rwin32 - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'] - ) - WIN32_FIND_DATA = platform.Struct('struct _WIN32_FIND_DATAA', - [('cFileName', lltype.FixedSizeArray(rffi.CHAR, 1))]) - ERROR_FILE_NOT_FOUND = platform.ConstantInteger( - 'ERROR_FILE_NOT_FOUND') - ERROR_NO_MORE_FILES = platform.ConstantInteger( - 'ERROR_NO_MORE_FILES') + @registering(os.getcwdu, condition=sys.platform=='win32') + def register_os_getcwdu(self): + os_wgetcwd = self.llexternal(underscore_on_windows + 'wgetcwd', + [rffi.CWCHARP, rffi.SIZE_T], + rffi.CWCHARP) - config = platform.configure(CConfig) - WIN32_FIND_DATA = config['WIN32_FIND_DATA'] - ERROR_FILE_NOT_FOUND = config['ERROR_FILE_NOT_FOUND'] - ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] - LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) - - FindFirstFile = self.llexternal('FindFirstFile', - [rwin32.LPCSTR, LPWIN32_FIND_DATA], - rwin32.HANDLE) - FindNextFile = self.llexternal('FindNextFile', - [rwin32.HANDLE, LPWIN32_FIND_DATA], - rwin32.BOOL) - FindClose = self.llexternal('FindClose', - [rwin32.HANDLE], - rwin32.BOOL) + def os_getcwd_llimpl(): + bufsize = 256 + while True: + buf = lltype.malloc(rffi.CWCHARP.TO, bufsize, flavor='raw') + res = os_wgetcwd(buf, rffi.cast(rffi.SIZE_T, bufsize)) + if res: + break # ok + error = rposix.get_errno() + lltype.free(buf, flavor='raw') + if error != errno.ERANGE: + raise OSError(error, "getcwd failed") + # else try again with a larger buffer, up to some sane limit + bufsize *= 4 + if bufsize > 1024*1024: # xxx hard-coded upper limit + raise OSError(error, "getcwd result too large") + result = rffi.wcharp2unicode(res) + lltype.free(buf, flavor='raw') + return result - def os_listdir_llimpl(path): - if path and path[-1] not in ('/', '\\', ':'): - path += '/' - path += '*.*' - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - try: - result = [] - hFindFile = FindFirstFile(path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - error = rwin32.GetLastError() - if error == ERROR_FILE_NOT_FOUND: - return result - else: - raise WindowsError(error, "FindFirstFile failed") - while True: - name = rffi.charp2str(rffi.cast(rffi.CCHARP, - filedata.c_cFileName)) - if name != "." and name != "..": # skip these - result.append(name) - if not FindNextFile(hFindFile, filedata): - break - # FindNextFile sets error to ERROR_NO_MORE_FILES if - # it got to the end of the directory - error = rwin32.GetLastError() - FindClose(hFindFile) - if error == ERROR_NO_MORE_FILES: - return result - else: - raise WindowsError(error, "FindNextFile failed") - finally: - lltype.free(filedata, flavor='raw') + return extdef([], unicode, + "ll_os.ll_os_wgetcwd", llimpl=os_getcwd_llimpl) + @registering_str_unicode(os.listdir) + def register_os_listdir(self, traits): + # we need a different approach on Windows and on Posix + if sys.platform.startswith('win'): + from pypy.rpython.module.ll_win32file import make_listdir_impl + os_listdir_llimpl = make_listdir_impl(traits) else: + assert traits.str is str compilation_info = ExternalCompilationInfo( includes = ['sys/types.h', 'dirent.h'] ) @@ -1057,9 +1031,9 @@ raise OSError(error, "os_readdir failed") return result - return extdef([str], # a single argument which is a str - [str], # returns a list of strings - "ll_os.ll_os_listdir", + return extdef([traits.str], # a single argument which is a str + [traits.str], # returns a list of strings + traits.ll_os_name('listdir'), llimpl=os_listdir_llimpl) @registering(os.pipe) @@ -1234,38 +1208,40 @@ return extdef([str], int, llimpl=system_llimpl, export_name="ll_os.ll_os_system") - @registering(os.unlink) - def register_os_unlink(self): - os_unlink = self.llexternal(underscore_on_windows+'unlink', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.unlink) + def register_os_unlink(self, traits): + os_unlink = self.llexternal(traits.posix_function_name('unlink'), + [traits.CCHARP], rffi.INT) def unlink_llimpl(pathname): res = rffi.cast(lltype.Signed, os_unlink(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_unlink failed") - return extdef([str], s_None, llimpl=unlink_llimpl, - export_name="ll_os.ll_os_unlink") + return extdef([traits.str], s_None, llimpl=unlink_llimpl, + export_name=traits.ll_os_name('unlink')) - @registering(os.chdir) - def register_os_chdir(self): - os_chdir = self.llexternal(underscore_on_windows+'chdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.chdir) + def register_os_chdir(self, traits): + os_chdir = self.llexternal(traits.posix_function_name('chdir'), + [traits.CCHARP], rffi.INT) def chdir_llimpl(path): res = rffi.cast(lltype.Signed, os_chdir(path)) if res < 0: raise OSError(rposix.get_errno(), "os_chdir failed") - return extdef([str], s_None, llimpl=chdir_llimpl, - export_name="ll_os.ll_os_chdir") + return extdef([traits.str], s_None, llimpl=chdir_llimpl, + export_name=traits.ll_os_name('chdir')) - @registering(os.mkdir) - def register_os_mkdir(self): + @registering_str_unicode(os.mkdir) + def register_os_mkdir(self, traits): if os.name == 'nt': ARG2 = [] # no 'mode' argument on Windows - just ignored else: ARG2 = [rffi.MODE_T] - os_mkdir = self.llexternal(underscore_on_windows+'mkdir', - [rffi.CCHARP]+ARG2, rffi.INT) + os_mkdir = self.llexternal(traits.posix_function_name('mkdir'), + [traits.CCHARP] + ARG2, rffi.INT) IGNORE_MODE = len(ARG2) == 0 def mkdir_llimpl(pathname, mode): @@ -1277,46 +1253,47 @@ if res < 0: raise OSError(rposix.get_errno(), "os_mkdir failed") - return extdef([str, int], s_None, llimpl=mkdir_llimpl, - export_name="ll_os.ll_os_mkdir") + return extdef([traits.str, int], s_None, llimpl=mkdir_llimpl, + export_name=traits.ll_os_name('mkdir')) - @registering(os.rmdir) - def register_os_rmdir(self): - os_rmdir = self.llexternal(underscore_on_windows+'rmdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.rmdir) + def register_os_rmdir(self, traits): + os_rmdir = self.llexternal(traits.posix_function_name('rmdir'), + [traits.CCHARP], rffi.INT) def rmdir_llimpl(pathname): res = rffi.cast(lltype.Signed, os_rmdir(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_rmdir failed") - return extdef([str], s_None, llimpl=rmdir_llimpl, - export_name="ll_os.ll_os_rmdir") + return extdef([traits.str], s_None, llimpl=rmdir_llimpl, + export_name=traits.ll_os_name('rmdir')) - @registering(os.chmod) - def register_os_chmod(self): - os_chmod = self.llexternal(underscore_on_windows+'chmod', [rffi.CCHARP, rffi.MODE_T], - rffi.INT) + @registering_str_unicode(os.chmod) + def register_os_chmod(self, traits): + os_chmod = self.llexternal(traits.posix_function_name('chmod'), + [traits.CCHARP, rffi.MODE_T], rffi.INT) def chmod_llimpl(path, mode): res = rffi.cast(lltype.Signed, os_chmod(path, rffi.cast(rffi.MODE_T, mode))) if res < 0: raise OSError(rposix.get_errno(), "os_chmod failed") - return extdef([str, int], s_None, llimpl=chmod_llimpl, - export_name="ll_os.ll_os_chmod") + return extdef([traits.str, int], s_None, llimpl=chmod_llimpl, + export_name=traits.ll_os_name('chmod')) - @registering(os.rename) - def register_os_rename(self): - os_rename = self.llexternal('rename', [rffi.CCHARP, rffi.CCHARP], - rffi.INT) + @registering_str_unicode(os.rename) + def register_os_rename(self, traits): + os_rename = self.llexternal(traits.posix_function_name('rename'), + [traits.CCHARP, traits.CCHARP], rffi.INT) def rename_llimpl(oldpath, newpath): res = rffi.cast(lltype.Signed, os_rename(oldpath, newpath)) if res < 0: raise OSError(rposix.get_errno(), "os_rename failed") - return extdef([str, str], s_None, llimpl=rename_llimpl, - export_name="ll_os.ll_os_rename") + return extdef([traits.str, traits.str], s_None, llimpl=rename_llimpl, + export_name=traits.ll_os_name('rename')) @registering(os.umask) def register_os_umask(self): @@ -1425,17 +1402,17 @@ @registering(os.fstat) def register_os_fstat(self): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('fstat') + return ll_os_stat.register_stat_variant('fstat', StringTraits()) - @registering(os.stat) - def register_os_stat(self): + @registering_str_unicode(os.stat) + def register_os_stat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('stat') + return ll_os_stat.register_stat_variant('stat', traits) - @registering(os.lstat) - def register_os_lstat(self): + @registering_str_unicode(os.lstat) + def register_os_lstat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('lstat') + return ll_os_stat.register_stat_variant('lstat', traits) # ------------------------------- os.W* --------------------------------- Modified: pypy/branch/fast-ctypes/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/branch/fast-ctypes/pypy/rpython/module/ll_os_stat.py Thu Jul 29 15:38:08 2010 @@ -5,13 +5,14 @@ import os, sys from pypy.annotation import model as annmodel from pypy.tool.pairtype import pairtype -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rpython import extregistry -from pypy.rpython.extfunc import register_external +from pypy.rpython.extfunc import register_external, extdef from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.annlowlevel import hlstr @@ -211,13 +212,27 @@ return make_stat_result(result) -def register_stat_variant(name): - if sys.platform.startswith('win'): - _functions = {'stat': '_stati64', - 'fstat': '_fstati64', - 'lstat': '_stati64'} # no lstat on Windows - c_func_name = _functions[name] - elif sys.platform.startswith('linux'): +def register_stat_variant(name, traits): + if name != 'fstat': + arg_is_path = True + s_arg = traits.str + ARG1 = traits.CCHARP + else: + arg_is_path = False + s_arg = int + ARG1 = rffi.INT + + if sys.platform == 'win32': + # See Win32 implementation below + posix_stat_llimpl = make_win32_stat_impl(name, traits) + + return extdef( + [s_arg], s_StatResult, traits.ll_os_name(name), + llimpl=posix_stat_llimpl) + + assert traits.str is str + + if sys.platform.startswith('linux'): # because we always use _FILE_OFFSET_BITS 64 - this helps things work that are not a c compiler _functions = {'stat': 'stat64', 'fstat': 'fstat64', @@ -226,22 +241,26 @@ else: c_func_name = name - arg_is_path = (name != 'fstat') + posix_mystat = rffi.llexternal(c_func_name, + [ARG1, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info) + @func_renamer('os_%s_llimpl' % (name,)) def posix_stat_llimpl(arg): stresult = lltype.malloc(STAT_STRUCT.TO, flavor='raw') try: if arg_is_path: - arg = rffi.str2charp(arg) + arg = traits.str2charp(arg) error = rffi.cast(rffi.LONG, posix_mystat(arg, stresult)) if arg_is_path: - rffi.free_charp(arg) + traits.free_charp(arg) if error != 0: raise OSError(rposix.get_errno(), "os_?stat failed") return build_stat_result(stresult) finally: lltype.free(stresult, flavor='raw') + @func_renamer('os_%s_fake' % (name,)) def posix_fakeimpl(arg): if s_arg == str: arg = hlstr(arg) @@ -259,40 +278,17 @@ setattr(ll_tup, 'item%d' % i, val) return ll_tup - if arg_is_path: - s_arg = str - ARG1 = rffi.CCHARP - else: - s_arg = int - ARG1 = rffi.INT + return extdef( + [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), + llimpl=posix_stat_llimpl, llfakeimpl=posix_fakeimpl) - if sys.platform != 'win32': - posix_mystat = rffi.llexternal(c_func_name, - [ARG1, STAT_STRUCT], rffi.INT, - compilation_info=compilation_info) - - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(posix_stat_llimpl, - 'os_%s_llimpl' % (name,)), - llfakeimpl=func_with_new_name(posix_fakeimpl, - 'os_%s_fake' % (name,)), - ) - else: - # See Win32 implementation below - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(globals()['win32_%s_llimpl' % (name,)], - 'os_%s_llimpl' % (name,)), - ) +def make_win32_stat_impl(name, traits): + from pypy.rlib import rwin32 + from pypy.rpython.module.ll_win32file import make_win32_traits + win32traits = make_win32_traits(traits) -# ____________________________________________________________ -if sys.platform == 'win32': # The CRT of Windows has a number of flaws wrt. its stat() implementation: - # - for when we implement subsecond resolution in RPython, time stamps - # would be restricted to second resolution + # - time stamps are restricted to second resolution # - file modification times suffer from forth-and-back conversions between # UTC and local time # Therefore, we implement our own stat, based on the Win32 API directly. @@ -302,122 +298,18 @@ assert len(STAT_FIELDS) == 10 # no extra fields on Windows - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h', 'winbase.h', 'sys/stat.h'], - ) - - GetFileExInfoStandard = platform.ConstantInteger( - 'GetFileExInfoStandard') - FILE_ATTRIBUTE_DIRECTORY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_DIRECTORY') - FILE_ATTRIBUTE_READONLY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_READONLY') - ERROR_SHARING_VIOLATION = platform.ConstantInteger( - 'ERROR_SHARING_VIOLATION') - _S_IFDIR = platform.ConstantInteger('_S_IFDIR') - _S_IFREG = platform.ConstantInteger('_S_IFREG') - _S_IFCHR = platform.ConstantInteger('_S_IFCHR') - _S_IFIFO = platform.ConstantInteger('_S_IFIFO') - FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN') - FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') - FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') - - WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( - 'WIN32_FILE_ATTRIBUTE_DATA', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - BY_HANDLE_FILE_INFORMATION = platform.Struct( - 'BY_HANDLE_FILE_INFORMATION', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('nNumberOfLinks', rwin32.DWORD), - ('nFileIndexHigh', rwin32.DWORD), - ('nFileIndexLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - WIN32_FIND_DATA = platform.Struct( - 'WIN32_FIND_DATAA', - # Only interesting fields - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - globals().update(platform.configure(CConfig)) - GET_FILEEX_INFO_LEVELS = rffi.ULONG # an enumeration - - GetFileAttributesEx = rffi.llexternal( - 'GetFileAttributesExA', - [rffi.CCHARP, GET_FILEEX_INFO_LEVELS, - lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)], - rwin32.BOOL, - calling_conv='win') - - GetFileInformationByHandle = rffi.llexternal( - 'GetFileInformationByHandle', - [rwin32.HANDLE, lltype.Ptr(BY_HANDLE_FILE_INFORMATION)], - rwin32.BOOL, - calling_conv='win') - - GetFileType = rffi.llexternal( - 'GetFileType', - [rwin32.HANDLE], - rwin32.DWORD, - calling_conv='win') - - FindFirstFile = rffi.llexternal( - 'FindFirstFileA', - [rffi.CCHARP, lltype.Ptr(WIN32_FIND_DATA)], - rwin32.HANDLE, - calling_conv='win') - - FindClose = rffi.llexternal( - 'FindClose', - [rwin32.HANDLE], - rwin32.BOOL, - calling_conv='win') - def attributes_to_mode(attributes): m = 0 - if attributes & FILE_ATTRIBUTE_DIRECTORY: - m |= _S_IFDIR | 0111 # IFEXEC for user,group,other + if attributes & win32traits.FILE_ATTRIBUTE_DIRECTORY: + m |= win32traits._S_IFDIR | 0111 # IFEXEC for user,group,other else: - m |= _S_IFREG - if attributes & FILE_ATTRIBUTE_READONLY: + m |= win32traits._S_IFREG + if attributes & win32traits.FILE_ATTRIBUTE_READONLY: m |= 0444 else: m |= 0666 return m - def make_longlong(high, low): - return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) - - # Seconds between 1.1.1601 and 1.1.1970 - secs_between_epochs = lltype.r_longlong(11644473600) - - def FILE_TIME_to_time_t_nsec(filetime): - ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) - # FILETIME is in units of 100 nsec - nsec = (ft % 10000000) * 100 - time = (ft / 10000000) - secs_between_epochs - return time, nsec - - def time_t_to_FILE_TIME(time, filetime): - ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) - filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) - filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) - def attribute_data_to_stat(info): st_mode = attributes_to_mode(info.c_dwFileAttributes) st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) @@ -456,65 +348,94 @@ return make_stat_result(result) def attributes_from_dir(l_path, data): - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - hFindFile = FindFirstFile(l_path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - return 0 - FindClose(hFindFile) - data.c_dwFileAttributes = filedata.c_dwFileAttributes - rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) - rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) - rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) - data.c_nFileSizeHigh = filedata.c_nFileSizeHigh - data.c_nFileSizeLow = filedata.c_nFileSizeLow - return 1 + filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') + try: + hFindFile = win32traits.FindFirstFile(l_path, filedata) + if hFindFile == rwin32.INVALID_HANDLE_VALUE: + return 0 + win32traits.FindClose(hFindFile) + data.c_dwFileAttributes = filedata.c_dwFileAttributes + rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) + rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) + rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) + data.c_nFileSizeHigh = filedata.c_nFileSizeHigh + data.c_nFileSizeLow = filedata.c_nFileSizeLow + return 1 + finally: + lltype.free(filedata, flavor='raw') def win32_stat_llimpl(path): - data = lltype.malloc(WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') + data = lltype.malloc(win32traits.WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') try: - l_path = rffi.str2charp(path) - res = GetFileAttributesEx(l_path, GetFileExInfoStandard, data) + l_path = traits.str2charp(path) + res = win32traits.GetFileAttributesEx(l_path, win32traits.GetFileExInfoStandard, data) errcode = rwin32.GetLastError() if res == 0: - if errcode == ERROR_SHARING_VIOLATION: + if errcode == win32traits.ERROR_SHARING_VIOLATION: res = attributes_from_dir(l_path, data) errcode = rwin32.GetLastError() - rffi.free_charp(l_path) + traits.free_charp(l_path) if res == 0: raise WindowsError(errcode, "os_stat failed") return attribute_data_to_stat(data) finally: lltype.free(data, flavor='raw') - win32_lstat_llimpl = win32_stat_llimpl def win32_fstat_llimpl(fd): handle = rwin32._get_osfhandle(fd) - filetype = GetFileType(handle) - if filetype == FILE_TYPE_CHAR: + filetype = win32traits.GetFileType(handle) + if filetype == win32traits.FILE_TYPE_CHAR: # console or LPT device - return make_stat_result((_S_IFCHR, + return make_stat_result((win32traits._S_IFCHR, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_PIPE: + elif filetype == win32traits.FILE_TYPE_PIPE: # socket or named pipe - return make_stat_result((_S_IFIFO, + return make_stat_result((win32traits._S_IFIFO, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_UNKNOWN: + elif filetype == win32traits.FILE_TYPE_UNKNOWN: error = rwin32.GetLastError() if error != 0: raise WindowsError(error, "os_fstat failed") # else: unknown but valid file # normal disk file (FILE_TYPE_DISK) - info = lltype.malloc(BY_HANDLE_FILE_INFORMATION, flavor='raw', - zero=True) + info = lltype.malloc(win32traits.BY_HANDLE_FILE_INFORMATION, + flavor='raw', zero=True) try: - res = GetFileInformationByHandle(handle, info) + res = win32traits.GetFileInformationByHandle(handle, info) if res == 0: raise WindowsError(rwin32.GetLastError(), "os_fstat failed") return by_handle_info_to_stat(info) finally: lltype.free(info, flavor='raw') + if name == 'fstat': + return win32_fstat_llimpl + else: + return win32_stat_llimpl + + +#__________________________________________________ +# Helper functions for win32 + +def make_longlong(high, low): + return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) + +# Seconds between 1.1.1601 and 1.1.1970 +secs_between_epochs = lltype.r_longlong(11644473600) + +def FILE_TIME_to_time_t_nsec(filetime): + ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) + # FILETIME is in units of 100 nsec + nsec = (ft % 10000000) * 100 + time = (ft / 10000000) - secs_between_epochs + return time, nsec + +def time_t_to_FILE_TIME(time, filetime): + ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) + filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) + filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) + Modified: pypy/branch/fast-ctypes/pypy/rpython/module/test/test_ll_os_stat.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rpython/module/test/test_ll_os_stat.py (original) +++ pypy/branch/fast-ctypes/pypy/rpython/module/test/test_ll_os_stat.py Thu Jul 29 15:38:08 2010 @@ -1,4 +1,4 @@ -from pypy.rpython.module import ll_os_stat +from pypy.rpython.module import ll_os_stat, ll_os import sys, os import py @@ -8,14 +8,18 @@ py.test.skip("win32 specific tests") def test_stat(self): - stat = ll_os_stat.win32_stat_llimpl + stat = ll_os_stat.make_win32_stat_impl('stat', ll_os.StringTraits()) + wstat = ll_os_stat.make_win32_stat_impl('stat', ll_os.UnicodeTraits()) def check(f): - assert stat(f).st_mtime == os.stat(f).st_mtime + expected = os.stat(f).st_mtime + assert stat(f).st_mtime == expected + assert wstat(unicode(f)).st_mtime == expected check('c:/') check('c:/temp') check('c:/pagefile.sys') def test_fstat(self): - stat = ll_os_stat.win32_fstat_llimpl(0) # stdout + fstat = ll_os_stat.make_win32_stat_impl('fstat', ll_os.StringTraits()) + stat = fstat(0) # stdout assert stat.st_mode != 0 Modified: pypy/branch/fast-ctypes/pypy/rpython/test/test_extfunc.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rpython/test/test_extfunc.py (original) +++ pypy/branch/fast-ctypes/pypy/rpython/test/test_extfunc.py Thu Jul 29 15:38:08 2010 @@ -6,150 +6,164 @@ from pypy.annotation.policy import AnnotatorPolicy from pypy.rpython.test.test_llinterp import interpret -def b(x): - return eval("x+40") +class TestExtFuncEntry: -class BTestFuncEntry(ExtFuncEntry): - _about_ = b - name = 'b' - signature_args = [annmodel.SomeInteger()] - signature_result = annmodel.SomeInteger() - -def test_annotation_b(): - def f(): - return b(1) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def test_rtyping_b(): - def f(): - return b(2) - - res = interpret(f, []) - assert res == 42 - -def c(y, x): - yyy - -class CTestFuncEntry(ExtFuncEntry): - _about_ = c - name = 'ccc' - signature_args = [annmodel.SomeInteger()] * 2 - signature_result = annmodel.SomeInteger() - - def lltypeimpl(y, x): - return y + x - lltypeimpl = staticmethod(lltypeimpl) - -def test_interp_c(): - def f(): - return c(3, 4) - - res = interpret(f, []) - assert res == 7 - -def d(y): - return eval("y()") - -class DTestFuncEntry(ExtFuncEntry): - _about_ = d - name = 'd' - signature_args = [annmodel.SomeGenericCallable(args=[], result= - annmodel.SomeFloat())] - signature_result = annmodel.SomeFloat() - -def test_callback(): - def callback(): - return 2.5 - - def f(): - return d(callback) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeFloat) - assert a.translator._graphof(callback) - -def dd(): - pass - -register_external(dd, [int], int) - -def test_register_external_signature(): - def f(): - return dd(3) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - - -def function_with_tuple_arg(): - """ - Dummy function which is declared via register_external to take a tuple as - an argument so that register_external's behavior for tuple-taking functions - can be verified. - """ -register_external(function_with_tuple_arg, [(int,)], int) - -def test_register_external_tuple_args(): - """ - Verify the annotation of a registered external function which takes a tuple - argument. - """ - def f(): - return function_with_tuple_arg((1,)) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - - # Not a very good assertion, but at least it means _something_ happened. - assert isinstance(s, annmodel.SomeInteger) - -def function_with_list(): - pass -register_external(function_with_list, [[int]], int) - -def function_returning_list(): - pass -register_external(function_returning_list, [], [int]) - -def test_register_external_return_goes_back(): - """ - Check whether it works to pass the same list from one external - fun to another - [bookkeeper and list joining issues] - """ - def f(): - return function_with_list(function_returning_list()) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def function_withspecialcase(arg): - return repr(arg) -register_external(function_withspecialcase, args=None, result=str) - -def test_register_external_specialcase(): - def f(): - x = function_withspecialcase - return x(33) + x("aaa") + x([]) + "\n" - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeString) + def test_basic(self): + """ + A ExtFuncEntry provides an annotation for a function, no need to flow + its graph. + """ + def b(x): + "NOT_RPYTHON" + return eval("x+40") + + class BTestFuncEntry(ExtFuncEntry): + _about_ = b + name = 'b' + signature_args = [annmodel.SomeInteger()] + signature_result = annmodel.SomeInteger() + + def f(): + return b(2) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + res = interpret(f, []) + assert res == 42 + + def test_lltypeimpl(self): + """ + interpret() calls lltypeimpl instead of of the function/ + """ + def c(y, x): + yyy + + class CTestFuncEntry(ExtFuncEntry): + _about_ = c + name = 'ccc' + signature_args = [annmodel.SomeInteger()] * 2 + signature_result = annmodel.SomeInteger() + + def lltypeimpl(y, x): + return y + x + lltypeimpl = staticmethod(lltypeimpl) + + def f(): + return c(3, 4) + + res = interpret(f, []) + assert res == 7 + + def test_callback(self): + """ + Verify annotation when a callback function is in the arguments list. + """ + def d(y): + return eval("y()") + + class DTestFuncEntry(ExtFuncEntry): + _about_ = d + name = 'd' + signature_args = [annmodel.SomeGenericCallable(args=[], result= + annmodel.SomeFloat())] + signature_result = annmodel.SomeFloat() + + def callback(): + return 2.5 + + def f(): + return d(callback) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeFloat) + assert a.translator._graphof(callback) + + def test_register_external_signature(self): + """ + Test the standard interface for external functions. + """ + def dd(): + pass + register_external(dd, [int], int) + + def f(): + return dd(3) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_tuple_args(self): + """ + Verify the annotation of a registered external function which takes a + tuple argument. + """ + + def function_with_tuple_arg(): + """ + Dummy function which is declared via register_external to take a + tuple as an argument so that register_external's behavior for + tuple-taking functions can be verified. + """ + register_external(function_with_tuple_arg, [(int,)], int) + + def f(): + return function_with_tuple_arg((1,)) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + + # Not a very good assertion, but at least it means _something_ happened. + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_return_goes_back(self): + """ + Check whether it works to pass the same list from one external + fun to another + [bookkeeper and list joining issues] + """ + def function_with_list(): + pass + register_external(function_with_list, [[int]], int) + + def function_returning_list(): + pass + register_external(function_returning_list, [], [int]) + + def f(): + return function_with_list(function_returning_list()) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_specialcase(self): + """ + When args=None, the external function accepts any arguments unmodified. + """ + def function_withspecialcase(arg): + return repr(arg) + register_external(function_withspecialcase, args=None, result=str) + + def f(): + x = function_withspecialcase + return x(33) + x("aaa") + x([]) + "\n" + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeString) Modified: pypy/branch/fast-ctypes/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/translator/goal/app_main.py (original) +++ pypy/branch/fast-ctypes/pypy/translator/goal/app_main.py Thu Jul 29 15:38:08 2010 @@ -223,7 +223,6 @@ path = os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath - newpath.insert(0, '') # remove duplicates _seen = {} del sys.path[:] @@ -327,6 +326,10 @@ except: print >> sys.stderr, "'import site' failed" + # update sys.path *after* loading site.py, in case there is a + # "site.py" file in the script's directory. + sys.path.insert(0, '') + if warnoptions: sys.warnoptions.append(warnoptions) from warnings import _processoptions Modified: pypy/branch/fast-ctypes/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/fast-ctypes/pypy/translator/goal/test2/test_app_main.py Thu Jul 29 15:38:08 2010 @@ -5,6 +5,7 @@ import sys, os, re import autopath from pypy.tool.udir import udir +from contextlib import contextmanager banner = sys.version.splitlines()[0] @@ -326,8 +327,9 @@ class TestNonInteractive: def run(self, cmdline, senddata='', expect_prompt=False, - expect_banner=False): - cmdline = '%s "%s" %s' % (sys.executable, app_main, cmdline) + expect_banner=False, python_flags=''): + cmdline = '%s %s "%s" %s' % (sys.executable, python_flags, + app_main, cmdline) print 'POPEN:', cmdline child_in, child_out_err = os.popen4(cmdline) child_in.write(senddata) @@ -449,6 +451,43 @@ assert data == '\x00(STDOUT)\n\x00' # from stdout child_out_err.close() + def test_proper_sys_path(self, tmpdir): + + @contextmanager + def chdir_and_unset_pythonpath(new_cwd): + old_cwd = new_cwd.chdir() + old_pythonpath = os.getenv('PYTHONPATH') + os.unsetenv('PYTHONPATH') + try: + yield + finally: + old_cwd.chdir() + os.putenv('PYTHONPATH', old_pythonpath) + + tmpdir.join('site.py').write('print "SHOULD NOT RUN"') + runme_py = tmpdir.join('runme.py') + runme_py.write('print "some text"') + + cmdline = str(runme_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline, python_flags='-S') + + assert data == "some text\n" + + runme2_py = tmpdir.mkdir('otherpath').join('runme2.py') + runme2_py.write('print "some new text"\n' + 'import sys\n' + 'print sys.path\n') + + cmdline2 = str(runme2_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline2, python_flags='-S') + + assert data.startswith("some new text\n") + assert repr(str(tmpdir.join('otherpath'))) in data + class AppTestAppMain: From fijal at codespeak.net Thu Jul 29 15:44:57 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Jul 2010 15:44:57 +0200 (CEST) Subject: [pypy-svn] r76391 - pypy/extradoc/planning Message-ID: <20100729134457.02EBB282C1B@codespeak.net> Author: fijal Date: Thu Jul 29 15:44:55 2010 New Revision: 76391 Modified: pypy/extradoc/planning/jit.txt Log: Add another task, but someone should overlook that file, it's already too messy :( Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Thu Jul 29 15:44:55 2010 @@ -95,8 +95,11 @@ Python interpreter: +- CALL_METHOD optimization only works for calls with no keyword args. This + is silly and slows down things quite a bit. + - goal: on average <=5 guards per original bytecode. - Almost achieved :-) pypy/jit/tool/otherviewer.py can view where we're + Almost achieved :-) pypy/jit/tool/traceviewer.py can view where we're failing (red boxes out of jit-log-opt logging) - put the class into the structure to get only one promote when using an From fijal at codespeak.net Thu Jul 29 21:31:13 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Jul 2010 21:31:13 +0200 (CEST) Subject: [pypy-svn] r76392 - in pypy/benchmarks: . own/twisted Message-ID: <20100729193113.4A637282C0E@codespeak.net> Author: fijal Date: Thu Jul 29 21:31:04 2010 New Revision: 76392 Modified: pypy/benchmarks/benchmarks.py pypy/benchmarks/own/twisted/benchlib.py Log: Try to come back essentially to the same number as we had before my changes (but shift from duration -> number of iterations, so interface changing can kick in) Modified: pypy/benchmarks/benchmarks.py ============================================================================== --- pypy/benchmarks/benchmarks.py (original) +++ pypy/benchmarks/benchmarks.py Thu Jul 29 21:31:04 2010 @@ -48,7 +48,7 @@ _register_new_bm(name, name, globals(), **opts.get(name, {})) for name in ['web', 'names', 'iteration', 'tcp', 'pb']:#, 'accepts']: if name == 'web': - iteration_scaling = 0.1 + iteration_scaling = 0.35 else: iteration_scaling = 1.0 _register_new_bm_twisted(name, 'twisted_' + name, Modified: pypy/benchmarks/own/twisted/benchlib.py ============================================================================== --- pypy/benchmarks/own/twisted/benchlib.py (original) +++ pypy/benchmarks/own/twisted/benchlib.py Thu Jul 29 21:31:04 2010 @@ -54,7 +54,7 @@ optParameters = [ ('iterations', 'n', 1, 'number of iterations', int), ('duration', 'd', 1, 'duration of each iteration', float), - ('warmup', 'w', 3, 'number of warmup iterations', int), + ('warmup', 'w', 15, 'number of warmup iterations', int), ] options = BenchmarkOptions() From hakanardo at codespeak.net Thu Jul 29 21:45:44 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 29 Jul 2010 21:45:44 +0200 (CEST) Subject: [pypy-svn] r76393 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100729194544.502B7282BAD@codespeak.net> Author: hakanardo Date: Thu Jul 29 21:45:42 2010 New Revision: 76393 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: list methods Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Thu Jul 29 21:45:42 2010 @@ -46,15 +46,25 @@ array_append = SMM('append', 2) array_extend = SMM('extend', 2) +array_count = SMM('count', 2) +array_index = SMM('index', 2) +array_reverse = SMM('reverse', 1) +array_remove = SMM('remove', 2) +array_pop = SMM('pop', 2, defaults=(-1,)) +array_insert = SMM('insert', 3) + array_tolist = SMM('tolist', 1) array_fromlist = SMM('fromlist', 2) array_tostring = SMM('tostring', 1) array_fromstring = SMM('fromstring', 2) array_tounicode = SMM('tounicode', 1) array_fromunicode = SMM('fromunicode', 2) -array_tofile = SMM('tofile', 1) +array_tofile = SMM('tofile', 2) array_fromfile = SMM('fromfile', 3) +array_buffer_info = SMM('buffer_info', 1) + + def descr_itemsize(space, self): return space.wrap(self.itemsize) @@ -246,7 +256,7 @@ return rffi.cast(rffi.CCHARP, self.buffer) - # List interface + # Basic get/set/append/extend methods def len__Array(space, self): return space.wrap(self.len) @@ -331,6 +341,65 @@ self.fromsequence(w_iterable) + # List interface + def array_count__Array_ANY(space, self, w_val): + val = self.item_w(w_val) + cnt = 0 + for i in range(self.len): + if self.buffer[i] == val: + cnt += 1 + return space.wrap(cnt) + + + def array_index__Array_ANY(space, self, w_val): + val = self.item_w(w_val) + cnt = 0 + for i in range(self.len): + if self.buffer[i] == val: + return space.wrap(i) + msg = 'array.index(x): x not in list' + raise OperationError(space.w_ValueError, space.wrap(msg)) + + + def array_reverse__Array(space, self): + b = self.buffer + for i in range(self.len/2): + b[i], b[self.len - i -1] = b[self.len - i -1], b[i] + + + def array_pop__Array_ANY(space, self, w_idx): + i = space.int_w(w_idx) + if i < 0: i += self.len + if i < 0 or i >= self.len: + msg = 'pop index out of range' + raise OperationError(space.w_IndexError, space.wrap(msg)) + w_val = getitem__Array_ANY(space, self, space.wrap(i)) + while i < self.len - 1: + self.buffer[i] = self.buffer[i + 1] + i += 1 + self.setlen(self.len-1) + return w_val + + + def array_remove__Array_ANY(space, self, w_val): + w_idx = array_index__Array_ANY(space, self, w_val) + array_pop__Array_ANY(space, self, w_idx) + + + def array_insert__Array_ANY_ANY(space, self, w_idx, w_val): + idx = space.int_w(w_idx) + if idx < 0: idx += self.len + if idx < 0: idx = 0 + if idx > self.len: idx = self.len + + val = self.item_w(w_val) + self.setlen(self.len + 1) + i = self.len-1 + while i > idx: + self.buffer[i] = self.buffer[i-1] + i -= 1 + self.buffer[i] = val + # Convertions def array_tolist__Array(space, self): @@ -388,6 +457,13 @@ raise OperationError(space.w_EOFError, space.wrap(msg)) array_fromstring__Array_ANY(space, self, w_item) + def array_tofile__Array_ANY(space, self, w_f): + if space.type(w_f).name != 'file': # FIXME: this cant be the right way? + msg = "arg1 must be open file" + raise OperationError(space.w_TypeError, space.wrap(msg)) + w_s = array_tostring__Array(space, self) + space.call_method(w_f, 'write', w_s) + if mytype.typecode == 'u': def array_fromunicode__Array_Unicode(space, self, w_ustr): # XXX the following probable bug is not emulated: @@ -396,13 +472,20 @@ # string arguments at multiples of the unicode byte size. # Let's only accept unicode arguments for now. self.fromsequence(w_ustr) + + def array_tounicode__Array(space, self): + u = u"" + for i in range(self.len): + u += self.buffer[i] + return space.wrap(u) else: def array_fromunicode__Array_Unicode(space, self, w_ustr): msg = "fromunicode() may only be called on type 'u' arrays" raise OperationError(space.w_ValueError, space.wrap(msg)) - - + def array_tounicode__Array(space, self): + msg = "tounicode() may only be called on type 'u' arrays" + raise OperationError(space.w_ValueError, space.wrap(msg)) def cmp__Array_ANY(space, self, other): @@ -414,6 +497,18 @@ raise OperationError(space.w_NotImplementedError, space.wrap('')) + def buffer__Array(space, self): + from pypy.interpreter.buffer import StringLikeBuffer + w_s = array_tostring__Array(space, self) + return space.wrap(StringLikeBuffer(space, w_s)) + + def array_buffer_info__Array(space, self): + w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer)) + w_len = space.wrap(self.len) + return space.newtuple([w_ptr, w_len]) + + + def repr__Array(space, self): if self.len == 0: return space.wrap("array('%s')" % self.typecode) Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Thu Jul 29 21:45:42 2010 @@ -441,10 +441,15 @@ assert a.count(2) == 2 assert a.index(3) == 2 assert a.index(2) == 1 + raises(ValueError, a.index, 10) a.reverse() assert repr(a) == "array('i', [1, 2, 1, 3, 2, 1])" + b=self.array('i', [1, 2, 3, 1, 2]) + b.reverse() + assert repr(b) == "array('i', [2, 1, 3, 2, 1])" + a.remove(3) assert repr(a) == "array('i', [1, 2, 1, 2, 1])" a.remove(1) @@ -460,10 +465,16 @@ assert repr(a) == "array('i', [2])" a.insert(1,7) + assert repr(a) == "array('i', [2, 7])" a.insert(0,8) a.insert(-1,9) assert repr(a) == "array('i', [8, 2, 9, 7])" + a.insert(100,10) + assert repr(a) == "array('i', [8, 2, 9, 7, 10])" + a.insert(-100,20) + assert repr(a) == "array('i', [20, 8, 2, 9, 7, 10])" + def test_compare(self): for v1,v2,tt in (([1, 2, 3], [1, 3, 2], 'bhilBHIL'), ('abc', 'acb', 'c'), From hakanardo at codespeak.net Thu Jul 29 22:20:21 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 29 Jul 2010 22:20:21 +0200 (CEST) Subject: [pypy-svn] r76396 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100729202021.5E481282BAD@codespeak.net> Author: hakanardo Date: Thu Jul 29 22:20:18 2010 New Revision: 76396 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: add/mul Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Thu Jul 29 22:20:18 2010 @@ -21,15 +21,16 @@ for tc in unroll_typecodes: if typecode == tc: a = types[tc].w_class(space) - if not space.is_w(w_initializer, space.w_None): - if space.type(w_initializer) is space.w_str: - space.call_method(a, 'fromstring', w_initializer) - elif space.type(w_initializer) is space.w_unicode: - space.call_method(a, 'fromunicode', w_initializer) - elif space.type(w_initializer) is space.w_list: - space.call_method(a, 'fromlist', w_initializer) - else: - space.call_method(a, 'extend', w_initializer) + if w_initializer is not None: + if not space.is_w(w_initializer, space.w_None): + if space.type(w_initializer) is space.w_str: + space.call_method(a, 'fromstring', w_initializer) + elif space.type(w_initializer) is space.w_unicode: + space.call_method(a, 'fromunicode', w_initializer) + elif space.type(w_initializer) is space.w_list: + space.call_method(a, 'fromlist', w_initializer) + else: + space.call_method(a, 'extend', w_initializer) break else: msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' @@ -63,7 +64,8 @@ array_fromfile = SMM('fromfile', 3) array_buffer_info = SMM('buffer_info', 1) - +array_reduce = SMM('__reduce__', 1) +array_byteswap = SMM('byteswap', 1) def descr_itemsize(space, self): @@ -75,6 +77,7 @@ type_typedef = StdTypeDef( 'array', __new__ = interp2app(w_array_sub), + __module__ = 'array', itemsize = GetSetProperty(descr_itemsize), typecode = GetSetProperty(descr_typecode), ) @@ -400,6 +403,47 @@ i -= 1 self.buffer[i] = val + + # Add and mul methods + + def add__Array_Array(space, self, other): + a = mytype.w_class(space) + a.setlen(self.len + other.len) + for i in range(self.len): + a.buffer[i] = self.buffer[i] + for i in range(other.len): + a.buffer[i + self.len] = other.buffer[i] + return a + + def inplace_add__Array_Array(space, self, other): + oldlen = self.len + otherlen = other.len + self.setlen(oldlen + otherlen) + for i in range(otherlen): + self.buffer[oldlen + i] = other.buffer[i] + return self + + def mul__Array_ANY(space, self, w_repeat): + repeat = space.int_w(w_repeat) + a = mytype.w_class(space) + a.setlen(self.len * repeat) + for r in range(repeat): + for i in range(self.len): + a.buffer[r * self.len + i] = self.buffer[i] + return a + + def mul__ANY_Array(space, w_repeat, self): + return mul__Array_ANY(space, self, w_repeat) + + def inplace_mul__Array_ANY(space, self, w_repeat): + repeat = space.int_w(w_repeat) + oldlen=self.len + self.setlen(self.len * repeat) + for r in range(1,repeat): + for i in range(oldlen): + self.buffer[r * oldlen + i] = self.buffer[i] + return self + # Convertions def array_tolist__Array(space, self): @@ -488,6 +532,7 @@ raise OperationError(space.w_ValueError, space.wrap(msg)) + # Compare methods def cmp__Array_ANY(space, self, other): if isinstance(other, W_ArrayBase): w_lst1 = array_tolist__Array(space, self) @@ -497,6 +542,8 @@ raise OperationError(space.w_NotImplementedError, space.wrap('')) + # Misc methods + def buffer__Array(space, self): from pypy.interpreter.buffer import StringLikeBuffer w_s = array_tostring__Array(space, self) @@ -506,8 +553,30 @@ w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer)) w_len = space.wrap(self.len) return space.newtuple([w_ptr, w_len]) + + def array_reduce__Array(space, self): + if self.len > 0: + w_s = array_tostring__Array(space, self) + args = [space.wrap(mytype.typecode), w_s] + else: + args = [space.wrap(mytype.typecode)] + return space.newtuple([space.type(self), space.newtuple(args)]) + + + def array_byteswap__Array(space, self): + if mytype.bytes not in [1, 2, 4, 8]: + msg="byteswap not supported for this array" + raise OperationError(space.w_RuntimeError, space.wrap(msg)) + if self.len == 0: return + bytes = self.charbuf() + tmp = [bytes[0]] * mytype.bytes + for start in range(0, self.len * mytype.bytes, mytype.bytes): + stop = start + mytype.bytes - 1 + for i in range(mytype.bytes): + tmp[i] = bytes[start + i] + for i in range(mytype.bytes): + bytes[stop - i] = tmp[i] - def repr__Array(space, self): if self.len == 0: Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Thu Jul 29 22:20:18 2010 @@ -582,7 +582,6 @@ b *= 3 assert repr(a) == "array('i', [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2])" assert a == b - print "\nok: ", type(a) a += self.array('i', (7,)) assert repr(a) == "array('i', [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 7])" From getxsick at codespeak.net Thu Jul 29 23:55:23 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 29 Jul 2010 23:55:23 +0200 (CEST) Subject: [pypy-svn] r76397 - in pypy/branch/fast-ctypes/pypy/rlib: . test Message-ID: <20100729215523.64B0D282BAD@codespeak.net> Author: getxsick Date: Thu Jul 29 23:55:21 2010 New Revision: 76397 Modified: pypy/branch/fast-ctypes/pypy/rlib/jit.py pypy/branch/fast-ctypes/pypy/rlib/test/test_jit.py Log: new get_cpu() function to return CPU instance Modified: pypy/branch/fast-ctypes/pypy/rlib/jit.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/jit.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/jit.py Thu Jul 29 23:55:21 2010 @@ -4,6 +4,7 @@ from pypy.rlib.objectmodel import CDefinedIntSymbolic from pypy.rlib.objectmodel import keepalive_until_here from pypy.rlib.unroll import unrolling_iterable +from pypy.jit.backend.detect_cpu import getcpuclass def purefunction(func): """ Decorate a function as pure. Pure means precisely that: @@ -77,6 +78,10 @@ return result return decorator +def get_cpu(): + CPUClass = getcpuclass() + return CPUClass(None, None) + class Entry(ExtRegistryEntry): _about_ = hint Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_jit.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_jit.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_jit.py Thu Jul 29 23:55:21 2010 @@ -1,9 +1,10 @@ import py from pypy.rlib.jit import hint, we_are_jitted, JitDriver, purefunction_promote -from pypy.rlib.jit import JitHintError +from pypy.rlib.jit import JitHintError, get_cpu from pypy.translator.translator import TranslationContext, graphof from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.rpython.lltypesystem import lltype +from pypy.jit.backend.detect_cpu import ProcessorAutodetectError class BaseTestJIT(BaseRtypingTest): def test_hint(self): @@ -107,6 +108,14 @@ return n py.test.raises(JitHintError, self.gengraph, fn, [int]) + def test_get_cpu(self): + try: + cpu = get_cpu() + except ProcessorAutodetectError: + pass + else: + from pypy.jit.backend.model import AbstractCPU + assert isinstance(cpu, AbstractCPU) class TestJITLLtype(BaseTestJIT, LLRtypeMixin): pass From fijal at codespeak.net Fri Jul 30 10:44:19 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Jul 2010 10:44:19 +0200 (CEST) Subject: [pypy-svn] r76398 - pypy/benchmarks Message-ID: <20100730084419.B6651282C0E@codespeak.net> Author: fijal Date: Fri Jul 30 10:44:17 2010 New Revision: 76398 Modified: pypy/benchmarks/benchmarks.py Log: Give up and disable web benchmark :( Modified: pypy/benchmarks/benchmarks.py ============================================================================== --- pypy/benchmarks/benchmarks.py (original) +++ pypy/benchmarks/benchmarks.py Fri Jul 30 10:44:17 2010 @@ -46,7 +46,7 @@ for name in ['float', 'nbody_modified', 'meteor-contest', 'fannkuch', 'spectral-norm', 'chaos', 'telco']: _register_new_bm(name, name, globals(), **opts.get(name, {})) -for name in ['web', 'names', 'iteration', 'tcp', 'pb']:#, 'accepts']: +for name in ['names', 'iteration', 'tcp', 'pb']:#, 'accepts', 'web']: if name == 'web': iteration_scaling = 0.35 else: From hakanardo at codespeak.net Fri Jul 30 15:24:09 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Fri, 30 Jul 2010 15:24:09 +0200 (CEST) Subject: [pypy-svn] r76402 - in pypy/branch/interplevel-array/pypy: module/array module/array/test objspace/std Message-ID: <20100730132409.08E1E282C0E@codespeak.net> Author: hakanardo Date: Fri Jul 30 15:24:07 2010 New Revision: 76402 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py pypy/branch/interplevel-array/pypy/objspace/std/itertype.py Log: multimethod implementation inplace Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Fri Jul 30 15:24:07 2010 @@ -11,8 +11,12 @@ from pypy.objspace.std.stdtypedef import SMM, StdTypeDef from pypy.objspace.std.register_all import register_all from pypy.objspace.std.model import W_Object +from pypy.interpreter.argument import Arguments, Signature -def w_array(space, typecode, w_initializer=None): +def w_array(space, w_cls, typecode, w_initializer=None, w_args=None): + if len(w_args.arguments_w) > 0: + msg = 'array() takes at most 2 arguments' + raise OperationError(space.w_TypeError, space.wrap(msg)) if len(typecode) != 1: msg = 'array() argument 1 must be char, not str' raise OperationError(space.w_TypeError, space.wrap(msg)) @@ -20,7 +24,9 @@ for tc in unroll_typecodes: if typecode == tc: - a = types[tc].w_class(space) + a = space.allocate_instance(types[tc].w_class, w_cls) + a.__init__(space) + if w_initializer is not None: if not space.is_w(w_initializer, space.w_None): if space.type(w_initializer) is space.w_str: @@ -37,11 +43,7 @@ raise OperationError(space.w_ValueError, space.wrap(msg)) return a -w_array.unwrap_spec = (ObjSpace, str, W_Root) - -def w_array_sub(space, w_cls, typecode, w_initializer=None): - return w_array(space, typecode, w_initializer) -w_array_sub.unwrap_spec = (ObjSpace, W_Root, str, W_Root) +w_array.unwrap_spec = (ObjSpace, W_Root, str, W_Root, Arguments) array_append = SMM('append', 2) @@ -76,7 +78,7 @@ type_typedef = StdTypeDef( 'array', - __new__ = interp2app(w_array_sub), + __new__ = interp2app(w_array), __module__ = 'array', itemsize = GetSetProperty(descr_itemsize), typecode = GetSetProperty(descr_typecode), @@ -133,22 +135,6 @@ unroll_typecodes = unrolling_iterable(types.keys()) def make_array(mytype): - class W_ArrayIter(Wrappable): - def __init__(self, a): - self.space = a.space - self.a = a - self.pos = 0 - - def iter_w(self): - return self.space.wrap(self) - - def next_w(self): - if self.pos >= self.a.len: - raise OperationError(self.space.w_StopIteration, self.space.w_None) - val = self.a.descr_getitem(self.space.wrap(self.pos)) - self.pos += 1 - return val - class W_Array(W_ArrayBase): itemsize = mytype.bytes typecode = mytype.typecode @@ -404,6 +390,15 @@ self.buffer[i] = val + def delitem__Array_ANY(space, self, w_idx): + w_lst = array_tolist__Array(space, self) + space.delitem(w_lst, w_idx) + self.setlen(0) + self.fromsequence(w_lst) + + def delslice__Array_ANY_ANY(space, self, w_i, w_j): + return space.delitem(self, space.newslice(w_i, w_j, space.w_None)) + # Add and mul methods def add__Array_Array(space, self, other): @@ -594,6 +589,11 @@ s = "array('%s', %s)" % (self.typecode, space.str_w(r)) return space.wrap(s) + init_signature = Signature(['typecode', 'initializer']) + init_defaults = [None, None] + + def init__Array(space, self, args): + args.parse_obj(None, 'array', init_signature, init_defaults) W_Array.__name__ = 'W_ArrayType_'+mytype.typecode mytype.w_class = W_Array Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Fri Jul 30 15:24:07 2010 @@ -539,7 +539,9 @@ assert len(b) == 0 and b.typecode == 'l' a = self.array('i', [1, 2, 4]) + print "itter" i = iter(a) + print "ok" raises(TypeError, pickle.dumps, i, 1) def test_copy_swap(self): @@ -585,6 +587,28 @@ a += self.array('i', (7,)) assert repr(a) == "array('i', [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 7])" + try: + raised = False + a = self.array('i') + 2 + except TypeError: + raised = True + assert raised + + try: + raised = False + a = self.array('i') + self.array('b') + except TypeError: + raised = True + assert raised + + try: + raised = False + a = self.array('i').__add__(2) + print a + except TypeError: + raised = True + assert raised + raises(TypeError, self.array('i').__add__, (2,)) raises(TypeError, self.array('i').__add__, self.array('b')) @@ -637,6 +661,7 @@ assert isinstance(self.array(t), self.array) def test_subclass(self): + print type(self.array('b')) assert len(self.array('b')) == 0 a=self.array('i') @@ -646,9 +671,11 @@ array=self.array class adder(array): def __getitem__(self, i): + print 25 return array.__getitem__(self, i) + 1 a=adder('i', (1,2,3)) + print type(a) assert len(a) == 3 assert a[0] == 2 Modified: pypy/branch/interplevel-array/pypy/objspace/std/itertype.py ============================================================================== --- pypy/branch/interplevel-array/pypy/objspace/std/itertype.py (original) +++ pypy/branch/interplevel-array/pypy/objspace/std/itertype.py Fri Jul 30 15:24:07 2010 @@ -1,5 +1,6 @@ from pypy.interpreter import gateway from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.interpreter.error import OperationError # ____________________________________________________________ @@ -8,6 +9,10 @@ XXX to do: remove this __reduce__ method and do a registration with copy_reg, instead. """ + + # cpython does not support pickling iterators + raise OperationError(space.w_TypeError, space.w_None) + from pypy.objspace.std.iterobject import W_AbstractSeqIterObject assert isinstance(w_self, W_AbstractSeqIterObject) from pypy.interpreter.mixedmodule import MixedModule From fijal at codespeak.net Fri Jul 30 19:07:01 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Jul 2010 19:07:01 +0200 (CEST) Subject: [pypy-svn] r76405 - in pypy/trunk/pypy/rlib: . test Message-ID: <20100730170701.12700282C0E@codespeak.net> Author: fijal Date: Fri Jul 30 19:06:59 2010 New Revision: 76405 Modified: pypy/trunk/pypy/rlib/objectmodel.py pypy/trunk/pypy/rlib/test/test_objectmodel.py Log: An enforceargs decorator Modified: pypy/trunk/pypy/rlib/objectmodel.py ============================================================================== --- pypy/trunk/pypy/rlib/objectmodel.py (original) +++ pypy/trunk/pypy/rlib/objectmodel.py Fri Jul 30 19:06:59 2010 @@ -82,6 +82,17 @@ specialize = _Specialize() +def enforceargs(*args): + """ Decorate a function with forcing of RPython-level types on arguments. + None means no enforcing. + + XXX shouldn't we also add asserts in function body? + """ + def decorator(f): + f._annenforceargs_ = args + return f + return decorator + # ____________________________________________________________ class Symbolic(object): Modified: pypy/trunk/pypy/rlib/test/test_objectmodel.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_objectmodel.py (original) +++ pypy/trunk/pypy/rlib/test/test_objectmodel.py Fri Jul 30 19:06:59 2010 @@ -404,6 +404,13 @@ assert f._annspecialcase_ == 'specialize:arg(1)' +def test_enforceargs_decorator(): + @enforceargs(int, str, None) + def f(a, b, c): + pass + + assert f._annenforceargs_ == (int, str, None) + def getgraph(f, argtypes): from pypy.translator.translator import TranslationContext, graphof from pypy.translator.backendopt.all import backend_optimizations From fijal at codespeak.net Fri Jul 30 19:33:38 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Jul 2010 19:33:38 +0200 (CEST) Subject: [pypy-svn] r76406 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20100730173338.239F8282C0E@codespeak.net> Author: fijal Date: Fri Jul 30 19:33:36 2010 New Revision: 76406 Modified: pypy/trunk/pypy/rpython/lltypesystem/rstr.py Log: Use cool new decorator to fix tests Modified: pypy/trunk/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rstr.py Fri Jul 30 19:33:36 2010 @@ -2,7 +2,7 @@ from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rlib.objectmodel import malloc_zero_filled, we_are_translated -from pypy.rlib.objectmodel import _hash_string +from pypy.rlib.objectmodel import _hash_string, enforceargs from pypy.rlib.debug import ll_assert from pypy.rlib.jit import purefunction from pypy.rpython.robject import PyObjRepr, pyobj_repr @@ -56,6 +56,7 @@ llmemory.itemoffsetof(TP.chars, 0) + llmemory.sizeof(CHAR_TP) * item) + @enforceargs(None, None, int, int, int) def copy_string_contents(src, dst, srcstart, dststart, length): assert srcstart >= 0 assert dststart >= 0 From agaynor at codespeak.net Fri Jul 30 20:12:44 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Fri, 30 Jul 2010 20:12:44 +0200 (CEST) Subject: [pypy-svn] r76407 - pypy/branch/call-method-kwarg Message-ID: <20100730181244.B7E21282C0E@codespeak.net> Author: agaynor Date: Fri Jul 30 20:12:41 2010 New Revision: 76407 Added: pypy/branch/call-method-kwarg/ (props changed) - copied from r76406, pypy/trunk/ Log: A branch to add support for keyword arguments to the CALL_METHOD opcode From agaynor at codespeak.net Fri Jul 30 20:17:58 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Fri, 30 Jul 2010 20:17:58 +0200 (CEST) Subject: [pypy-svn] r76408 - in pypy/branch/call-method-kwarg/pypy: interpreter/astcompiler interpreter/test objspace/std objspace/std/test Message-ID: <20100730181758.4D987282C0E@codespeak.net> Author: agaynor Date: Fri Jul 30 20:17:55 2010 New Revision: 76408 Modified: pypy/branch/call-method-kwarg/pypy/interpreter/astcompiler/codegen.py pypy/branch/call-method-kwarg/pypy/interpreter/test/test_compiler.py pypy/branch/call-method-kwarg/pypy/objspace/std/callmethod.py pypy/branch/call-method-kwarg/pypy/objspace/std/test/test_callmethod.py Log: Started implementing kwarg support for CALL_METHOD, the compiler test needs work. Modified: pypy/branch/call-method-kwarg/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/call-method-kwarg/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/call-method-kwarg/pypy/interpreter/astcompiler/codegen.py Fri Jul 30 20:17:55 2010 @@ -960,9 +960,12 @@ elif call_type == 3: op = ops.CALL_FUNCTION_VAR_KW self.emit_op_arg(op, arg) + + def _call_has_no_star_args(self, call): + return not call.starargs and not call.kwargs def _call_has_simple_args(self, call): - return not call.starargs and not call.kwargs and not call.keywords + return self._call_has_no_star_args(call) and not call.keywords def _optimize_builtin_call(self, call): if not self.space.config.objspace.opcodes.CALL_LIKELY_BUILTIN or \ @@ -988,7 +991,7 @@ def _optimize_method_call(self, call): if not self.space.config.objspace.opcodes.CALL_METHOD or \ - not self._call_has_simple_args(call) or \ + not self._call_has_no_star_args(call) or \ not isinstance(call.func, ast.Attribute): return False attr_lookup = call.func @@ -1000,7 +1003,12 @@ arg_count = len(call.args) else: arg_count = 0 - self.emit_op_arg(ops.CALL_METHOD, arg_count) + if call.kwargs: + call.kwargs.walkabout(self) + kwarg_count = len(call.kwargs) + else: + kwarg_count = 0 + self.emit_op_arg(ops.CALL_METHOD, (kwarg_count << 8) + arg_count) return True def _listcomp_generator(self, list_name, gens, gen_index, elt): Modified: pypy/branch/call-method-kwarg/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/call-method-kwarg/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/call-method-kwarg/pypy/interpreter/test/test_compiler.py Fri Jul 30 20:17:55 2010 @@ -848,13 +848,38 @@ import StringIO, sys, dis s = StringIO.StringIO() + out = sys.stdout sys.stdout = s try: dis.dis(code) finally: - sys.stdout = sys.__stdout__ + sys.stdout = out output = s.getvalue() assert "LOAD_GLOBAL" not in output + + def test_call_method_kwargs(self): + source = """def _f(a): + return a.f(a=a) + """ + CALL_METHOD = self.space.config.objspace.opcodes.CALL_METHOD + self.space.config.objspace.opcodes.CALL_METHOD = True + try: + exec source + finally: + self.space.config.objspace.opcodes.CALL_METHOD = CALL_METHOD + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + out = sys.stdout + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = out + output = s.getvalue() + assert "CALL_METHOD" in output + class AppTestExceptions: def test_indentation_error(self): Modified: pypy/branch/call-method-kwarg/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/call-method-kwarg/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/call-method-kwarg/pypy/objspace/std/callmethod.py Fri Jul 30 20:17:55 2010 @@ -56,16 +56,37 @@ f.pushvalue(w_value) f.pushvalue(None) -def CALL_METHOD(f, nargs, *ignored): - # 'nargs' is the argument count excluding the implicit 'self' - w_self = f.peekvalue(nargs) - w_callable = f.peekvalue(nargs + 1) - n = nargs + (w_self is not None) +def CALL_METHOD(f, oparg, *ignored): + # opargs contains the arg, and kwarg count, excluding the implicit 'self' + n_args = oparg & 0xff + n_kwargs = (oparg >> 8) & 0xff + w_self = f.peekvalue(n_args + (2 * n_kwargs)) + w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) + + if n_kwargs: + keywords = [None] * n_kwargs + keywords_w = [None] * n_kwargs + while True: + n_kwargs -= 1 + if n_kwargs < 0: + break + w_value = f.popvalue() + w_key = f.popvalue() + key = f.space.str_w(w_key) + keywords[n_keywords] = key + keywords_w[n_keywords] = w_value + else: + keywords = None + keywords_w = None + + arguments = f.popvalues(n_args + (w_self is not None)) + args = f.argument_factory(arguments, keywords, keywords_w, None, None) + try: - w_result = f.space.call_valuestack(w_callable, n, f) - rstack.resume_point("CALL_METHOD", f, nargs, returns=w_result) + w_result = f.space.call_args(w_callable, args) + rstack.resume_point("CALL_METHOD", f, returns=w_result) finally: - f.dropvalues(nargs + 2) + f.dropvalues(1 + (w_self is None)) f.pushvalue(w_result) Modified: pypy/branch/call-method-kwarg/pypy/objspace/std/test/test_callmethod.py ============================================================================== --- pypy/branch/call-method-kwarg/pypy/objspace/std/test/test_callmethod.py (original) +++ pypy/branch/call-method-kwarg/pypy/objspace/std/test/test_callmethod.py Fri Jul 30 20:17:55 2010 @@ -106,6 +106,15 @@ else: raise Exception("did not raise?") """ + + def test_kwargs(self): + exec """if 1: + class C(object): + def f(self, a): + return a + 2 + + assert C().f(a=3) == 5 + """ class AppTestCallMethodWithGetattributeShortcut(AppTestCallMethod): From fijal at codespeak.net Fri Jul 30 20:37:52 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Jul 2010 20:37:52 +0200 (CEST) Subject: [pypy-svn] r76409 - pypy/branch/call-method-kwarg/pypy/interpreter/test Message-ID: <20100730183752.A49BC282C0E@codespeak.net> Author: fijal Date: Fri Jul 30 20:37:50 2010 New Revision: 76409 Modified: pypy/branch/call-method-kwarg/pypy/interpreter/test/test_compiler.py Log: fix the test Modified: pypy/branch/call-method-kwarg/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/call-method-kwarg/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/call-method-kwarg/pypy/interpreter/test/test_compiler.py Fri Jul 30 20:37:50 2010 @@ -4,6 +4,7 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.error import OperationError from pypy.interpreter.argument import Arguments +from pypy.conftest import gettestobjspace class BaseTestCompiler: def setup_method(self, method): @@ -856,17 +857,16 @@ sys.stdout = out output = s.getvalue() assert "LOAD_GLOBAL" not in output - + +class AppTestCallMethod(object): + def setup_class(cls): + cls.space = gettestobjspace(**{'objspace.opcodes.CALL_METHOD': True}) + def test_call_method_kwargs(self): source = """def _f(a): return a.f(a=a) """ - CALL_METHOD = self.space.config.objspace.opcodes.CALL_METHOD - self.space.config.objspace.opcodes.CALL_METHOD = True - try: - exec source - finally: - self.space.config.objspace.opcodes.CALL_METHOD = CALL_METHOD + exec source code = _f.func_code import StringIO, sys, dis From agaynor at codespeak.net Fri Jul 30 21:16:36 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Fri, 30 Jul 2010 21:16:36 +0200 (CEST) Subject: [pypy-svn] r76410 - in pypy/branch/call-method-kwarg/pypy: interpreter/astcompiler objspace/std Message-ID: <20100730191636.6D771282C0E@codespeak.net> Author: agaynor Date: Fri Jul 30 21:15:47 2010 New Revision: 76410 Modified: pypy/branch/call-method-kwarg/pypy/interpreter/astcompiler/assemble.py pypy/branch/call-method-kwarg/pypy/interpreter/astcompiler/codegen.py pypy/branch/call-method-kwarg/pypy/objspace/std/callmethod.py Log: Fixes to stack analysis and compiler. It works. Modified: pypy/branch/call-method-kwarg/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/call-method-kwarg/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/call-method-kwarg/pypy/interpreter/astcompiler/assemble.py Fri Jul 30 21:15:47 2010 @@ -581,7 +581,7 @@ return -(arg & 0xFF) + 1 def _compute_CALL_METHOD(arg): - return -arg - 1 + return _num_args(arg) - 1 _stack_effect_computers = {} Modified: pypy/branch/call-method-kwarg/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/call-method-kwarg/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/call-method-kwarg/pypy/interpreter/astcompiler/codegen.py Fri Jul 30 21:15:47 2010 @@ -1003,12 +1003,12 @@ arg_count = len(call.args) else: arg_count = 0 - if call.kwargs: - call.kwargs.walkabout(self) - kwarg_count = len(call.kwargs) + if call.keywords: + self.visit_sequence(call.keywords) + kwarg_count = len(call.keywords) else: kwarg_count = 0 - self.emit_op_arg(ops.CALL_METHOD, (kwarg_count << 8) + arg_count) + self.emit_op_arg(ops.CALL_METHOD, (kwarg_count << 8) | arg_count) return True def _listcomp_generator(self, list_name, gens, gen_index, elt): Modified: pypy/branch/call-method-kwarg/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/call-method-kwarg/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/call-method-kwarg/pypy/objspace/std/callmethod.py Fri Jul 30 21:15:47 2010 @@ -73,8 +73,8 @@ w_value = f.popvalue() w_key = f.popvalue() key = f.space.str_w(w_key) - keywords[n_keywords] = key - keywords_w[n_keywords] = w_value + keywords[n_kwargs] = key + keywords_w[n_kwargs] = w_value else: keywords = None keywords_w = None From agaynor at codespeak.net Fri Jul 30 22:01:01 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Fri, 30 Jul 2010 22:01:01 +0200 (CEST) Subject: [pypy-svn] r76411 - pypy/branch/call-method-kwarg/pypy/objspace/std Message-ID: <20100730200101.BFB1A282C0E@codespeak.net> Author: agaynor Date: Fri Jul 30 22:00:58 2010 New Revision: 76411 Modified: pypy/branch/call-method-kwarg/pypy/objspace/std/callmethod.py Log: reinclude a fastpath, this also fixes the test_gateway tests Modified: pypy/branch/call-method-kwarg/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/call-method-kwarg/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/call-method-kwarg/pypy/objspace/std/callmethod.py Fri Jul 30 22:00:58 2010 @@ -62,8 +62,15 @@ n_kwargs = (oparg >> 8) & 0xff w_self = f.peekvalue(n_args + (2 * n_kwargs)) w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) + n = n_args + (w_self is not None) - if n_kwargs: + if not n_kwargs: + try: + w_result = f.space.call_valuestack(w_callable, n, f) + rstack.resume_point("CALL_METHOD", f, n_args, returns=w_result) + finally: + f.dropvalues(n_args + 2) + else: keywords = [None] * n_kwargs keywords_w = [None] * n_kwargs while True: @@ -75,18 +82,15 @@ key = f.space.str_w(w_key) keywords[n_kwargs] = key keywords_w[n_kwargs] = w_value - else: - keywords = None - keywords_w = None - - arguments = f.popvalues(n_args + (w_self is not None)) - args = f.argument_factory(arguments, keywords, keywords_w, None, None) - try: - w_result = f.space.call_args(w_callable, args) - rstack.resume_point("CALL_METHOD", f, returns=w_result) - finally: - f.dropvalues(1 + (w_self is None)) + arguments = f.popvalues(n) + args = f.argument_factory(arguments, keywords, keywords_w, None, None) + + try: + w_result = f.space.call_args(w_callable, args) + rstack.resume_point("CALL_METHOD", f, returns=w_result) + finally: + f.dropvalues(1 + (w_self is None)) f.pushvalue(w_result) From agaynor at codespeak.net Fri Jul 30 22:12:02 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Fri, 30 Jul 2010 22:12:02 +0200 (CEST) Subject: [pypy-svn] r76412 - in pypy/trunk: . pypy/interpreter/astcompiler pypy/interpreter/test pypy/objspace/std pypy/objspace/std/test Message-ID: <20100730201202.9B7E9282C11@codespeak.net> Author: agaynor Date: Fri Jul 30 22:12:00 2010 New Revision: 76412 Modified: pypy/trunk/ (props changed) pypy/trunk/pypy/interpreter/astcompiler/assemble.py pypy/trunk/pypy/interpreter/astcompiler/codegen.py pypy/trunk/pypy/interpreter/test/test_compiler.py pypy/trunk/pypy/objspace/std/callmethod.py pypy/trunk/pypy/objspace/std/test/test_callmethod.py Log: Merged call-method-kwarg into trunk. The optimized CALL_METHOD opcode is now used for calls with keyword arguments. Modified: pypy/trunk/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/assemble.py Fri Jul 30 22:12:00 2010 @@ -581,7 +581,7 @@ return -(arg & 0xFF) + 1 def _compute_CALL_METHOD(arg): - return -arg - 1 + return _num_args(arg) - 1 _stack_effect_computers = {} Modified: pypy/trunk/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/codegen.py Fri Jul 30 22:12:00 2010 @@ -960,9 +960,12 @@ elif call_type == 3: op = ops.CALL_FUNCTION_VAR_KW self.emit_op_arg(op, arg) + + def _call_has_no_star_args(self, call): + return not call.starargs and not call.kwargs def _call_has_simple_args(self, call): - return not call.starargs and not call.kwargs and not call.keywords + return self._call_has_no_star_args(call) and not call.keywords def _optimize_builtin_call(self, call): if not self.space.config.objspace.opcodes.CALL_LIKELY_BUILTIN or \ @@ -988,7 +991,7 @@ def _optimize_method_call(self, call): if not self.space.config.objspace.opcodes.CALL_METHOD or \ - not self._call_has_simple_args(call) or \ + not self._call_has_no_star_args(call) or \ not isinstance(call.func, ast.Attribute): return False attr_lookup = call.func @@ -1000,7 +1003,12 @@ arg_count = len(call.args) else: arg_count = 0 - self.emit_op_arg(ops.CALL_METHOD, arg_count) + if call.keywords: + self.visit_sequence(call.keywords) + kwarg_count = len(call.keywords) + else: + kwarg_count = 0 + self.emit_op_arg(ops.CALL_METHOD, (kwarg_count << 8) | arg_count) return True def _listcomp_generator(self, list_name, gens, gen_index, elt): Modified: pypy/trunk/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_compiler.py (original) +++ pypy/trunk/pypy/interpreter/test/test_compiler.py Fri Jul 30 22:12:00 2010 @@ -4,6 +4,7 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.error import OperationError from pypy.interpreter.argument import Arguments +from pypy.conftest import gettestobjspace class BaseTestCompiler: def setup_method(self, method): @@ -848,14 +849,38 @@ import StringIO, sys, dis s = StringIO.StringIO() + out = sys.stdout sys.stdout = s try: dis.dis(code) finally: - sys.stdout = sys.__stdout__ + sys.stdout = out output = s.getvalue() assert "LOAD_GLOBAL" not in output +class AppTestCallMethod(object): + def setup_class(cls): + cls.space = gettestobjspace(**{'objspace.opcodes.CALL_METHOD': True}) + + def test_call_method_kwargs(self): + source = """def _f(a): + return a.f(a=a) + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + out = sys.stdout + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = out + output = s.getvalue() + assert "CALL_METHOD" in output + + class AppTestExceptions: def test_indentation_error(self): source = """if 1: Modified: pypy/trunk/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/callmethod.py Fri Jul 30 22:12:00 2010 @@ -56,16 +56,41 @@ f.pushvalue(w_value) f.pushvalue(None) -def CALL_METHOD(f, nargs, *ignored): - # 'nargs' is the argument count excluding the implicit 'self' - w_self = f.peekvalue(nargs) - w_callable = f.peekvalue(nargs + 1) - n = nargs + (w_self is not None) - try: - w_result = f.space.call_valuestack(w_callable, n, f) - rstack.resume_point("CALL_METHOD", f, nargs, returns=w_result) - finally: - f.dropvalues(nargs + 2) +def CALL_METHOD(f, oparg, *ignored): + # opargs contains the arg, and kwarg count, excluding the implicit 'self' + n_args = oparg & 0xff + n_kwargs = (oparg >> 8) & 0xff + w_self = f.peekvalue(n_args + (2 * n_kwargs)) + w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) + n = n_args + (w_self is not None) + + if not n_kwargs: + try: + w_result = f.space.call_valuestack(w_callable, n, f) + rstack.resume_point("CALL_METHOD", f, n_args, returns=w_result) + finally: + f.dropvalues(n_args + 2) + else: + keywords = [None] * n_kwargs + keywords_w = [None] * n_kwargs + while True: + n_kwargs -= 1 + if n_kwargs < 0: + break + w_value = f.popvalue() + w_key = f.popvalue() + key = f.space.str_w(w_key) + keywords[n_kwargs] = key + keywords_w[n_kwargs] = w_value + + arguments = f.popvalues(n) + args = f.argument_factory(arguments, keywords, keywords_w, None, None) + + try: + w_result = f.space.call_args(w_callable, args) + rstack.resume_point("CALL_METHOD", f, returns=w_result) + finally: + f.dropvalues(1 + (w_self is None)) f.pushvalue(w_result) Modified: pypy/trunk/pypy/objspace/std/test/test_callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_callmethod.py Fri Jul 30 22:12:00 2010 @@ -106,6 +106,15 @@ else: raise Exception("did not raise?") """ + + def test_kwargs(self): + exec """if 1: + class C(object): + def f(self, a): + return a + 2 + + assert C().f(a=3) == 5 + """ class AppTestCallMethodWithGetattributeShortcut(AppTestCallMethod): From agaynor at codespeak.net Fri Jul 30 22:13:02 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Fri, 30 Jul 2010 22:13:02 +0200 (CEST) Subject: [pypy-svn] r76413 - pypy/branch/call-method-kwarg Message-ID: <20100730201302.AFD02282C0E@codespeak.net> Author: agaynor Date: Fri Jul 30 22:13:01 2010 New Revision: 76413 Removed: pypy/branch/call-method-kwarg/ Log: Remove merged call-method-kwarg branch. From getxsick at codespeak.net Sat Jul 31 14:43:31 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 31 Jul 2010 14:43:31 +0200 (CEST) Subject: [pypy-svn] r76414 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100731124331.746A7282C0E@codespeak.net> Author: getxsick Date: Sat Jul 31 14:43:29 2010 New Revision: 76414 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: fix translation Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Sat Jul 31 14:43:29 2010 @@ -77,6 +77,7 @@ calldescr = self.get_calldescr() self.looptoken = LoopToken() + bargs = list(bargs) # make sure it's not resized before ResOperation oplist = [ResOperation(rop.CALL, bargs, bres, descr=calldescr), ResOperation(rop.FINISH, [bres], None, descr=BasicFailDescr(0))] From getxsick at codespeak.net Sat Jul 31 15:13:26 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 31 Jul 2010 15:13:26 +0200 (CEST) Subject: [pypy-svn] r76415 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100731131326.ECAEB282C0E@codespeak.net> Author: getxsick Date: Sat Jul 31 15:13:25 2010 New Revision: 76415 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: clean up in get_calldescr Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Sat Jul 31 15:13:25 2010 @@ -88,9 +88,6 @@ self.setup_stack() def get_calldescr(self): - arg_classes = ''.join(self.args_type) - gccache = self.cpu.gc_ll_descr - if self.res_type == 'i': cls = SignedCallDescr elif self.res_type == 'f': @@ -103,6 +100,7 @@ raise NotImplementedError('Unknown type of descr: %s' % self.res_type) + arg_classes = ''.join(self.args_type) calldescr = cls(arg_classes) return calldescr From getxsick at codespeak.net Sat Jul 31 15:27:57 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 31 Jul 2010 15:27:57 +0200 (CEST) Subject: [pypy-svn] r76416 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100731132757.9CC07282C0E@codespeak.net> Author: getxsick Date: Sat Jul 31 15:27:56 2010 New Revision: 76416 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: don't use x86 CPU directly. use new (unfinished) pypy.rlib.jit.get_cpu function instead. Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Sat Jul 31 15:27:56 2010 @@ -1,13 +1,13 @@ from pypy.rlib import rdynload +from pypy.rlib.jit import get_cpu from pypy.rpython.lltypesystem import rffi, lltype from pypy.jit.backend.llsupport import descr, symbolic -from pypy.jit.backend.x86.runner import CPU from pypy.jit.metainterp.history import LoopToken, BasicFailDescr from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr, NULLBOX from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.typesystem import deref -GLOBAL_CPU = CPU(None, None) +GLOBAL_CPU = get_cpu() class CDLL(object): def __init__(self, name, load=True): From getxsick at codespeak.net Sat Jul 31 15:38:41 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 31 Jul 2010 15:38:41 +0200 (CEST) Subject: [pypy-svn] r76417 - pypy/branch/fast-ctypes/pypy/module/jitffi Message-ID: <20100731133841.C6BDC282C0E@codespeak.net> Author: getxsick Date: Sat Jul 31 15:38:39 2010 New Revision: 76417 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Log: fix Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Sat Jul 31 15:38:39 2010 @@ -89,7 +89,7 @@ raise OperationError( space.w_TypeError, space.wrap('Unsupported type of argument: %s' - % self.args_type[0])) + % self.rget.args_type[0])) i += 1 return self.rget.call(space.wrap) From agaynor at codespeak.net Sat Jul 31 17:24:50 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sat, 31 Jul 2010 17:24:50 +0200 (CEST) Subject: [pypy-svn] r76418 - in pypy/trunk/pypy: interpreter/astcompiler objspace/std Message-ID: <20100731152450.8567D282C0E@codespeak.net> Author: agaynor Date: Sat Jul 31 17:24:48 2010 New Revision: 76418 Modified: pypy/trunk/pypy/interpreter/astcompiler/assemble.py pypy/trunk/pypy/objspace/std/callmethod.py Log: FIx CALL_FUNCTION and family effect on stack. Modified: pypy/trunk/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/assemble.py Sat Jul 31 17:24:48 2010 @@ -566,22 +566,22 @@ return (oparg % 256) + 2 * (oparg / 256) def _compute_CALL_FUNCTION(arg): - return _num_args(arg) + return -_num_args(arg) def _compute_CALL_FUNCTION_VAR(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_KW(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_VAR_KW(arg): - return _num_args(arg) - 2 + return -_num_args(arg) - 2 def _compute_CALL_LIKELY_BUILTIN(arg): return -(arg & 0xFF) + 1 def _compute_CALL_METHOD(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 _stack_effect_computers = {} Modified: pypy/trunk/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/callmethod.py Sat Jul 31 17:24:48 2010 @@ -12,7 +12,7 @@ from pypy.interpreter import function from pypy.objspace.descroperation import object_getattribute -from pypy.rlib import rstack # for resume points +from pypy.rlib import jit, rstack # for resume points # This module exports two extra methods for StdObjSpaceFrame implementing # the LOOKUP_METHOD and CALL_METHOD opcodes in an efficient way, as well @@ -56,6 +56,7 @@ f.pushvalue(w_value) f.pushvalue(None) + at jit.unroll_safe def CALL_METHOD(f, oparg, *ignored): # opargs contains the arg, and kwarg count, excluding the implicit 'self' n_args = oparg & 0xff From getxsick at codespeak.net Sat Jul 31 17:27:38 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 31 Jul 2010 17:27:38 +0200 (CEST) Subject: [pypy-svn] r76419 - in pypy/branch/fast-ctypes/pypy: module/jitffi rlib Message-ID: <20100731152738.52671282C0E@codespeak.net> Author: getxsick Date: Sat Jul 31 17:27:36 2010 New Revision: 76419 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: temporary remove support for C pointers. it has never worked actually. Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Sat Jul 31 17:27:36 2010 @@ -83,8 +83,6 @@ self.rget.push_int(space.int_w(w_arg)) elif self.rget.args_type[i] == 'f': self.rget.push_float(space.float_w(w_arg)) - elif self.rget.args_type[i] == 'p': - self.rget.push_ref(space.int_w(w_arg)) else: raise OperationError( space.w_TypeError, Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Sat Jul 31 17:27:36 2010 @@ -3,7 +3,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.jit.backend.llsupport import descr, symbolic from pypy.jit.metainterp.history import LoopToken, BasicFailDescr -from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr, NULLBOX +from pypy.jit.metainterp.history import BoxInt, BoxFloat, NULLBOX from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.typesystem import deref @@ -59,8 +59,6 @@ bargs.append(BoxInt()) elif arg == 'f': bargs.append(BoxFloat()) - elif arg == 'p': - bargs.append(BoxPtr()) else: raise ValueError(arg) @@ -68,8 +66,6 @@ bres = BoxInt() elif self.res_type == 'f': bres = BoxFloat() - elif self.res_type == 'p': - bres = BoxPtr() elif self.res_type == 'v': bres = NULLBOX else: @@ -92,8 +88,6 @@ cls = SignedCallDescr elif self.res_type == 'f': cls = descr.FloatCallDescr - elif self.res_type == 'p': - cls = descr.NonGcPtrCallDescr elif self.res_type == 'v': cls = descr.VoidCallDescr else: @@ -111,8 +105,6 @@ r = push_result(self.cpu.get_latest_value_int(0)) elif self.res_type == 'f': r = push_result(self.cpu.get_latest_value_float(0)) - elif self.res_type == 'p': - r = push_result(self.cpu.get_latest_value_ref(0)) elif self.res_type == 'v': r = None else: @@ -134,10 +126,6 @@ self.cpu.set_future_value_float(self.esp, value) self.esp += 1 - def push_ref(self, value): - self.cpu.set_future_value_ref(self.esp, value) - self.esp += 1 - # ____________________________________________________________ # CallDescrs From getxsick at codespeak.net Sat Jul 31 19:17:01 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 31 Jul 2010 19:17:01 +0200 (CEST) Subject: [pypy-svn] r76420 - pypy/branch/fast-ctypes/pypy/module/jitffi Message-ID: <20100731171701.265B8282BEB@codespeak.net> Author: getxsick Date: Sat Jul 31 19:16:59 2010 New Revision: 76420 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Log: raise the proper exception Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Sat Jul 31 19:16:59 2010 @@ -84,8 +84,9 @@ elif self.rget.args_type[i] == 'f': self.rget.push_float(space.float_w(w_arg)) else: + # should never happen (raised earlier) raise OperationError( - space.w_TypeError, + space.w_ValueError, space.wrap('Unsupported type of argument: %s' % self.rget.args_type[0])) i += 1 From getxsick at codespeak.net Sat Jul 31 19:18:33 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 31 Jul 2010 19:18:33 +0200 (CEST) Subject: [pypy-svn] r76421 - pypy/branch/fast-ctypes/pypy/module/jitffi Message-ID: <20100731171833.BC6F4282BEB@codespeak.net> Author: getxsick Date: Sat Jul 31 19:18:32 2010 New Revision: 76421 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Log: comment out unused code Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Sat Jul 31 19:18:32 2010 @@ -92,11 +92,11 @@ i += 1 return self.rget.call(space.wrap) -def W_Get___new__(space, w_type, cpu, lib, func, args_type, res_type): - try: - return space.wrap(W_Get(space, w_type, cpu, lib, func, args_type, res_type)) - except OSError, e: - raise wrap_oserror(space, e) +#def W_Get___new__(space, w_type, cpu, lib, func, args_type, res_type): +# try: +# return space.wrap(W_Get(space, w_type, cpu, lib, func, args_type, res_type)) +# except OSError, e: +# raise wrap_oserror(space, e) W_Get.typedef = TypeDef( 'Get',